1.1--- a/makefile Mon Nov 27 18:26:13 2023 -0500
1.2+++ b/makefile Mon Nov 27 22:33:13 2023 -0500
1.3@@ -91,7 +91,7 @@
1.4 ### Web
1.5
1.6 ### Dist
1.7-dist/code:scripts/bundle.sh
1.8+dist/comp:scripts/bundle-dir.sh
1.9 $<
1.10
1.11 dist/cdn:cdn
1.12@@ -105,6 +105,9 @@
1.13
1.14 dist/emacs:emacs;
1.15
1.16-dist:dist/code dist/cdn dist/sbcl dist/linux dist/rocksdb
1.17+dist:dist/comp dist/cdn dist/sbcl dist/linux dist/rocksdb
1.18
1.19 dist-clean::;rm -rf dist/*
1.20+
1.21+### Quickstart
1.22+quick:code
2.1--- a/readme.org Mon Nov 27 18:26:13 2023 -0500
2.2+++ b/readme.org Mon Nov 27 22:33:13 2023 -0500
2.3@@ -7,10 +7,54 @@
2.4 This project contains scripts, utils, and applications used to
2.5 host and maintain The Compiler Company infrastructure.
2.6
2.7-* Usage
2.8+* Quickstart
2.9+- Dependencies
2.10+ - Mercurial
2.11+ - Make
2.12 #+begin_src shell
2.13-# ( linux emacs rocksdb sbcl rust code virt dist )
2.14-make
2.15-make install
2.16-# make deploy
2.17+sudo pacman -Sy make mercurial
2.18+# make quick
2.19+# make install
2.20+#+end_src
2.21+* Scripts
2.22+** [[file:scripts/eval.sh][eval.sh]]
2.23+Eval a lisp script using SBCL.
2.24+** [[file:scripts/check.lisp][check.lisp]]
2.25+Check the host for compatibility.
2.26+#+begin_src shell :results pp replace :exports both
2.27+./scripts/eval.sh ./scripts/check.lisp
2.28 #+end_src
2.29+
2.30+#+RESULTS:
2.31+#+begin_example
2.32+
2.33+#HOST-FEATURES{
2.34+(lisp . (SBCL 2.3.8
2.35+ (CL-PPCRE NAMED-READTABLES ASDF3.3 ASDF3.2 ASDF3.1 ASDF3 ASDF2 ASDF
2.36+ OS-UNIX NON-BASE-CHARS-EXIST-P ASDF-UNICODE ARENA-ALLOCATOR X86-64
2.37+ GENCGC 64-BIT ANSI-CL COMMON-LISP ELF IEEE-FLOATING-POINT LINUX
2.38+ LITTLE-ENDIAN PACKAGE-LOCAL-NICKNAMES SB-CORE-COMPRESSION SB-LDB
2.39+ SB-PACKAGE-LOCKS SB-THREAD SB-UNICODE SBCL UNIX)))
2.40+(shell . /home/ellis/dev/comp/infra)
2.41+(rocksdb . librocksdb.so)
2.42+(uring . liburing.so)
2.43+(btrfs . libbtrfs.so)
2.44+(btrfsutil . libbtrfsutil.so)
2.45+(tree-sitter . libtree-sitter.so)
2.46+(gtk-4 . libgtk-4.so)
2.47+(blake3 . libblake3.so)
2.48+(k . libk.so)
2.49+(cbqn . libcbqn.so)}
2.50+#+end_example
2.51+** [[file:scripts/autogen.lisp][autogen.lisp]]
2.52+** get-*
2.53+Pull and build dependencies.
2.54+** [[file:scripts/git-vendor-pull.sh][git-vendor-pull.sh]]
2.55+Pull an upstream vendored repository from Git and push it to our
2.56+private mirror.
2.57+
2.58+#+begin_src shell :noeval t :exports code
2.59+scripts/git-vendor-pull.sh git.savannah.gnu.org/git emacs master
2.60+#+end_src
2.61+** [[file:scripts/bundle-dir.sh][bundle-dir.sh]]
2.62+Bundle a local directory of source-code repos.
3.1--- /dev/null Thu Jan 01 00:00:00 1970 +0000
3.2+++ b/scripts/autogen.lisp Mon Nov 27 22:33:13 2023 -0500
3.3@@ -0,0 +1,9 @@
3.4+;;; scripts/autogen.lisp --- prepare the local environment
3.5+
3.6+;;; Code:
3.7+(in-package :cl-user)
3.8+(use-package :std)
3.9+
3.10+(defmain ()
3.11+ "Prepare the local environment for bootstrapping a complete system."
3.12+ (format t "Starting scripts/autogen.lisp...~%"))
4.1--- /dev/null Thu Jan 01 00:00:00 1970 +0000
4.2+++ b/scripts/bundle-dir.sh Mon Nov 27 22:33:13 2023 -0500
4.3@@ -0,0 +1,34 @@
4.4+#!/bin/bash
4.5+# bundle a tar.zst archive of Mercurial repositories.
4.6+#WD=/mnt/y/data/packy
4.7+WD=$(realpath dist)
4.8+OUT=$WD/bundle/src
4.9+SRC_PATH=$HOME/dev/comp
4.10+BUNDLE_NAME="${1:-comp}"
4.11+echo "Bundling $BUNDLE_NAME in $OUT..."
4.12+
4.13+rm -rf $OUT/*
4.14+rm -rf $WD/$BUNDLE_NAME.*
4.15+mkdir -pv $WD/vc/{zst,stream}
4.16+
4.17+pushd $SRC_PATH
4.18+# Find all mercurial repositories, create bundles and dump them to $OUT dir
4.19+for i in $(find . -name ".hg" | cut -c 3-); do
4.20+ echo "";
4.21+ echo $i;
4.22+
4.23+ cd "$i";
4.24+ cd ..;
4.25+ hg bundle -a -t zstd-v2 $WD/vc/zst/$(basename $(hg root)).hg.zst;
4.26+ hg bundle -a -t none-v2 $WD/vc/$(basename $(hg root)).hg;
4.27+ hg debugcreatestreamclonebundle $WD/vc/$(basename $(hg root)).hg.stream;
4.28+ echo "... Done.";
4.29+ cd $SRC_PATH
4.30+done
4.31+popd
4.32+pushd $WD
4.33+tar -I 'zstd' -cf $OUT/$BUNDLE_NAME.tar.zst vc/zst/*.hg.zst
4.34+# tar -cf $OUT/$BUNDLE_NAME.tar.stream vc/stream/*.hg.stream
4.35+# tar -cf $OUT/$BUNDLE_NAME.tar vc/*.hg
4.36+echo "Done."
4.37+popd
5.1--- a/scripts/bundle.sh Mon Nov 27 18:26:13 2023 -0500
5.2+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
5.3@@ -1,35 +0,0 @@
5.4-#!/bin/bash
5.5-# bundle a tar.zst archive of Mercurial repositories.
5.6-#WD=/mnt/y/data/packy
5.7-WD=$(realpath dist)
5.8-OUT=$WD/code
5.9-SRC_PATH=$HOME/dev/comp
5.10-BUNDLE_NAME=code
5.11-echo "Bundling $BUNDLE_NAME in $OUT..."
5.12-
5.13-rm -rf $OUT/*
5.14-rm -rf $WD/$BUNDLE_NAME.*
5.15-mkdir -pv $OUT/{zst,stream}
5.16-
5.17-pushd $SRC_PATH
5.18-
5.19-# Find all mercurial repositories, create bundles and dump them to $OUT dir
5.20-for i in $(find . -name ".hg" | cut -c 3-); do
5.21- echo "";
5.22- echo $i;
5.23-
5.24- cd "$i";
5.25- cd ..;
5.26- hg bundle -a -t zstd-v2 $OUT/zst/$(basename $(hg root)).hg.zst;
5.27- hg bundle -a -t none-v2 $OUT/$(basename $(hg root)).hg;
5.28- hg debugcreatestreamclonebundle $OUT/stream/$(basename $(hg root)).hg.stream;
5.29- echo "... Done.";
5.30- cd $SRC_PATH
5.31-done
5.32-popd
5.33-pushd $WD
5.34-tar -I 'zstd --ultra -22' -cf $BUNDLE_NAME.tar.zst code/zst/*.hg.zst
5.35-tar -cf $BUNDLE_NAME.tar.stream code/stream/*.hg.stream
5.36-tar -cf $BUNDLE_NAME.tar code/*.hg
5.37-echo "Done."
5.38-popd
6.1--- /dev/null Thu Jan 01 00:00:00 1970 +0000
6.2+++ b/scripts/check.lisp Mon Nov 27 22:33:13 2023 -0500
6.3@@ -0,0 +1,105 @@
6.4+;;; scripts/check.lisp --- check host features
6.5+
6.6+;; This script runs some basic checks to ensure the host is ready for
6.7+;; bootstrapping.
6.8+
6.9+;;; Code:
6.10+(require 'asdf)
6.11+(require 'sb-posix)
6.12+(asdf:load-asd #P"~/dev/comp/core/lisp/std/std.asd")
6.13+(asdf:load-system :std)
6.14+
6.15+(defpackage :infra/scripts/check
6.16+ (:use :cl :std :sb-alien)
6.17+ (:export :main
6.18+ :*results*))
6.19+
6.20+(in-package :infra/scripts/check)
6.21+
6.22+(defvar *results* (make-hash-table :size 32 :test 'equal))
6.23+
6.24+(defparameter *library-path* (ld-library-path-list))
6.25+(defparameter *exec-path* (exec-path-list))
6.26+
6.27+(defun get-result (k) (gethash k *results*))
6.28+
6.29+(defun push-result (k &optional v)
6.30+ (setf (gethash k *results*) v))
6.31+
6.32+(defun check-for-shared-lib (name)
6.33+ (if-let ((lib (ignore-errors (load-shared-object (format nil "lib~a.so" name)))))
6.34+ (prog1
6.35+ (push-result name lib)
6.36+ (unload-shared-object lib))
6.37+ (push-result name)))
6.38+
6.39+(defun check-for-bin (name)
6.40+ (push-result name))
6.41+
6.42+(defun check-for-src (name)
6.43+ (push-result name))
6.44+
6.45+(defun check-system ()
6.46+ (destructuring-bind (lisp version features)
6.47+ (my-lisp-implementation)
6.48+ (push-result "lisp" (list lisp version features)
6.49+ #-sbcl (push-result "lisp"))))
6.50+
6.51+(defun check-shell ()
6.52+ (push-result "shell" (sb-posix:getcwd)))
6.53+
6.54+(defun check-display ()
6.55+ (push-result "display"))
6.56+
6.57+(defun check-net ()
6.58+ (push-result "net"))
6.59+
6.60+(defun check-cpu ()
6.61+ (push-result "cpu"))
6.62+
6.63+(defmethod print-object ((object hash-table) stream)
6.64+ (format stream "#HOST-FEATURES{~{~{~%(~a . ~a)~}~^ ~}}"
6.65+ (loop for k being the hash-keys of *results*
6.66+ using (hash-value v)
6.67+ unless (null v)
6.68+ collect (list k v))))
6.69+
6.70+(defmain ()
6.71+ "Check the host for required features."
6.72+
6.73+ (setq *library-path* (ld-library-path-list)
6.74+ *exec-path* (exec-path-list))
6.75+
6.76+ (debug! (format nil "LD_LIBRARY_PATH: ~A~%" *library-path*))
6.77+ (debug! (format nil "PATH: ~A~%" *exec-path*))
6.78+ (check-system)
6.79+ (check-shell)
6.80+
6.81+ (check-for-bin "sbcl")
6.82+ (check-for-bin "rustc")
6.83+ (check-for-bin "clang")
6.84+ (check-for-bin "gcc")
6.85+ (check-for-bin "emacs")
6.86+ ;; vcs
6.87+ (check-for-bin "hg")
6.88+ (check-for-bin "git")
6.89+ ;; virt
6.90+ (check-for-bin "podman")
6.91+
6.92+ (check-for-src "core")
6.93+ ;; core dependencies
6.94+ (check-for-shared-lib "rocksdb")
6.95+ (check-for-shared-lib "uring")
6.96+ (check-for-shared-lib "btrfs")
6.97+ (check-for-shared-lib "btrfsutil")
6.98+ (check-for-shared-lib "tree-sitter")
6.99+ ;; extras
6.100+ (check-for-shared-lib "sbcl") ;; requires compiling sbcl as shared_lib
6.101+ (check-for-shared-lib "gtk-4")
6.102+ (check-for-shared-lib "blake3")
6.103+ (check-for-shared-lib "k")
6.104+ (check-for-shared-lib "cbqn")
6.105+ (print *results*)
6.106+ *results*)
6.107+
6.108+(main)
7.1--- /dev/null Thu Jan 01 00:00:00 1970 +0000
7.2+++ b/scripts/eval.sh Mon Nov 27 22:33:13 2023 -0500
7.3@@ -0,0 +1,10 @@
7.4+#!/bin/sh
7.5+
7.6+# eval lisp code
7.7+LISP="/usr/bin/sbcl"
7.8+args=""
7.9+FILE="${1}"
7.10+[[ -z "$FILE" ]] && : || args="--script $FILE "
7.11+FORM=${2}
7.12+[[ -z "$FORM" ]] && : || args="--eval $FORM "
7.13+$LISP $args --eval "(quit)"
8.1--- a/scripts/get-code.sh Mon Nov 27 18:26:13 2023 -0500
8.2+++ b/scripts/get-code.sh Mon Nov 27 22:33:13 2023 -0500
8.3@@ -4,7 +4,7 @@
8.4 OUT=build/comp-$(date "+%Y%m%d")
8.5 mkdir -pv $OUT
8.6 pushd $OUT
8.7-wget https://packy.compiler.company/$SRC.tar.zst
8.8+wget https://packy.compiler.company/bundle/src/$SRC.tar.zst
8.9 unzstd $SRC.tar.zst
8.10 tar -xvf $SRC.tar
8.11 rm -rf $SRC.tar.zst $SRC.tar
9.1--- a/scripts/git-vendor-pull.sh Mon Nov 27 18:26:13 2023 -0500
9.2+++ b/scripts/git-vendor-pull.sh Mon Nov 27 22:33:13 2023 -0500
9.3@@ -1,10 +1,12 @@
9.4 #!/usr/bin/bash
9.5 # scripts/git-vendor-pull.sh git.savannah.gnu.org/git emacs master
9.6-DOMAIN="${1}"
9.7-REMOTE="https://$DOMAIN/${2}"
9.8-REPO="ssh://git@vc.compiler.company/packy/shed/vendor/${2}"
9.9-BRANCH="${3:-master}"
9.10-OUT="${4:-build/${2}}"
9.11+NAME="${1}"
9.12+DOMAIN="${2}"
9.13+REMOTE="https://$DOMAIN/$NAME"
9.14+BRANCH="${2:-master}"
9.15+REPO="ssh://git@vc.compiler.company/packy/shed/vendor/${1}"
9.16+OUT="${3:-build/src/${1}}"
9.17+mkdir -pv build/src
9.18 git clone $REPO $OUT
9.19 pushd $OUT
9.20 git pull $REMOTE $BRANCH
10.1--- /dev/null Thu Jan 01 00:00:00 1970 +0000
10.2+++ b/skelfile Mon Nov 27 22:33:13 2023 -0500
10.3@@ -0,0 +1,4 @@
10.4+;;; skelfile --- infra skelfile -*- mode: skel; -*-
10.5+:name "infra"
10.6+:description "The Compiler Company Infrastructure"
10.7+:author "Richard Westhaver <ellis@rwest.io>"
