1.1--- a/.hgsubstate	Wed Jun 12 23:02:34 2024 -0400
     1.2+++ b/.hgsubstate	Fri Jun 14 19:20:13 2024 +0000
     1.3@@ -1,4 +1,4 @@
     1.4 0000000000000000000000000000000000000000 box
     1.5 d5b76348e00ed20d362eb1a4109abd45a784cd7b etc
     1.6 411f423567fcee89ca7f50fbeca938ada70c2cf1 home
     1.7-0000000000000000000000000000000000000000 pod
     1.8+98aa8d30816a26ccbb544ee8d156bfb40660a3bf pod

     2.1--- a/autogen.lisp	Wed Jun 12 23:02:34 2024 -0400
     2.2+++ b/autogen.lisp	Fri Jun 14 19:20:13 2024 +0000
     2.3@@ -18,14 +18,14 @@
     2.4 
     2.5 (defparameter *profile* (uiop:read-file-forms
     2.6                          (if-let ((profile (sb-posix:getenv "INFRA_PROFILE")))
     2.7-                           profile
     2.8-                           #P"default.sxp")))
     2.9+                                 profile
    2.10+                                 #P"default.sxp")))
    2.11 (defparameter *core* sb-ext:*core-pathname*)
    2.12 (defparameter *host* (uiop:read-file-forms
    2.13-                             (let ((hcfg (format nil "~a.sxp" (sb-unix:unix-gethostname))))
    2.14-                               (unless (probe-file hcfg)
    2.15-                                 (print #0$./check.sh$#))
    2.16-                               hcfg)))
    2.17+                      (let ((hcfg (format nil "~a.sxp" (sb-unix:unix-gethostname))))
    2.18+                        (unless (probe-file hcfg)
    2.19+                          (print #0$./check.sh$#))
    2.20+                        hcfg)))
    2.21 (defun gethost (k) (getf *host* k))
    2.22 (defun getprofile (k) (getf *profile* k))
    2.23 (init-skel-vars)
    2.24@@ -39,32 +39,33 @@
    2.25                              (setf (gethash k table) (sb-posix:getenv k)))))
    2.26 (defun getenv (k) (gethash *host-env* k))
    2.27 
    2.28-(info! "starting autogen.lisp" sb-ext:*core-pathname*)
    2.29-(terpri)
    2.30-(format t "core: ~A~%" *core*)
    2.31-(terpri)
    2.32-(println "host:")
    2.33-(loop for (k v) on *host* by 'cddr
    2.34-      do (format t "  ~A = ~A~%" k v))
    2.35-(println "env:")
    2.36-(loop for k being the hash-key
    2.37-      using (hash-value v) of *host-env*
    2.38-      do (format t "  ~A = ~:A~%" k v))
    2.39-(println "profile:")
    2.40-(loop for (k v) on *profile* by 'cddr
    2.41-      do (format t "  ~A = ~A~%" k v))
    2.42+(defun autogen ()
    2.43+  "Auto-generate the INFRA system."
    2.44+  (info! "starting autogen.lisp" sb-ext:*core-pathname*)
    2.45+  (terpri)
    2.46+  (format t "core: ~A~%" *core*)
    2.47+  (terpri)
    2.48+  (println "host:")
    2.49+  (loop for (k v) on *host* by 'cddr
    2.50+        do (format t "  ~A = ~A~%" k v))
    2.51+  (println "env:")
    2.52+  (loop for k being the hash-key
    2.53+        using (hash-value v) of *host-env*
    2.54+        do (format t "  ~A = ~:A~%" k v))
    2.55+  (println "profile:")
    2.56+  (loop for (k v) on *profile* by 'cddr
    2.57+        do (format t "  ~A = ~A~%" k v))
    2.58 
    2.59 ;;; init stash (via skel)
    2.60-(sk-call* *skel-project* :clean :src)
    2.61+  (sk-call* *skel-project* :clean :src)
    2.62 
    2.63-(let ((rocksdb-builder (sb-thread:make-thread (lambda () (sk-call* *skel-project* :rocksdb))))
    2.64-      (sbcl-builder (sb-thread:make-thread (lambda () (sk-call* *skel-project* :sbcl :sbcl-shared))))
    2.65-      (archlinux-pod-builder (sb-thread:make-thread (lambda () (sk-call *skel-project* :archlinux))))
    2.66-      (alpine-pod-builder (sb-thread:make-thread (lambda () (sk-call *skel-project* :alpine)))))
    2.67-  (std/thread:wait-for-threads
    2.68-   (list rocksdb-builder sbcl-builder archlinux-pod-builder alpine-pod-builder)))
    2.69+  (let ((rocksdb-builder (sb-thread:make-thread (lambda () (sk-call* *skel-project* :rocksdb))))
    2.70+        (sbcl-builder (sb-thread:make-thread (lambda () (sk-call* *skel-project* :sbcl :sbcl-shared))))
    2.71+        (archlinux-pod-builder (sb-thread:make-thread (lambda () (sk-call *skel-project* :archlinux :operator))))
    2.72+        (alpine-pod-builder (sb-thread:make-thread (lambda () (sk-call *skel-project* :alpine :worker)))))
    2.73+    (std/thread:wait-for-threads
    2.74+     (list rocksdb-builder sbcl-builder archlinux-pod-builder alpine-pod-builder))))
    2.75 ;;; *host*
    2.76 
    2.77 ;;; *profile*
    2.78-
    2.79-(sb-ext:quit)
    2.80+;; (sb-ext:quit)

     3.1--- a/bootstrap.sh	Wed Jun 12 23:02:34 2024 -0400
     3.2+++ b/bootstrap.sh	Fri Jun 14 19:20:13 2024 +0000
     3.3@@ -1,5 +1,6 @@
     3.4 #!/bin/sh
     3.5 set -eu
     3.6+
     3.7 main() {
     3.8   . ./check.sh
     3.9   download --check
    3.10@@ -19,46 +20,28 @@
    3.11   fi
    3.12   ensure mkdir -p "${_stash}/src"
    3.13   ensure mkdir -p "${_stash}/bin"
    3.14-  local _sk_url="${_url}/bin/sk"
    3.15+  cd "${_stash}"
    3.16+  hg clone https://vc.compiler.company/comp/core src/core
    3.17   local _sbcl_url="${_url}/pack/sbcl.tar.zst"
    3.18   local _rocksdb_url="${_url}/pack/rocksdb.tar.zst"
    3.19-  ensure download "$_sbcl_url" "${_stash}/src/sbcl.tar.zst" "$_arch"
    3.20-  ensure download "$_rocksdb_url" "${_stash}/src/rocksdb.tar.zst" "$_arch"
    3.21-  ensure download "$_sk_url" "${_stash}/bin/sk" "$_arch"
    3.22-  chmod +x "${_stash}/bin/sk"
    3.23-  say "${_stash}/bin/sk"
    3.24+  local _core_url="${_url}/pack/core.tar.zst"
    3.25+  ensure download "$_sbcl_url" "sbcl.tar.zst" "$_arch"
    3.26+  unzstd "sbcl.tar.zst"
    3.27+  tar -xvf "sbcl.tar"
    3.28+  ensure download "$_rocksdb_url" "rocksdb.tar.zst" "$_arch"
    3.29+  unzstd "rocksdb.tar.zst"
    3.30+  tar -xvf "rocksdb.tar"
    3.31+  chmod +x bin/*
    3.32+  say "${_stash}/src/sbcl"
    3.33+  say "${_stash}/src/rocksdb"
    3.34+  say "${_stash}/bin/cl"
    3.35+  rm -rf *.tar*
    3.36 }
    3.37 
    3.38 _read() {
    3.39   grep ":$1" $INFRA_HOST_CONFIG | cut -d' ' -f 2-
    3.40 }
    3.41 
    3.42-say() {
    3.43-  printf 'bootstrap.sh: %s\n' "$1"
    3.44-}
    3.45-
    3.46-err() {
    3.47-  say "$1" >&2
    3.48-  exit 1
    3.49-}
    3.50-
    3.51-check_cmd() {
    3.52-  command -v "$1" > /dev/null 2>&1
    3.53-}
    3.54-
    3.55-need_cmd() {
    3.56-  if ! check_cmd "$1"; then
    3.57-    err "need '$1' (command not found)"
    3.58-  fi
    3.59-}
    3.60-
    3.61-# Run a command that should never fail. If the command fails execution
    3.62-# will immediately terminate with an error showing the failing
    3.63-# command.
    3.64-ensure() {
    3.65-  if ! "$@"; then err "command failed: $*"; fi
    3.66-}
    3.67-
    3.68 # Check if curl supports the --retry flag, then pass it to the curl invocation.
    3.69 check_curl_for_retry_support() {
    3.70   local _retry_supported=""
    3.71@@ -214,64 +197,6 @@
    3.72   fi
    3.73 }
    3.74 
    3.75-check_proc() {
    3.76-  # Check for /proc by looking for the /proc/self/exe link
    3.77-  # This is only run on Linux
    3.78-  if ! test -L /proc/self/exe ; then
    3.79-    err "fatal: Unable to find /proc/self/exe.  Is /proc mounted?  Installation cannot proceed without /proc."
    3.80-  fi
    3.81-}
    3.82-
    3.83-get_bitness() {
    3.84-  need_cmd head
    3.85-  # Architecture detection without dependencies beyond coreutils.
    3.86-  # ELF files start out "\x7fELF", and the following byte is
    3.87-  #   0x01 for 32-bit and
    3.88-  #   0x02 for 64-bit.
    3.89-  # The printf builtin on some shells like dash only supports octal
    3.90-  # escape sequences, so we use those.
    3.91-  local _current_exe_head
    3.92-  _current_exe_head=$(head -c 5 /proc/self/exe )
    3.93-  if [ "$_current_exe_head" = "$(printf '\177ELF\001')" ]; then
    3.94-    echo 32
    3.95-  elif [ "$_current_exe_head" = "$(printf '\177ELF\002')" ]; then
    3.96-    echo 64
    3.97-  else
    3.98-    err "unknown platform bitness"
    3.99-  fi
   3.100-}
   3.101-
   3.102-is_host_amd64_elf() {
   3.103-  need_cmd head
   3.104-  need_cmd tail
   3.105-  # ELF e_machine detection without dependencies beyond coreutils.
   3.106-  # Two-byte field at offset 0x12 indicates the CPU,
   3.107-  # but we're interested in it being 0x3E to indicate amd64, or not that.
   3.108-  local _current_exe_machine
   3.109-  _current_exe_machine=$(head -c 19 /proc/self/exe | tail -c 1)
   3.110-  [ "$_current_exe_machine" = "$(printf '\076')" ]
   3.111-}
   3.112-
   3.113-get_endianness() {
   3.114-  local cputype=$1
   3.115-  local suffix_eb=$2
   3.116-  local suffix_el=$3
   3.117-
   3.118-  # detect endianness without od/hexdump, like get_bitness() does.
   3.119-  need_cmd head
   3.120-  need_cmd tail
   3.121-
   3.122-  local _current_exe_endianness
   3.123-  _current_exe_endianness="$(head -c 6 /proc/self/exe | tail -c 1)"
   3.124-  if [ "$_current_exe_endianness" = "$(printf '\001')" ]; then
   3.125-    echo "${cputype}${suffix_el}"
   3.126-  elif [ "$_current_exe_endianness" = "$(printf '\002')" ]; then
   3.127-    echo "${cputype}${suffix_eb}"
   3.128-  else
   3.129-    err "unknown platform endianness"
   3.130-  fi
   3.131-}
   3.132-
   3.133 # This wraps curl or wget. Try curl first, if not installed,
   3.134 # use wget instead.
   3.135 download() {

     4.1--- a/containers/Containerfile.alpine	Wed Jun 12 23:02:34 2024 -0400
     4.2+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
     4.3@@ -1,2 +0,0 @@
     4.4-FROM alpine:latest
     4.5-RUN apk add --no-cache openssh mercurial wireguard-tools liburing libc6-compat coreutils
     4.6\ No newline at end of file

     5.1--- a/containers/Containerfile.archlinux	Wed Jun 12 23:02:34 2024 -0400
     5.2+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
     5.3@@ -1,4 +0,0 @@
     5.4-FROM archlinux:latest
     5.5-RUN --network=host pacman-key --init
     5.6-RUN --network=host pacman-key --populate archlinux
     5.7-RUN --network=host pacman -Syu base-devel less uutils-coreutils openssh mercurial git sqlite btrfs-progs liburing wireguard-tools clang tree-sitter libxcb glibc openssl zstd --noconfirm
     5.8\ No newline at end of file

     6.1--- a/containers/Containerfile.archriscv	Wed Jun 12 23:02:34 2024 -0400
     6.2+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
     6.3@@ -1,2 +0,0 @@
     6.4-FROM --platform=linux/riscv64 riscfive/archlinux
     6.5-RUN pacman -Syu
     6.6\ No newline at end of file

     8.1--- a/containers/Containerfile.box	Wed Jun 12 23:02:34 2024 -0400
     8.2+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
     8.3@@ -1,52 +0,0 @@
     8.4-FROM localhost/archlinux:latest
     8.5-RUN --network=host pacman -Sy uutils-coreutils sysstat iotop imagemagick giflib libgccjit libxaw snappy sbcl liburing libxkbcommon --noconfirm
     8.6-# custom
     8.7-ARG DEV=dev
     8.8-ARG DEV_HOME=/home/${DEV}
     8.9-ARG ID=808
    8.10-# stage0
    8.11-RUN mkdir -p /usr/share/lisp /usr/local/share/lisp
    8.12-WORKDIR /usr/local/src
    8.13-RUN hg clone https://vc.compiler.company/comp/infra
    8.14-RUN hg clone https://vc.compiler.company/comp/core
    8.15-WORKDIR infra
    8.16-ENV CARGO_HOME="/usr/local/share/cargo"
    8.17-ENV RUSTUP_HOME="/usr/local/share/rustup"
    8.18-# stage1
    8.19-RUN ./bootstrap.sh
    8.20-ENV PATH="$PATH:/usr/local/share/cargo/bin"
    8.21-RUN .stash/bin/sk make rustup
    8.22-RUN rustup update
    8.23-# dev rustc components
    8.24-RUN rustup default nightly
    8.25-RUN rustup component add --toolchain nightly rust-src rust-analyzer rustc-dev llvm-tools-preview
    8.26-RUN .stash/bin/sk make sbcl
    8.27-RUN .stash/bin/sk make quicklisp
    8.28-RUN .stash/bin/sk make sbcl-install
    8.29-RUN .stash/bin/sk make tree-sitter
    8.30-RUN .stash/bin/sk make tree-sitter-langs
    8.31-RUN .stash/bin/sk make emacs-mini
    8.32-RUN .stash/bin/sk make emacs-install
    8.33-RUN .stash/bin/sk make rocksdb
    8.34-RUN .stash/bin/sk make rocksdb-install
    8.35-#ADD etc/sbclrc /etc/sbclrc
    8.36-#COPY etc/skel/ /etc/skel/
    8.37-#ADD etc/shells /etc/
    8.38-RUN useradd --system -s /usr/bin/bash -u ${ID} -k /etc/skel -m ${DEV}
    8.39-RUN groupadd -g 128 engineer -U ${DEV}
    8.40-RUN groupadd -g 256 operator -U ${DEV}
    8.41-# RUN make core-install
    8.42-WORKDIR /usr/local/src
    8.43-RUN cp -rf core/lisp /usr/local/share/lisp/core
    8.44-RUN rm -rf ./*
    8.45-RUN chown -R ${DEV}:engineer /usr/local/share/
    8.46-RUN chown -R ${DEV}:engineer /usr/local/src/
    8.47-RUN chown -R ${DEV}:engineer /usr/local/bin/
    8.48-RUN chown -R ${DEV}:engineer /usr/local/etc/
    8.49-RUN chown -R ${DEV}:engineer /usr/local/man/
    8.50-USER ${DEV}
    8.51-WORKDIR ${DEV_HOME}
    8.52-RUN hg clone https://vc.compiler.company/comp/infra
    8.53-RUN hg clone https://vc.compiler.company/comp/core
    8.54-ENV DEV=${DEV}
    8.55-ENV DEV_ID=${ID}
    8.56\ No newline at end of file

     9.1--- a/containers/Containerfile.fedora.arm64v8	Wed Jun 12 23:02:34 2024 -0400
     9.2+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
     9.3@@ -1,1 +0,0 @@
     9.4-FROM arm64v8/fedora:latest
     9.5\ No newline at end of file

    10.1--- a/containers/Containerfile.mail	Wed Jun 12 23:02:34 2024 -0400
    10.2+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
    10.3@@ -1,1 +0,0 @@
    10.4-FROM stalwartlabs/mail-server:latest
    10.5\ No newline at end of file

    11.1--- a/containers/Containerfile.operator	Wed Jun 12 23:02:34 2024 -0400
    11.2+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
    11.3@@ -1,2 +0,0 @@
    11.4-FROM localhost/box:latest
    11.5-RUN infra/scripts/wg-gen-keys.sh private.key public.key
    11.6\ No newline at end of file

    12.1--- a/containers/Containerfile.ubuntu	Wed Jun 12 23:02:34 2024 -0400
    12.2+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
    12.3@@ -1,3 +0,0 @@
    12.4-FROM ubuntu:latest
    12.5-RUN apt-get update
    12.6-RUN apt -y install git ssh python3-pip mercurial

    13.1--- a/containers/Containerfile.vc	Wed Jun 12 23:02:34 2024 -0400
    13.2+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
    13.3@@ -1,4 +0,0 @@
    13.4-FROM octobus/heptapod:latest
    13.5-RUN gitlab-ctl upgrade
    13.6-# TODO: configure instance here, pull from comp/etc
    13.7-# COPY etc/gitlab/gitlab.rb /etc/gitlab/gitlab.rb
    13.8\ No newline at end of file

    14.1--- a/containers/Containerfile.vc-runner	Wed Jun 12 23:02:34 2024 -0400
    14.2+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
    14.3@@ -1,8 +0,0 @@
    14.4-FROM localhost/worker:latest
    14.5-USER root
    14.6-RUN curl -L --output /usr/local/bin/gitlab-runner https://download.heptapod.net/runner/runner/latest/binaries/heptapod-runner-linux-amd64
    14.7-ARG RUNNER_TOKEN
    14.8-RUN chmod +x /usr/local/bin/gitlab-runner
    14.9-RUN echo ${WORKER_USER}
   14.10-RUN echo ${WORKER_HOME}
   14.11-RUN gitlab-runner install --user=${WORKER_USER} --working-directory=${WORKER_HOME}
   14.12\ No newline at end of file

    15.1--- a/containers/Containerfile.worker	Wed Jun 12 23:02:34 2024 -0400
    15.2+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
    15.3@@ -1,30 +0,0 @@
    15.4-FROM localhost/alpine:latest
    15.5-# VOLUME /opt/store
    15.6-# VOLUME /opt/stash
    15.7-# ADD etc/skel/* /etc/skel/
    15.8-ARG ID=908
    15.9-ARG WORKER=worker
   15.10-ARG WORKER_HOME=/home/${WORKER}
   15.11-# TODO: build sbcl for musl
   15.12-RUN apk add build-base zstd-dev curl make git linux-headers openssl-dev pkgconf clang m4 tree-sitter sbcl
   15.13-RUN mkdir /usr/share/lisp
   15.14-RUN mkdir /usr/local/share/lisp
   15.15-WORKDIR /usr/local/src
   15.16-RUN hg clone https://vc.compiler.company/comp/infra
   15.17-RUN hg clone https://vc.compiler.company/comp/core
   15.18-WORKDIR infra
   15.19-RUN scripts/get-sk.sh
   15.20-ENV CARGO_HOME="/usr/local/share/cargo"
   15.21-ENV RUSTUP_HOME="/usr/local/share/rustup"
   15.22-ENV PATH="$PATH:/usr/local/share/cargo/bin:/usr/local/share/lisp/bin"
   15.23-# RUN sk make sbcl sbcl-install quicklisp rustup tree-sitter-langs rocksdb
   15.24-RUN make sbcl-install-gencgc
   15.25-RUN make quicklisp-install
   15.26-RUN make rocksdb-install
   15.27-RUN adduser -D -u ${ID} -k /etc/skel ${WORKER}
   15.28-USER ${WORKER}
   15.29-WORKDIR ${WORKER_HOME}
   15.30-ENV WORKER=${WORKER}
   15.31-ENV WORKER_ID=${ID}
   15.32-ENV WORKER_HOME=${WORKER_HOME}
   15.33-ENTRYPOINT ["sh"]
   15.34\ No newline at end of file

    16.1--- a/infra.asd	Wed Jun 12 23:02:34 2024 -0400
    16.2+++ b/infra.asd	Fri Jun 14 19:20:13 2024 +0000
    16.3@@ -1,5 +1,5 @@
    16.4 (defsystem :infra
    16.5   :depends-on (:std :dat :cli :skel :log :net :packy)
    16.6-  :components ((:file "bootstrap")
    16.7+  :components ((:file "autogen")
    16.8                (:file "deploy"))
    16.9-  :build-pathname "infra")
   16.10+  :build-pathname ".stash/bin/infra")

    17.1--- a/skelfile	Wed Jun 12 23:02:34 2024 -0400
    17.2+++ b/skelfile	Fri Jun 14 19:20:13 2024 +0000
    17.3@@ -14,12 +14,12 @@
    17.4 ((check () #$./check.sh$#)
    17.5  (clean () #$rm -rvf .stash$#)
    17.6  ;; containers
    17.7- (archlinux () #$podman build -f containers/Containerfile.archlinux --no-cache -t localhost/archlinux$#)
    17.8- (alpine () #$podman build -f containers/Containerfile.alpine --no-cache -t localhost/alpine$#)
    17.9- (ubuntu () #$podman build -f containers/Containerfile.alpine --no-cache -t localhost/alpine$#)
   17.10- (box (archlinux) #$podman build -f containers/Containerfile.box --no-cache -t localhost/box$#)
   17.11- (worker (alpine) #$podman build -f containers/Containerfile.worker --no-cache -t localhost/worker$#)
   17.12- (operator (box) #$podman build -f containers/Containerfile.operator --no-cache -t localhost/operator$#)
   17.13+ (archlinux () #$podman build -f pod/Containerfile.archlinux --no-cache -t localhost/archlinux$#)
   17.14+ (alpine () #$podman build -f pod/Containerfile.alpine --no-cache -t localhost/alpine$#)
   17.15+ (ubuntu () #$podman build -f pod/Containerfile.alpine --no-cache -t localhost/alpine$#)
   17.16+ (box (archlinux) #$podman build -f pod/Containerfile.box --no-cache -t localhost/box$#)
   17.17+ (worker (alpine) #$podman build -f pod/Containerfile.worker --no-cache -t localhost/worker$#)
   17.18+ (operator (box) #$podman build -f pod/Containerfile.operator --no-cache -t localhost/operator$#)
   17.19  (containers (archlinux alpine ubuntu box worker operator))
   17.20  ;; programs
   17.21  (src () #$mkdir -pv .stash/src$#)