1.1--- a/makefile Mon Dec 04 21:28:10 2023 -0500
1.2+++ b/makefile Tue Dec 05 19:03:29 2023 -0500
1.3@@ -1,26 +1,33 @@
1.4 ### infra/makefile --- The Compiler Company Infrastructure
1.5-__ := $(.VARIABLES)
1.6+
1.7+# this makefile is used to build all of our source code, package it,
1.8+# and ship it to various locations. It is for internal-use only.
1.9+
1.10+# If possible, use the packy (packy.compiler.company) to find and
1.11+# download a prepared bundle that suits your project needs.
1.12+
1.13+# You probably don't want to install all targets unless you have lots
1.14+# of time and a powerful machine, although it is possible. Instead,
1.15+# just run the targets for the components you are missing in your
1.16+# local environment (compatible compiler versions, shared libraries,
1.17+# etc)
1.18+
1.19+VERSION="0.1.0"
1.20 LINUX_VERSION:=$(shell uname -r | cut -d- -f1)
1.21 EMACS_VERSION:=main
1.22 ROCKSDB_VERSION:=main
1.23 SBCL_VERSION:=main
1.24 RUST_VERSION:=main
1.25 B:=build
1.26-D:=$(realpath dist)
1.27+D:=dist
1.28 SRC:=comp
1.29-SHELL=/bin/sh
1.30-UNAME=$(shell uname)
1.31-CURL:=curl
1.32-CPU_COUNT:=$(shell getconf _NPROCESSORS_ONLN)
1.33 HG_COMMIT:=$(shell hg id -i)
1.34-VERSION:=
1.35
1.36-VARS:=$(foreach v,$(filter-out $(__) __,$(.VARIABLES)),"\n$(v) = $($(v))")
1.37-
1.38-all:linux emacs rocksdb sbcl rust code virt dist;
1.39-
1.40-clean:clean-linux clean-code clean-sbcl clean-dist;
1.41-
1.42+init:sbcl rust emacs rocksdb code virt;
1.43+dist:dist/bundle dist/cdn dist/sbcl dist/rocksdb # dist/linux dist/rust
1.44+clean:;rm -rf $(B) $(D)
1.45+$(B):;mkdir -pv $@/src
1.46+$(D):;mkdir -pv $@
1.47 ### Linux
1.48 LINUX_TARGET:=linux-$(LINUX_VERSION)
1.49 linux:$(LINUX_TARGET) linux-config;
1.50@@ -36,8 +43,8 @@
1.51
1.52 ### Emacs
1.53 EMACS_TARGET:=build/src/emacs-$(EMACS_VERSION)
1.54-EMACS_DIST:=$(DIST)/src/emacs
1.55-emacs:scripts/get-emacs.sh;
1.56+EMACS_DIST:=$(D)/src/emacs
1.57+emacs:$(B) scripts/get-emacs.sh;
1.58 $< $(EMACS_VERSION)
1.59
1.60 emacs-build:emacs scripts/build-emacs.sh;
1.61@@ -59,7 +66,7 @@
1.62
1.63 ### SBCL
1.64 SBCL_TARGET:=build/src/sbcl-$(SBCL_VERSION)
1.65-sbcl:scripts/get-sbcl.sh;
1.66+sbcl:scripts/get-sbcl.sh $(B);
1.67 $< $(SBCL_VERSION)
1.68 cd $(SBCL_TARGET) && \
1.69 echo '"2.3.12+main"' > version.lisp-expr && \
1.70@@ -75,7 +82,7 @@
1.71
1.72 ### Rust
1.73 RUST_TARGET:=build/src/rust-$(RUST_VERSION)
1.74-rust:scripts/get-rust.sh
1.75+rust:scripts/get-rust.sh $(B);
1.76 $< $(RUST_VERSION)
1.77 rust-x:rust;
1.78 cargo install --path $(RUST_TARGET)/src/tools/x
1.79@@ -90,7 +97,7 @@
1.80
1.81 ### Code
1.82 CODE_TARGET:=build/src/$(SRC)
1.83-code:scripts/get-code.sh
1.84+code:scripts/get-code.sh $(B)
1.85 $< $(SRC)
1.86
1.87 clean-code::;rm -rf $(CODE_TARGET)
1.88@@ -116,31 +123,29 @@
1.89 virt:pod box bbdb vc
1.90
1.91 ### Dist
1.92-dist/bundle:scripts/bundle-dir.sh
1.93+dist/bundle:scripts/bundle-dir.sh $(D)
1.94 $<
1.95
1.96-dist/cdn:cdn
1.97+dist/cdn:cdn $(D)
1.98 mkdir -pv $@
1.99 cp -r $^ $@
1.100
1.101-dist/sbcl:sbcl;
1.102+dist/sbcl:sbcl $(D);
1.103 $(SBCL_TARGET)/install.sh --prefix=$(D)
1.104
1.105-dist/linux:linux;
1.106+dist/linux:linux $(D);
1.107
1.108-dist/rocksdb:rocksdb;
1.109+dist/rocksdb:rocksdb $(D);
1.110 cd $(ROCKSDB_TARGET)
1.111 cp -rf include/* $(D)
1.112 cp -f librocksdb.so* $(D)/lib/
1.113
1.114-dist/rust:rust-build;
1.115+dist/rust:rust-build $(D);
1.116 cd $(RUST_TARGET) && x dist
1.117
1.118-dist/emacs:emacs;
1.119+dist/emacs:emacs-build $(D);
1.120
1.121-dist:dist/bundle dist/cdn dist/sbcl dist/rocksdb # dist/linux dist/rust
1.122-
1.123-clean-dist::;rm -rf dist/*
1.124+clean-dist::;rm -rf dist
1.125
1.126 ### Quickstart
1.127 quick:code
2.1--- a/scripts/autogen.lisp Mon Dec 04 21:28:10 2023 -0500
2.2+++ b/scripts/autogen.lisp Tue Dec 05 19:03:29 2023 -0500
2.3@@ -1,9 +1,25 @@
2.4+#!/usr/local/bin/sbcl --script
2.5 ;;; scripts/autogen.lisp --- prepare the local environment
2.6
2.7 ;;; Code:
2.8+(require 'sb-posix)
2.9+(require 'asdf)
2.10 (in-package :cl-user)
2.11-(use-package :std)
2.12+
2.13+(format t "Starting comp/infra autogen...~%")
2.14+
2.15+(defvar *scripts* (directory-namestring (or *load-truename* #P"./scripts/")))
2.16+(defvar *systems* (list #P"~/quicklisp/dists/quicklisp/software/cl-ppcre-20230618-git/"
2.17+ #P"~/dev/comp/core/lisp/std/"))
2.18
2.19-(defmain ()
2.20- "Prepare the local environment for bootstrapping a complete system."
2.21- (format t "Starting scripts/autogen.lisp...~%"))
2.22+(mapc (lambda (p) (pushnew (sb-ext:native-pathname p) asdf:*central-registry*)) *systems*)
2.23+
2.24+(asdf:load-systems :cl-ppcre :std)
2.25+(load (merge-pathnames *scripts* "check.lisp"))
2.26+;; check system compatibility
2.27+(check:check :warn)
2.28+
2.29+(std:println "DONE.")
2.30+
2.31+(std:println "saving lisp executable to './check'... Bye.")
2.32+(sb-ext:save-lisp-and-die "check" :toplevel #'check:main :executable t :compression t)
3.1--- a/scripts/check.lisp Mon Dec 04 21:28:10 2023 -0500
3.2+++ b/scripts/check.lisp Tue Dec 05 19:03:29 2023 -0500
3.3@@ -4,15 +4,12 @@
3.4 ;; bootstrapping.
3.5
3.6 ;;; Code:
3.7-(require 'asdf)
3.8-(require 'sb-posix)
3.9-(asdf:load-asd #P"~/dev/comp/core/lisp/std/std.asd")
3.10-(asdf:load-system :std)
3.11-
3.12 (defpackage :infra/scripts/check
3.13- (:use :cl :std :sb-alien)
3.14+ (:nicknames :check)
3.15+ (:use :cl :std :std/fmt :sb-alien)
3.16 (:export :main
3.17- :*results*))
3.18+ :*results*
3.19+ :check))
3.20
3.21 (in-package :infra/scripts/check)
3.22
3.23@@ -26,36 +23,59 @@
3.24 (defun push-result (k &optional v)
3.25 (setf (gethash k *results*) v))
3.26
3.27-(defun check-for-shared-lib (name)
3.28+(defmacro check-err (is-warn ctrl name)
3.29+ `(if ,is-warn
3.30+ (and
3.31+ (warn 'simple-warning
3.32+ :format-control ,ctrl
3.33+ :format-arguments (list ,name))
3.34+ (push-result ,name))
3.35+ (error 'simple-program-error
3.36+ :format-control ,ctrl
3.37+ :format-arguments (list ,name))))
3.38+
3.39+(defun check-for-shared-lib (name &optional warn)
3.40+ "Check for a shared library by loading it in the current session with dlopen.
3.41+
3.42+When WARN is non-nil, signal a warning instead of an error."
3.43 (if-let ((lib (ignore-errors (load-shared-object (format nil "lib~a.so" name)))))
3.44 (prog1
3.45 (push-result name lib)
3.46 (unload-shared-object lib))
3.47- (push-result name)))
3.48+ (check-err warn "shared library missing: ~x" name)))
3.49
3.50-(defun check-for-bin (name)
3.51- (push-result name))
3.52+(defun check-for-bin (name &optional warn)
3.53+ (if-let ((exe (find-exe name)))
3.54+ (push-result name exe)
3.55+ (check-err warn "executable program missing: ~x" name)))
3.56
3.57 (defun check-for-src (name)
3.58 (push-result name))
3.59
3.60+(defun check-hostname () (push-result "hostname" (machine-instance)))
3.61+
3.62+(defun check-user () (push-result "user" (cons (sb-posix:getenv "USER") (user-homedir-pathname))))
3.63+
3.64 (defun check-system ()
3.65 (destructuring-bind (lisp version features)
3.66 (my-lisp-implementation)
3.67 (push-result "lisp" (list lisp version features)
3.68- #-sbcl (push-result "lisp"))))
3.69+ #-sbcl (push-result "lisp" "unsupported")
3.70+ #-sb-core-compression (println "WARNING: feature sb-core-compression disabled")
3.71+ #-mark-region-gc (println "WARNING: feature mark-region-gc disabled")
3.72+)))
3.73
3.74 (defun check-shell ()
3.75- (push-result "shell" (sb-posix:getcwd)))
3.76+ (push-result "shell" (sb-posix:getenv "SHELL")))
3.77
3.78 (defun check-display ()
3.79- (push-result "display"))
3.80+ (push-result "display" (sb-posix:getenv "DISPLAY")))
3.81
3.82 (defun check-net ()
3.83 (push-result "net"))
3.84
3.85 (defun check-cpu ()
3.86- (push-result "cpu"))
3.87+ (push-result "cpu" (machine-version)))
3.88
3.89 (defmethod print-object ((object hash-table) stream)
3.90 (format stream "#HOST-FEATURES{~{~{~%(~a . ~a)~}~^ ~}}"
3.91@@ -64,7 +84,7 @@
3.92 unless (null v)
3.93 collect (list k v))))
3.94
3.95-(defmain ()
3.96+(defun check (&optional warn)
3.97 "Check the host for required features."
3.98
3.99 (setq *library-path* (ld-library-path-list)
3.100@@ -73,33 +93,35 @@
3.101 (debug! (format nil "LD_LIBRARY_PATH: ~A~%" *library-path*))
3.102 (debug! (format nil "PATH: ~A~%" *exec-path*))
3.103 (check-system)
3.104+ (check-user)
3.105+ (check-net)
3.106+ (check-hostname)
3.107+ (check-cpu)
3.108 (check-shell)
3.109-
3.110- (check-for-bin "sbcl")
3.111- (check-for-bin "rustc")
3.112- (check-for-bin "clang")
3.113- (check-for-bin "gcc")
3.114- (check-for-bin "emacs")
3.115+ (check-display)
3.116+ (check-for-bin "sbcl" warn)
3.117+ (check-for-bin "rustc" warn)
3.118+ (check-for-bin "clang" warn)
3.119+ (check-for-bin "gcc" warn)
3.120+ (check-for-bin "emacs" warn)
3.121 ;; vcs
3.122- (check-for-bin "hg")
3.123- (check-for-bin "git")
3.124+ (check-for-bin "hg" warn)
3.125+ (check-for-bin "git" warn)
3.126 ;; virt
3.127- (check-for-bin "podman")
3.128+ (check-for-bin "podman" warn)
3.129
3.130 (check-for-src "core")
3.131 ;; core dependencies
3.132- (check-for-shared-lib "rocksdb")
3.133- (check-for-shared-lib "uring")
3.134- (check-for-shared-lib "btrfs")
3.135- (check-for-shared-lib "btrfsutil")
3.136- (check-for-shared-lib "tree-sitter")
3.137+ (check-for-shared-lib "rocksdb" warn)
3.138+ (check-for-shared-lib "uring" warn)
3.139+ (check-for-shared-lib "btrfs" warn)
3.140+ (check-for-shared-lib "btrfsutil" warn)
3.141+ (check-for-shared-lib "tree-sitter" warn)
3.142 ;; extras
3.143- (check-for-shared-lib "sbcl") ;; requires compiling sbcl as shared_lib
3.144- (check-for-shared-lib "gtk-4")
3.145- (check-for-shared-lib "blake3")
3.146- (check-for-shared-lib "k")
3.147- (check-for-shared-lib "cbqn")
3.148- (print *results*)
3.149+ (check-for-shared-lib "gtk-4" warn)
3.150+ (check-for-shared-lib "blake3" warn)
3.151+ (check-for-shared-lib "k" warn)
3.152+ (check-for-shared-lib "cbqn" warn)
3.153 *results*)
3.154
3.155-(main)
3.156+(defmain () (println (check)))
4.1--- a/scripts/eval.sh Mon Dec 04 21:28:10 2023 -0500
4.2+++ b/scripts/eval.sh Tue Dec 05 19:03:29 2023 -0500
4.3@@ -1,7 +1,7 @@
4.4 #!/bin/sh
4.5
4.6-# eval lisp code
4.7-LISP="/usr/bin/sbcl"
4.8+# eval lisp code with sbcl
4.9+LISP=sbcl
4.10 args=""
4.11 FILE="${1}"
4.12 [[ -z "$FILE" ]] && : || args="--script $FILE "
5.1--- a/scripts/get-code.sh Mon Dec 04 21:28:10 2023 -0500
5.2+++ b/scripts/get-code.sh Tue Dec 05 19:03:29 2023 -0500
5.3@@ -1,5 +1,5 @@
5.4 #!/bin/bash
5.5-# unbundle a tar.zst archive of Mercurial repositories.
5.6+# Get The Compiler Company source code
5.7 SRC="${1:-comp}"
5.8 OUT=build/comp-$(date "+%Y%m%d")
5.9 mkdir -pv $OUT
6.1--- a/scripts/get-emacs.sh Mon Dec 04 21:28:10 2023 -0500
6.2+++ b/scripts/get-emacs.sh Tue Dec 05 19:03:29 2023 -0500
6.3@@ -1,4 +1,5 @@
6.4 #!/usr/bin/bash
6.5+# get Emacs source code
6.6 VER="${1:-main}"
6.7 TARGETDIR=${2:-$(realpath build/src/emacs-$VER)}
6.8 git clone https://vc.compiler.company/packy/shed/vendor/emacs.git $TARGETDIR
7.1--- a/scripts/get-linux.sh Mon Dec 04 21:28:10 2023 -0500
7.2+++ b/scripts/get-linux.sh Tue Dec 05 19:03:29 2023 -0500
7.3@@ -1,6 +1,11 @@
7.4 #!/usr/bin/bash
7.5 # get-linux
7.6+
7.7 # from https://git.kernel.org/pub/scm/linux/kernel/git/mricon/korg-helpers.git/tree/get-verified-tarball
7.8+
7.9+# there are minimal modifications to this script - be careful, it
7.10+# pulls from the upstream kernel source archives.
7.11+
7.12 # --------------------
7.13 # Get Linux kernel tarball and cryptographically verify it,
7.14 # retrieving the PGP keys using the Web Key Directory (WKD)
8.1--- a/scripts/get-lust.sh Mon Dec 04 21:28:10 2023 -0500
8.2+++ b/scripts/get-lust.sh Tue Dec 05 19:03:29 2023 -0500
8.3@@ -1,4 +1,5 @@
8.4 #!/usr/bin/bash
8.5+# install the Lonely Rust compiler source code
8.6 VER="${1:-main}"
8.7 TARGETDIR=${2:-$(realpath build/lust-$VER)}
8.8 hg clone https://vc.compiler.company/lust $TARGETDIR
9.1--- a/scripts/get-rocksdb.sh Mon Dec 04 21:28:10 2023 -0500
9.2+++ b/scripts/get-rocksdb.sh Tue Dec 05 19:03:29 2023 -0500
9.3@@ -1,4 +1,5 @@
9.4 #!/usr/bin/bash
9.5+# get RocksDB source code
9.6 VER="${1:-main}"
9.7 TARGETDIR=${2:-$(realpath build/src/rocksdb-$VER)}
9.8 git clone https://vc.compiler.company/packy/shed/vendor/rocksdb.git $TARGETDIR
10.1--- a/scripts/get-rust.sh Mon Dec 04 21:28:10 2023 -0500
10.2+++ b/scripts/get-rust.sh Tue Dec 05 19:03:29 2023 -0500
10.3@@ -1,9 +1,13 @@
10.4 #!/usr/bin/bash
10.5
10.6+# get rust source code
10.7+
10.8+# You may want to have an upstream rust compiler on hand:
10.9+
10.10 #curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
10.11 #rustup update
10.12 #rustup default nightly
10.13
10.14 VER="${1:-main}"
10.15-TARGETDIR=${2:-$(realpath build/rust-$VER)}
10.16+TARGETDIR=${2:-$(realpath build/src/rust-$VER)}
10.17 git clone https://vc.compiler.company/packy/shed/vendor/rust.git $TARGETDIR
11.1--- a/scripts/get-sbcl.sh Mon Dec 04 21:28:10 2023 -0500
11.2+++ b/scripts/get-sbcl.sh Tue Dec 05 19:03:29 2023 -0500
11.3@@ -1,4 +1,5 @@
11.4 #!/usr/bin/bash
11.5+# get SBCL source code
11.6 VER="${1:-main}"
11.7 TARGETDIR=${2:-$(realpath build/src/sbcl-$VER)}
11.8 git clone https://vc.compiler.company/packy/shed/vendor/sbcl.git $TARGETDIR
12.1--- a/scripts/git-vendor-pull.sh Mon Dec 04 21:28:10 2023 -0500
12.2+++ b/scripts/git-vendor-pull.sh Tue Dec 05 19:03:29 2023 -0500
12.3@@ -1,4 +1,6 @@
12.4 #!/usr/bin/bash
12.5+# pull a vendored dependency and push to upstream vc
12.6+
12.7 # scripts/git-vendor-pull.sh git.savannah.gnu.org/git emacs master
12.8 NAME="${1}"
12.9 DOMAIN="${2}"
13.1--- a/scripts/init.sh Mon Dec 04 21:28:10 2023 -0500
13.2+++ b/scripts/init.sh Tue Dec 05 19:03:29 2023 -0500
13.3@@ -1,6 +1,10 @@
13.4 #!/bin/sh
13.5
13.6-# install the compiler.company installer.
13.7+# run the compiler.company core installer (cc-init)
13.8+
13.9+# this script downloads and calls the 'cc-init' installer for the
13.10+# current host. Follow the prompts and read the documentation to build
13.11+# a fully compliant operator system.
13.12
13.13 set -u
13.14
