| changeset 274: | 7046a8e3e517 |
|---|---|
| parent 273: | 33ba3773e420 (current diff) |
| parent 271: | cd797f4eb846 (diff) |
| child 275: | 8314299755f9 |
| author: | Richard Westhaver <ellis@rwest.io> |
| date: | Sun, 16 Jun 2024 23:56:05 +0000 |
| files: | bootstrap.sh |
| description: | merge |
1.1--- a/autogen.lisp Sun Jun 16 23:54:32 2024 +0000 1.2+++ b/autogen.lisp Sun Jun 16 23:56:05 2024 +0000 1.3@@ -1,6 +1,18 @@ 1.4-;;; autogen.lisp --- Setup the current directory for 1.5+;;; autogen.lisp --- Auto-generate CC Infrastructure 1.6+ 1.7+;; This script must be ran with a compliant CC lisp core image from a complian 1.8+;; CC lisp compiler. The easiest way to get started is by running the 1.9+;; 'bootstrap.sh' script first which will download these for you into the 1.10+;; local 'STASH' directory and run this script automatically. 1.11 1.12-;; sbcl --core .stash/share/lisp/user.core --load autogen.lisp 1.13+#| 1.14+# download pre-compiled binaries and run autogen.lisp 1.15+./bootstrap.sh 1.16+ 1.17+# or run manually with local lisp runtime and core 1.18+sbcl --core $LISP_HOME/user.core --script autogen.lisp \ 1.19+ --eval "(infra/autogen:autogen)" 1.20+|# 1.21 1.22 ;;; Code: 1.23 (in-package :std-user) 1.24@@ -10,27 +22,22 @@ 1.25 :dat/json :dat/sxp :net/fetch :net/util 1.26 :cli/progress :cli/ansi :cli/ed :cli/prompt 1.27 :cli/shell :std/hash-table :std/alien :std/macs 1.28- :std/fmt)) 1.29+ :std/fmt) 1.30+ (:export :autogen)) 1.31 1.32 (in-package :infra/autogen) 1.33- 1.34 (in-readtable :shell) 1.35- 1.36+;;; Vars 1.37 (defparameter *profile* (uiop:read-file-forms 1.38 (if-let ((profile (sb-posix:getenv "INFRA_PROFILE"))) 1.39- profile 1.40- #P"default.sxp"))) 1.41+ profile 1.42+ #P"default.sxp"))) 1.43 (defparameter *core* sb-ext:*core-pathname*) 1.44 (defparameter *host* (uiop:read-file-forms 1.45 (let ((hcfg (format nil "~a.sxp" (sb-unix:unix-gethostname)))) 1.46 (unless (probe-file hcfg) 1.47 (print #0$./check.sh$#)) 1.48 hcfg))) 1.49- 1.50-(defun gethost (k) (getf *host* k)) 1.51-(defun getprofile (k) (getf *profile* k)) 1.52-(init-skel-vars) 1.53-(setq *skel-project* (find-skelfile *default-pathname-defaults* :load t)) 1.54 (defparameter *host-env* (let ((table (make-hash-table :test 'equal)) 1.55 (keys (list "STASH" "STORE" "DIST" "PACKY_URL" "VC_URL" "INSTALL_PREFIX" 1.56 "CC" "AR" "HG" "GIT" "LISP" "RUST" "LD" "SHELL" "DEV" "DEV_HOME" 1.57@@ -38,20 +45,49 @@ 1.58 "LISP_HOME" "INFRA_PROFILE"))) 1.59 (dolist (k keys table) 1.60 (setf (gethash k table) (sb-posix:getenv k))))) 1.61+ 1.62+;;; Utils 1.63+(defun gethost (k) (getf *host* k)) 1.64+(defun getprofile (k) (getf *profile* k)) 1.65+ 1.66 (defun getenv (k) (gethash *host-env* k)) 1.67 1.68-(defun build-default () 1.69- (let ((comp-builder (sb-thread:make-thread (lambda () (sk-call* *skel-project* :repos)))) 1.70- (packy-builder (sb-thread:make-thread (lambda () (sk-call* *skel-project* :packy-repos)))) 1.71- (operator-builder (sb-thread:make-thread (lambda () (sk-call* *skel-project* :archlinux :box)))) 1.72- (worker-builder (sb-thread:make-thread (lambda () (sk-call* *skel-project* :alpine :worker))))) 1.73- (std/thread:wait-for-threads 1.74- (list comp-builder packy-builder)))) 1.75+(defun setenv (k v) 1.76+ (sb-posix:setenv k v 1) 1.77+ (setf (gethash k *host-env*) v)) 1.78+ 1.79+;;; Config 1.80+(defun init-profile () 1.81+ (let ((stash (getprofile :stash)) 1.82+ (cc (getprofile :cc))) 1.83+ (if-let ((stash (probe-file stash))) 1.84+ (setenv "STASH" (namestring stash)) 1.85+ (error "STASH not found: ~A" stash)) 1.86+ (if-let ((cc (cli:find-exe cc))) 1.87+ (setenv "CC" (namestring cc)) 1.88+ (error "STASH not found: ~A" cc)))) 1.89+ 1.90+;;; Build 1.91+(defun make-default () 1.92+ (std/thread:wait-for-threads 1.93+ (list (sb-thread:make-thread (lambda () (sk-call* *skel-project* :repos))) 1.94+ (sb-thread:make-thread (lambda () (sk-call* *skel-project* :packy-repos)))))) 1.95+ 1.96+(defun make-pods () 1.97+ (std/thread:wait-for-threads 1.98+ (list (sb-thread:make-thread (lambda () (sk-call* *skel-project* :archlinux :box))) 1.99+ (sb-thread:make-thread (lambda () (sk-call* *skel-project* :alpine :worker)))))) 1.100 1.101 (defun autogen () 1.102 "Auto-generate the INFRA system." 1.103- (info! "starting autogen.lisp" sb-ext:*core-pathname*) 1.104+ (info! "starting autogen") 1.105+ (in-readtable :shell) 1.106 (terpri) 1.107+ (init-profile) 1.108+ (init-skel-vars) 1.109+ (setq *skel-project* (find-skelfile *default-pathname-defaults* :load t)) 1.110+ (unless (probe-file #p".stash") 1.111+ (sk-call* *skel-project* :bootstrap)) 1.112 ;; print host, env, profile 1.113 (format t "core: ~A~%" *core*) 1.114 (terpri) 1.115@@ -65,8 +101,7 @@ 1.116 (println "profile:") 1.117 (loop for (k v) on *profile* by 'cddr 1.118 do (format t " ~A = ~A~%" k v)) 1.119- 1.120- (build-default)) 1.121+ (make-default)) 1.122 1.123 ;;; *host* 1.124 ;; The host profile is generated automatically by 'check.sh'. After running 1.125@@ -75,12 +110,3 @@ 1.126 ;;; *profile* 1.127 ;; The default profile is defined in 'default.sxp'. You can use that as a base 1.128 ;; configuration and override it with INFRA_PROFILE 1.129- 1.130-;; (sb-ext:quit) 1.131- 1.132-(unless (probe-file #p".stash") 1.133- (sk-call* *skel-project* :bootstrap)) 1.134- 1.135-;; (build-default) 1.136-(autogen) 1.137-
2.1--- a/bootstrap.sh Sun Jun 16 23:54:32 2024 +0000 2.2+++ b/bootstrap.sh Sun Jun 16 23:56:05 2024 +0000 2.3@@ -55,8 +55,11 @@ 2.4 rm -rf *.tar 2.5 say "successfully unpacked core" 2.6 say "starting lisp..." 2.7- cd .. 2.8- .stash/bin/sbcl --core .stash/share/lisp/prelude.core --script autogen.lisp 2.9+ cd .. && \ 2.10+ .stash/bin/sbcl --core .stash/share/lisp/user.core \ 2.11+ --load autogen.lisp \ 2.12+ --eval "(infra/autogen:autogen)" \ 2.13+ --non-interactive 2.14 say "OK" 2.15 } 2.16
3.1--- a/makefile Sun Jun 16 23:54:32 2024 +0000 3.2+++ /dev/null Thu Jan 01 00:00:00 1970 +0000 3.3@@ -1,277 +0,0 @@ 3.4-### infra/makefile --- The Compiler Company Infrastructure 3.5- 3.6-# this makefile is used to build all of our source code, package it, 3.7-# and ship it to various locations. It is for internal-use only. 3.8- 3.9-# If possible, use the packy (packy.compiler.company) to find and 3.10-# download a prepared bundle that suits your project needs. 3.11- 3.12-# You probably don't want to install all targets unless you have lots 3.13-# of time and a powerful machine, although it is possible. Instead, 3.14-# just run the targets for the components you are missing in your 3.15-# local environment (compatible compiler versions, shared libraries, 3.16-# etc) 3.17- 3.18-VERSION="0.1.0" 3.19-LINUX_VERSION:=$(shell uname -r | cut -d- -f1) 3.20-B:=build 3.21-D:=dist 3.22-SRC:=code 3.23-CONTAINERS:=containers 3.24-HG_COMMIT:=$(shell hg id -i) 3.25-DESTINATION:=/mnt/y/data/packy 3.26-# requires emacs-build-minimal 3.27-# artifacts can deploy to dist/TARGET - need target triple first 3.28-# init:sbcl rust emacs rocksdb code 3.29-# dist/linux dist/rust dist/bundle 3.30-archlinux:$(CONTAINERS)/Containerfile.archlinux;podman build -f $< -t archlinux 3.31-box:$(CONTAINERS)/Containerfile.box archlinux;podman build -f $< -t box 3.32-alpine:$(CONTAINERS)/Containerfile.alpine;podman build -f $< -t alpine 3.33-ubuntu:$(CONTAINERS)/Containerfile.ubuntu;podman build -f $< -t ubuntu 3.34-worker:$(CONTAINERS)/Containerfile.worker alpine;podman build -f $< -t worker 3.35-operator:$(CONTAINERS)/Containerfile.operator box;podman build -f $< -t operator 3.36-pods:archlinux alpine ubuntu box worker operator 3.37-quick:code 3.38-all:dist/code dist/lisp dist/rust dist/sbcl dist/rocksdb dist/emacs dist/pods 3.39-clean:;rm -rf $(B) $(D) 3.40-$(B):;mkdir -pv $@/src 3.41-$(D):;mkdir -pv $@/bin $@/lib $@/include $@/bundle $@/share 3.42-$(DESTINATION):$(D);cd $< && cp -rf ./* $@ 3.43-### Linux 3.44-LINUX_TARGET:=linux-$(LINUX_VERSION) 3.45-linux:$(LINUX_TARGET) linux-config; 3.46-$(LINUX_TARGET):scripts/get-linux.sh; 3.47- mkdir -pv .stash/$@ 3.48- gpg --export autosigner@ torvalds@ gregkh@ > .stash/$@/keyring.gpg 3.49- $< $(LINUX_VERSION) build .stash/$@/keyring.gpg 3.50- cd build && unxz $@.tar.xz && tar -xvf $@.tar $(LINUX_TARGET) 3.51-linux-config:$(LINUX_TARGET); 3.52- cd .stash/$< && make mrproper -j && zcat /proc/config.gz > .config && yes N | make localmodconfig 3.53-clean-linux::;rm -rf .stash/$(LINUX_TARGET) 3.54- 3.55-### Emacs 3.56-EMACS_TARGET:=.stash/src/emacs 3.57-EMACS_DIST:=$(D)/src/emacs 3.58-$(EMACS_TARGET):scripts/get-emacs.sh $(B); 3.59- $< 3.60-emacs:$(EMACS_TARGET) 3.61-emacs-build:scripts/build-emacs.sh $(EMACS_TARGET) 3.62- cd $(EMACS_TARGET) && ./autogen.sh 3.63- $< 3.64-emacs-build-mini:scripts/build-emacs-mini.sh emacs 3.65- cd $(EMACS_TARGET) && ./autogen.sh 3.66- $< $(EMACS_TARGET) 3.67-emacs-install:$(EMACS_TARGET); 3.68- cd $< && make install 3.69- 3.70-### RocksDB 3.71-ROCKSDB_TARGET:=.stash/src/rocksdb 3.72-$(ROCKSDB_TARGET):scripts/get-rocksdb.sh $(B) 3.73- $< 3.74-rocksdb:$(ROCKSDB_TARGET) 3.75- 3.76-rocksdb-build-shared:$(ROCKSDB_TARGET) 3.77- cd $< && make shared_lib DISABLE_JEMALLOC=1 3.78- 3.79-rocksdb-build-static:$(ROCKSDB_TARGET) 3.80- cd $< && make static_lib DISABLE_JEMALLOC=1 3.81- 3.82-rocksdb-install:$(ROCKSDB_TARGET) 3.83- cd $< && make install-shared && cp -r include/* /usr/local/include/ 3.84- 3.85-### Nushell 3.86-NUSHELL_TARGET:=.stash/src/nushell 3.87-$(NUSHELL_TARGET):scripts/get-nushell.sh;$< 3.88-nushell:$(NUSHELL_TARGET) 3.89-# build without clipboard to avoid errors at runtime in container env 3.90-nushell-build:$(NUSHELL_TARGET) 3.91- cd $< && cargo build --workspace --release --no-default-features --features=default-no-clipboard,dataframe,extra --locked 3.92-nushell-install:$(NUSHELL_TARGET) nushell-build 3.93- cd $< && ./scripts/install-all.sh 3.94-### SBCL 3.95-SBCL_TARGET:=.stash/src/sbcl 3.96-SBCL_VERSION:=2.4.5 3.97-$(SBCL_TARGET):scripts/get-sbcl.sh $(B) 3.98- $< 3.99- cd $(SBCL_TARGET) && \ 3.100- echo '"$(SBCL_VERSION)"' > version.lisp-expr 3.101-sbcl:$(SBCL_TARGET) 3.102-#### --with-sb-fasteval --without-sb-eval < broken 3.103-sbcl-build:$(SBCL_TARGET) 3.104- cd $< && \ 3.105- ./make.sh \ 3.106- --without-gencgc \ 3.107- --with-mark-region-gc \ 3.108- --dynamic-space-size=8Gb \ 3.109- --fancy 3.110-sbcl-build-shared:$(SBCL_TARGET) sbcl-build 3.111- cd $< && \ 3.112- ./make-shared-library.sh 3.113-sbcl-build-gencgc:$(SBCL_TARGET) 3.114- cd $< && \ 3.115- ./make.sh \ 3.116- --dynamic-space-size=8Gb \ 3.117- --fancy 3.118-sbcl-docs:sbcl-build;## REQUIRES TEXLIVE 3.119- cd $(SBCL_TARGET)/doc/manual && make 3.120-sbcl-install:sbcl-build;cd $(SBCL_TARGET) && INSTALL_ROOT=/usr/local sh install.sh 3.121-sbcl-install-gencgc:sbcl-build-gencgc;cd $(SBCL_TARGET) && INSTALL_ROOT=/usr/local sh install.sh 3.122-clean-sbcl:$(SBCL_TARGET);cd $(SBCL_TARGET) && ./clean.sh 3.123- 3.124-.stash/quicklisp.lisp:$(B);cd $< && curl -O https://beta.quicklisp.org/quicklisp.lisp 3.125-quicklisp-install:scripts/quicklisp-install.sh .stash/quicklisp.lisp;$< 3.126- 3.127-STUMPWM_TARGET:=.stash/src/stumpwm 3.128-$(STUMPWM_TARGET):scripts/get-stumpwm.sh $(B);$< 3.129-stumpwm:$(STUMPWM_TARGET); 3.130-stumpwm-build:stumpwm; 3.131- cd $(STUMPWM_TARGET) && ./autogen.sh && ./configure && make 3.132-stumpwm-install:stumpwm-build; 3.133- cd $(STUMPWM_TARGET) && make install 3.134-### Rust 3.135-RUST_TARGET:=.stash/src/rust 3.136-$(RUST_TARGET):scripts/get-rust.sh $(B);$< 3.137-rust:$(RUST_TARGET) 3.138-rust-install-x:rust; 3.139- cargo install --path $(RUST_TARGET)/src/tools/x 3.140-rust-build:rust rust-install-x; 3.141- cd $(RUST_TARGET) && x build library 3.142-rust-doc:rust rust-install-x; 3.143- cd $(RUST_TARGET) && x doc 3.144-rust-build-full:rust-build; 3.145- cd $(RUST_TARGET) && x build --stage 2 compiler/rustc 3.146-rust-install:rust-build; 3.147- cd $(RUST_TARGET) && x install 3.148-rustup-install:;curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y 3.149-cargo-tools-install:scripts/install-cargo-tools.sh 3.150- $< 3.151-### Tree-sitter 3.152-TREE_SITTER_TARGET:=.stash/src/tree-sitter 3.153-$(TREE_SITTER_TARGET):scripts/get-tree-sitter.sh 3.154- $< 3.155-tree-sitter:$(TREE_SITTER_TARGET) 3.156-tree-sitter-build:$(TREE_SITTER_TARGET) 3.157-tree-sitter-install:$(TREE_SITTER_TARGET) tree-sitter-build 3.158- 3.159-### Tree-sitter Langs 3.160-TREE_SITTER_LANGS_TARGET:=.stash/src/tree-sitter-langs 3.161-tree-sitter-langs-install:scripts/tree-sitter-install-langs.sh 3.162- $< 3.163-### Etc 3.164-ETC_TARGET:=.stash/etc 3.165-$(ETC_TARGET):scripts/get-etc.sh 3.166- $< 3.167-etc:$(ETC_TARGET) 3.168- 3.169-### Code 3.170-CODE_TARGET:=.stash/src/$(SRC) 3.171-$(CODE_TARGET):scripts/get-code.sh $(B) 3.172- $< $(SRC) 3.173-code:$(CODE_TARGET) 3.174-clean-code::;rm -rf $(CODE_TARGET) 3.175- 3.176-### Dist 3.177-dist/bundle:scripts/bundle-code.sh $(CODE_TARGET) 3.178- mkdir -pv $@ 3.179- $< 3.180- 3.181-dist/sbcl:$(D); 3.182- scripts/dist-sbcl-binary.sh $(SBCL_TARGET) $(D) 3.183- cd $(SBCL_TARGET) && sh ./clean.sh 3.184- scripts/dist-sbcl-source.sh $(SBCL_TARGET) $(D) 3.185- 3.186-dist/linux:linux $(D); 3.187- 3.188-dist/rocksdb:$(D) rocksdb; 3.189- cd .stash/src && \ 3.190- tar -I 'zstd' -cf ../../$</rocksdb.tar.zst rocksdb/include/* rocksdb/librocksdb.* 3.191- 3.192-CORE_SRC?=/usr/local/src/core 3.193- 3.194-dist/rust:rust-build $(D); 3.195- cd $(RUST_TARGET) && x dist 3.196- 3.197-dist/rust/bin:scripts/cargo-install.sh 3.198- mkdir -pv $@ 3.199- $< "$(CORE_SRC)/rust/app/cli/alik" dist/rust 3.200- $< "$(CORE_SRC)/rust/app/cli/krypt" dist/rust 3.201- $< "$(CORE_SRC)/rust/app/cli/tz" dist/rust 3.202- $< "$(CORE_SRC)/rust/app/cli/cc-install" dist/rust 3.203- $< "$(CORE_SRC)/rust/app/cli/mailman" dist/rust 3.204- 3.205-dist/emacs:emacs-build $(D); 3.206- cd $(EMACS_TARGET) && ./make-dist --no-info --no-changelog && \ 3.207- mv emacs-*.*.* emacs && \ 3.208- tar -I 'zstd' -cf ../../../dist/emacs-binary.tar.zst emacs 3.209- 3.210-dist/emacs-mini:emacs-build-mini $(D); 3.211- cd $(EMACS_TARGET) && ./make-dist --no-info --no-changelog && \ 3.212- mv emacs-*.*.* emacs && \ 3.213- tar -I 'zstd' -cf ../../../dist/emacs-mini-binary.tar.zst emacs 3.214- 3.215-dist/nushell:$(D) nushell-build 3.216- cd $(NUSHELL_TARGET)/target/release/ && \ 3.217- tar -I 'zstd' -cf ../../../../../$</nushell.tar.zst \ 3.218- nu nu_plugin_custom_values nu_plugin_formats nu_plugin_gstat \ 3.219- nu_plugin_inc nu_plugin_query 3.220- 3.221-dist/tree-sitter:scripts/tree-sitter-install-langs.sh $(D) 3.222- PREFIX=$(D) $< 3.223- 3.224-# requires quicklisp loaded in .skelrc 3.225-dist/lisp/fasl:scripts/sbcl-save-core.sh 3.226- mkdir -pv $@ 3.227- $< "$@/std.core" 3.228- $< "$@/prelude.core" "(mapc #'ql:quickload \ 3.229- (list :nlp :rdb :organ :packy :skel :obj :net :parse :pod :dat :log :packy :rt :syn :xdb :doc :vc))" 3.230- 3.231-dist/lisp/bin:scripts/sbcl-make-bin.sh 3.232- mkdir -pv $@ 3.233- $< bin/skel 3.234- mv $(CORE_SRC)/lisp/app/bin/skel $@ 3.235- rm -f $(CORE_SRC)/lisp/app/bin/skel.fasl 3.236- $< bin/organ 3.237- cp $(CORE_SRC)/lisp/app/bin/organ $@ 3.238- rm -f $(CORE_SRC)/lisp/app/bin/organ.fasl 3.239- $< bin/homer 3.240- cp $(CORE_SRC)/lisp/app/bin/homer $@ 3.241- rm -f $(CORE_SRC)/lisp/app/bin/homer.fasl 3.242- $< bin/packy 3.243- cp $(CORE_SRC)/lisp/app/bin/packy $@ 3.244- rm -f $(CORE_SRC)/lisp/app/bin/packy.fasl 3.245- $< bin/rdb 3.246- cp $(CORE_SRC)/lisp/app/bin/rdb $@ 3.247- rm -f $(CORE_SRC)/lisp/app/bin/rdb.fasl 3.248- 3.249-dist/lisp:dist/lisp/fasl dist/lisp/bin 3.250- 3.251-core-lisp-install:dist/lisp 3.252- install -m 755 $</bin/* /usr/local/bin/ 3.253- install -m 755 $</fasl/* /usr/local/lib/sbcl/ 3.254- 3.255-core-rust-install:dist/rust/bin 3.256- install -m 755 $</* /usr/local/bin/ 3.257- 3.258-core-install:core-lisp-install core-rust-install 3.259- 3.260-dist/core:dist/rust/bin dist/lisp 3.261- mkdir -pv $(D)/core 3.262- cp -rf $(D)/lisp/fasl $(D)/lisp/bin $(D)/core 3.263- cp -rf $< $(D)/core 3.264- cd $(D) && tar -I 'zstd' -cf core.tar.zst core 3.265- 3.266-ETC:=$(USER) 3.267- 3.268-dist/etc:$(ETC_TARGET) 3.269- cd $(ETC_TARGET) && hg up $(ETC) 3.270- cp -rf $(ETC_TARGET) $(D)/etc 3.271- 3.272-dist/code:code 3.273- mkdir -pv $@ 3.274- cp -r $(CODE_TARGET)/{org,core,infra,demo,etc} $@ 3.275-dist/pods:pods 3.276- mkdir -pv $@ 3.277- podman image save -o $@/all.tar archlinux alpine ubuntu box worker operator 3.278- cd $@ && zstd --ultra -T4 --rm all.tar -o all.tar.zst 3.279-clean-dist:;rm -rf $(D) 3.280-clean-build:;rm -rf $(B)
4.1--- a/readme.org Sun Jun 16 23:54:32 2024 +0000 4.2+++ b/readme.org Sun Jun 16 23:56:05 2024 +0000 4.3@@ -1,8 +1,9 @@ 4.4-{{{header(infra, 4.5-Richard Westhaver, 4.6-ellis@rwest.io, 4.7-The Compiler Company Infrastructure)}}} 4.8+#+title: infra 4.9+#+author: Richard Westhaver 4.10+#+email: richard.westhaver@gmail.com 4.11+#+description: The Compiler Company Infrastructure 4.12 #+EXPORT_FILE_NAME: index 4.13+#+setupfile: https://cdn.compiler.company/org/clean.theme 4.14 4.15 This project contains scripts, utils, and applications used to host 4.16 and maintain The Compiler Company infrastructure.