changeset 298: | 4b7478f58f15 |
parent: | 61f88f16e0f1 |
child: | d0429338ca65 |
author: | Richard Westhaver <ellis@rwest.io> |
date: | Wed, 19 Jun 2024 19:45:37 -0400 |
permissions: | -rwxr-xr-x |
description: | edit tree-sitter-langs |
239 | 1 | #!/bin/sh |
295 | 2 | |
255
ad70dec68fa1
autogen updates and moved Containerfiles
Richard Westhaver <ellis@rwest.io>
parents:
254
diff
changeset
|
3 | |
239 | 4 | main() { |
5 | . ./check.sh |
|
295 | 6 | if [[ -z "${INFRA_PROFILE:-}" ]]; then |
7 | export INFRA_PROFILE="${1:-default.sxp}" |
|
8 | else |
|
9 | export INFRA_PROFILE="${INFRA_PROFILE}" |
|
10 | fi |
|
11 | set -eu |
|
239 | 12 | download --check |
13 | local _arch=$(_read arch | tr -d '"') |
|
14 | local _ext="" |
|
15 | case "$_arch" in |
|
16 | *windows*) |
|
17 | _ext=".exe" |
|
18 | ;; |
|
19 | esac |
|
260 | 20 | local _url="https://packy.compiler.company/dist/${_arch}/pack" |
239 | 21 | local _stash |
22 | if ! _stash=".stash"; then |
|
23 | # Because the previous command ran in a subshell, we must manually |
|
24 | # propagate exit status. |
|
25 | exit 1 |
|
26 | fi |
|
291 | 27 | # setup default directories |
239 | 28 | ensure mkdir -p "${_stash}/src" |
256 | 29 | ensure mkdir -p "${_stash}/share/lisp/fasl" |
291 | 30 | ensure mkdir -p "${_stash}/tmp" |
31 | ensure mkdir -p "${_stash}/share/store/dist" |
|
239 | 32 | ensure mkdir -p "${_stash}/bin" |
256 | 33 | ensure mkdir -p "${_stash}/lib" |
34 | ensure mkdir -p "${_stash}/include" |
|
291 | 35 | |
255
ad70dec68fa1
autogen updates and moved Containerfiles
Richard Westhaver <ellis@rwest.io>
parents:
254
diff
changeset
|
36 | cd "${_stash}" |
260 | 37 | local _sbcl_pack="sbcl.tar.zst" |
266 | 38 | # local _rocksdb_pack="rocksdb.tar.zst" |
260 | 39 | local _core_pack="core.tar.zst" |
293
39291a549477
speedup bootstrap with compressed infra.core
Richard Westhaver <ellis@rwest.io>
parents:
292
diff
changeset
|
40 | local _infra_core="infra.core" |
272 | 41 | # local _core_src_pack="core-source.tar.zst" |
256 | 42 | local _sbcl_url="${_url}/${_sbcl_pack}" |
266 | 43 | # local _rocksdb_url="${_url}/${_rocksdb_pack}" |
256 | 44 | local _core_url="${_url}/${_core_pack}" |
293
39291a549477
speedup bootstrap with compressed infra.core
Richard Westhaver <ellis@rwest.io>
parents:
292
diff
changeset
|
45 | local _infra_core_url="https://packy.compiler.company/dist/${_arch}/lisp/${_infra_core}" |
272 | 46 | # local _core_src_url="${_url}/${_core_src_pack}" |
291 | 47 | if [ ! -f "tmp/$_sbcl_pack" ]; then |
48 | ensure download "$_sbcl_url" "tmp/$_sbcl_pack" "$_arch" |
|
49 | unzstd "tmp/${_sbcl_pack}" |
|
50 | tar -C tmp -xf "tmp/sbcl.tar" |
|
51 | cd tmp/sbcl && INSTALL_ROOT=$(realpath ../..) sh install.sh && cd ../.. |
|
52 | fi |
|
53 | ||
272 | 54 | # ensure download "$_core_src_url" "$_core_src_pack" "$_arch" |
55 | # unzstd "${_core_src_pack}" |
|
56 | # tar -xvf "core-source.tar" |
|
57 | # mv core src/ |
|
260 | 58 | # ensure download "$_rocksdb_url" "${_rocksdb_pack}" "$_arch" |
59 | # unzstd "${_rocksdb_pack}" |
|
60 | # tar -xvf "pack/rocksdb.tar" |
|
61 | # cp -rf rocksdb/include/* include/ |
|
62 | # cp -rf rocksdb/*.so lib/ |
|
291 | 63 | |
294 | 64 | if [ ! -f "bin/skel" ]; then |
298 | 65 | ensure download "$_core_url" "tmp/${_core_pack}" "$_arch" |
66 | unzstd "tmp/${_core_pack}" |
|
67 | tar -xf "tmp/core.tar" |
|
68 | cp -rf tmp/core/bin/* bin/ |
|
69 | cp -rf tmp/core/share/* share/ |
|
294 | 70 | fi |
71 | ||
292 | 72 | if [ ! -f "share/lisp/$_infra_core" ]; then |
73 | ensure download "$_infra_core_url" "share/lisp/$_infra_core" "$_arch" |
|
291 | 74 | fi |
75 | ||
292 | 76 | chmod +x bin/* |
77 | ||
291 | 78 | say "bootstrap complete" |
79 | ||
264 | 80 | say "starting lisp..." |
271 | 81 | cd .. && \ |
292 | 82 | .stash/bin/sbcl --core .stash/share/lisp/infra.core \ |
271 | 83 | --load autogen.lisp \ |
290 | 84 | --eval "(infra/autogen:autogen)" \ |
85 | --non-interactive \ |
|
86 | --no-userinit --no-sysinit |
|
291 | 87 | |
264 | 88 | say "OK" |
239 | 89 | } |
90 | ||
91 | _read() { |
|
92 | grep ":$1" $INFRA_HOST_CONFIG | cut -d' ' -f 2- |
|
93 | } |
|
94 | ||
95 | # Check if curl supports the --retry flag, then pass it to the curl invocation. |
|
96 | check_curl_for_retry_support() { |
|
97 | local _retry_supported="" |
|
98 | # "unspecified" is for arch, allows for possibility old OS using macports, homebrew, etc. |
|
99 | if check_help_for "notspecified" "curl" "--retry"; then |
|
100 | _retry_supported="--retry 3" |
|
101 | if check_help_for "notspecified" "curl" "--continue-at"; then |
|
102 | # "-C -" tells curl to automatically find where to resume the download when retrying. |
|
103 | _retry_supported="--retry 3 -C -" |
|
104 | fi |
|
105 | fi |
|
106 | RETVAL="$_retry_supported" |
|
107 | } |
|
108 | ||
109 | # Return cipher suite string specified by user, otherwise return strong TLS 1.2-1.3 cipher suites |
|
110 | # if support by local tools is detected. Detection currently supports these curl backends: |
|
111 | # GnuTLS and OpenSSL (possibly also LibreSSL and BoringSSL). Return value can be empty. |
|
112 | get_ciphersuites_for_curl() { |
|
113 | if [ -n "${TLS_CIPHERSUITES-}" ]; then |
|
114 | # user specified custom cipher suites, assume they know what they're doing |
|
115 | RETVAL="$TLS_CIPHERSUITES" |
|
116 | return |
|
117 | fi |
|
118 | local _openssl_syntax="no" |
|
119 | local _gnutls_syntax="no" |
|
120 | local _backend_supported="yes" |
|
121 | if curl -V | grep -q ' OpenSSL/'; then |
|
122 | _openssl_syntax="yes" |
|
123 | elif curl -V | grep -iq ' LibreSSL/'; then |
|
124 | _openssl_syntax="yes" |
|
125 | elif curl -V | grep -iq ' BoringSSL/'; then |
|
126 | _openssl_syntax="yes" |
|
127 | elif curl -V | grep -iq ' GnuTLS/'; then |
|
128 | _gnutls_syntax="yes" |
|
129 | else |
|
130 | _backend_supported="no" |
|
131 | fi |
|
132 | local _args_supported="no" |
|
133 | if [ "$_backend_supported" = "yes" ]; then |
|
134 | # "unspecified" is for arch, allows for possibility old OS using macports, homebrew, etc. |
|
135 | if check_help_for "notspecified" "curl" "--tlsv1.2" "--ciphers" "--proto"; then |
|
136 | _args_supported="yes" |
|
137 | fi |
|
138 | fi |
|
139 | local _cs="" |
|
140 | if [ "$_args_supported" = "yes" ]; then |
|
141 | if [ "$_openssl_syntax" = "yes" ]; then |
|
142 | _cs=$(get_strong_ciphersuites_for "openssl") |
|
143 | elif [ "$_gnutls_syntax" = "yes" ]; then |
|
144 | _cs=$(get_strong_ciphersuites_for "gnutls") |
|
145 | fi |
|
146 | fi |
|
147 | RETVAL="$_cs" |
|
148 | } |
|
149 | ||
150 | # Return cipher suite string specified by user, otherwise return strong TLS 1.2-1.3 cipher suites |
|
151 | # if support by local tools is detected. Detection currently supports these wget backends: |
|
152 | # GnuTLS and OpenSSL (possibly also LibreSSL and BoringSSL). Return value can be empty. |
|
153 | get_ciphersuites_for_wget() { |
|
154 | if [ -n "${TLS_CIPHERSUITES-}" ]; then |
|
155 | # user specified custom cipher suites, assume they know what they're doing |
|
156 | RETVAL="$TLS_CIPHERSUITES" |
|
157 | return |
|
158 | fi |
|
159 | local _cs="" |
|
160 | if wget -V | grep -q '\-DHAVE_LIBSSL'; then |
|
161 | # "unspecified" is for arch, allows for possibility old OS using macports, homebrew, etc. |
|
162 | if check_help_for "notspecified" "wget" "TLSv1_2" "--ciphers" "--https-only" "--secure-protocol"; then |
|
163 | _cs=$(get_strong_ciphersuites_for "openssl") |
|
164 | fi |
|
165 | elif wget -V | grep -q '\-DHAVE_LIBGNUTLS'; then |
|
166 | # "unspecified" is for arch, allows for possibility old OS using macports, homebrew, etc. |
|
167 | if check_help_for "notspecified" "wget" "TLSv1_2" "--ciphers" "--https-only" "--secure-protocol"; then |
|
168 | _cs=$(get_strong_ciphersuites_for "gnutls") |
|
169 | fi |
|
170 | fi |
|
171 | RETVAL="$_cs" |
|
172 | } |
|
173 | ||
174 | check_help_for() { |
|
175 | local _arch |
|
176 | local _cmd |
|
177 | local _arg |
|
178 | _arch="$1" |
|
179 | shift |
|
180 | _cmd="$1" |
|
181 | shift |
|
182 | local _category |
|
183 | if "$_cmd" --help | grep -q 'For all options use the manual or "--help all".'; then |
|
184 | _category="all" |
|
185 | else |
|
186 | _category="" |
|
187 | fi |
|
188 | ||
189 | case "$_arch" in |
|
190 | *darwin*) |
|
191 | if check_cmd sw_vers; then |
|
192 | case $(sw_vers -productVersion) in |
|
193 | 10.*) |
|
194 | # If we're running on macOS, older than 10.13, then we always |
|
195 | # fail to find these options to force fallback |
|
196 | if [ "$(sw_vers -productVersion | cut -d. -f2)" -lt 13 ]; then |
|
197 | # Older than 10.13 |
|
198 | echo "Warning: Detected macOS platform older than 10.13" |
|
199 | return 1 |
|
200 | fi |
|
201 | ;; |
|
202 | 11.*) |
|
203 | # We assume Big Sur will be OK for now |
|
204 | ;; |
|
205 | *) |
|
206 | # Unknown product version, warn and continue |
|
207 | echo "Warning: Detected unknown macOS major version: $(sw_vers -productVersion)" |
|
208 | echo "Warning TLS capabilities detection may fail" |
|
209 | ;; |
|
210 | esac |
|
211 | fi |
|
212 | ;; |
|
213 | esac |
|
214 | for _arg in "$@"; do |
|
215 | if ! "$_cmd" --help "$_category" | grep -q -- "$_arg"; then |
|
216 | return 1 |
|
217 | fi |
|
218 | done |
|
219 | true # not strictly needed |
|
220 | } |
|
221 | ||
222 | # Return strong TLS 1.2-1.3 cipher suites in OpenSSL or GnuTLS syntax. TLS 1.2 |
|
223 | # excludes non-ECDHE and non-AEAD cipher suites. DHE is excluded due to bad |
|
224 | # DH params often found on servers (see RFC 7919). Sequence matches or is |
|
225 | # similar to Firefox 68 ESR with weak cipher suites disabled via about:config. |
|
226 | # $1 must be openssl or gnutls. |
|
227 | get_strong_ciphersuites_for() { |
|
228 | if [ "$1" = "openssl" ]; then |
|
229 | # OpenSSL is forgiving of unknown values, no problems with TLS 1.3 values on versions that don't support it yet. |
|
230 | echo "TLS_AES_128_GCM_SHA256:TLS_CHACHA20_POLY1305_SHA256:TLS_AES_256_GCM_SHA384:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384" |
|
231 | elif [ "$1" = "gnutls" ]; then |
|
232 | # GnuTLS isn't forgiving of unknown values, so this may require a GnuTLS version that supports TLS 1.3 even if wget doesn't. |
|
233 | # Begin with SECURE128 (and higher) then remove/add to build cipher suites. Produces same 9 cipher suites as OpenSSL but in slightly different order. |
|
234 | echo "SECURE128:-VERS-SSL3.0:-VERS-TLS1.0:-VERS-TLS1.1:-VERS-DTLS-ALL:-CIPHER-ALL:-MAC-ALL:-KX-ALL:+AEAD:+ECDHE-ECDSA:+ECDHE-RSA:+AES-128-GCM:+CHACHA20-POLY1305:+AES-256-GCM" |
|
235 | fi |
|
236 | } |
|
237 | ||
238 | # This wraps curl or wget. Try curl first, if not installed, |
|
239 | # use wget instead. |
|
240 | download() { |
|
241 | local _dld |
|
242 | local _ciphersuites |
|
243 | local _err |
|
244 | local _status |
|
245 | local _retry |
|
246 | if check_cmd curl; then |
|
247 | _dld=curl |
|
248 | elif check_cmd wget; then |
|
249 | _dld=wget |
|
250 | else |
|
251 | _dld='curl or wget' # to be used in error message of need_cmd |
|
252 | fi |
|
253 | if [ "$1" = --check ]; then |
|
254 | need_cmd "$_dld" |
|
255 | elif [ "$_dld" = curl ]; then |
|
256 | check_curl_for_retry_support |
|
257 | _retry="$RETVAL" |
|
258 | get_ciphersuites_for_curl |
|
259 | _ciphersuites="$RETVAL" |
|
260 | if [ -n "$_ciphersuites" ]; then |
|
261 | _err=$(curl $_retry --proto '=https' --tlsv1.2 --ciphers "$_ciphersuites" --silent --show-error --fail --location "$1" --output "$2" 2>&1) |
|
262 | _status=$? |
|
263 | else |
|
264 | echo "Warning: Not enforcing strong cipher suites for TLS, this is potentially less secure" |
|
265 | if ! check_help_for "$3" curl --proto --tlsv1.2; then |
|
266 | echo "Warning: Not enforcing TLS v1.2, this is potentially less secure" |
|
267 | _err=$(curl $_retry --silent --show-error --fail --location "$1" --output "$2" 2>&1) |
|
268 | _status=$? |
|
269 | else |
|
270 | _err=$(curl $_retry --proto '=https' --tlsv1.2 --silent --show-error --fail --location "$1" --output "$2" 2>&1) |
|
271 | _status=$? |
|
272 | fi |
|
273 | fi |
|
274 | if [ -n "$_err" ]; then |
|
275 | echo "$_err" >&2 |
|
276 | if echo "$_err" | grep -q 404$; then |
|
277 | err "installer for platform '$3' not found, this may be unsupported" |
|
278 | fi |
|
279 | fi |
|
280 | return $_status |
|
281 | elif [ "$_dld" = wget ]; then |
|
282 | if [ "$(wget -V 2>&1|head -2|tail -1|cut -f1 -d" ")" = "BusyBox" ]; then |
|
283 | echo "Warning: using the BusyBox version of wget. Not enforcing strong cipher suites for TLS or TLS v1.2, this is potentially less secure" |
|
284 | _err=$(wget "$1" -O "$2" 2>&1) |
|
285 | _status=$? |
|
286 | else |
|
287 | get_ciphersuites_for_wget |
|
288 | _ciphersuites="$RETVAL" |
|
289 | if [ -n "$_ciphersuites" ]; then |
|
290 | _err=$(wget --https-only --secure-protocol=TLSv1_2 --ciphers "$_ciphersuites" "$1" -O "$2" 2>&1) |
|
291 | _status=$? |
|
292 | else |
|
293 | echo "Warning: Not enforcing strong cipher suites for TLS, this is potentially less secure" |
|
294 | if ! check_help_for "$3" wget --https-only --secure-protocol; then |
|
295 | echo "Warning: Not enforcing TLS v1.2, this is potentially less secure" |
|
296 | _err=$(wget "$1" -O "$2" 2>&1) |
|
297 | _status=$? |
|
298 | else |
|
299 | _err=$(wget --https-only --secure-protocol=TLSv1_2 "$1" -O "$2" 2>&1) |
|
300 | _status=$? |
|
301 | fi |
|
302 | fi |
|
303 | fi |
|
304 | if [ -n "$_err" ]; then |
|
305 | echo "$_err" >&2 |
|
306 | if echo "$_err" | grep -q ' 404 Not Found$'; then |
|
307 | err "installer for platform '$3' not found, this may be unsupported" |
|
308 | fi |
|
309 | fi |
|
310 | return $_status |
|
311 | else |
|
312 | err "Unknown downloader" # should not reach here |
|
313 | fi |
|
314 | } |
|
315 | ||
316 | main "$@" || exit 1 |