返回 CodeWhale
dev-cache.sh
根目录 / scripts / dev-cache.sh
1 #!/bin/sh
2 # Portable opt-in Cargo cache topology for Codewhale worktrees.
3 #
4 # Isolated per-workspace intermediates (Cargo build-dir + {workspace-path-hash})
5 # plus optional shared sccache. Never hard-codes a machine path. Does not
6 # change product behavior. This file is safe to source from other /bin/sh
7 # helpers; the CLI path is the only one that calls `set -eu` and `exit`.
8 #
9 # Usage:
10 # . scripts/dev-cache.sh && codewhale_dev_cache_apply
11 # scripts/dev-cache.sh --status
12 # scripts/dev-cache.sh --self-check
13 # scripts/dev-cache.sh --print-exports
14 # scripts/dev-cache.sh --help
15 #
16 # Environment:
17 # CODEWHALE_CACHE_ROOT cache root (default:
18 # ${XDG_CACHE_HOME:-$HOME/.cache}/codewhale)
19 # CODEWHALE_DEV_CACHE auto|1|force|local|0 (default auto)
20 # auto/1/force = isolated build-dir (still
21 # respects an already-set
22 # CARGO_TARGET_DIR / CARGO_BUILD_BUILD_DIR)
23 # local = keep compiling into ./target
24 # 0 = do nothing
25 # CODEWHALE_SCCACHE auto|1|0 (default auto)
26 # auto = wrap rustc only when incremental is
27 # already off and sccache is on PATH
28 # 1 = request CARGO_INCREMENTAL=0 (unless
29 # the caller already set it to on) and
30 # wrap if sccache exists
31 # 0 = never wrap
32 # CODEWHALE_DEV_CACHE_QUIET 1 to suppress the one-line status
33 # CODEWHALE_DEV_CACHE_REPO_ROOT
34 # workspace root used for ./target detection
35 # (default: $PWD)
36 # CODEWHALE_RUSTC_COMMIT override the rustc commit used to namespace
37 # SCCACHE_DIR (tests / offline hosts)
38 #
39 # sccache cannot cache incremental units. This helper never enables the
40 # wrapper while CARGO_INCREMENTAL is on, and it never turns incremental off
41 # unless CODEWHALE_SCCACHE=1. Everyday edits stay on cargo's incremental
42 # default. New-worktree / CI-like rebuilds that already set
43 # CARGO_INCREMENTAL=0 get the measured shared-compiler-output path.
44
45 codewhale_dev_cache_truthy() {
46 case ${1:-} in
47 1|true|yes|on|force) return 0 ;;
48 *) return 1 ;;
49 esac
50 }
51
52 codewhale_dev_cache_falsey() {
53 case ${1:-} in
54 0|false|no|off|never|disabled) return 0 ;;
55 *) return 1 ;;
56 esac
57 }
58
59 codewhale_dev_cache_root() {
60 if [ -n "${CODEWHALE_CACHE_ROOT:-}" ]; then
61 printf '%s\n' "$CODEWHALE_CACHE_ROOT"
62 return
63 fi
64 if [ -n "${XDG_CACHE_HOME:-}" ]; then
65 printf '%s\n' "$XDG_CACHE_HOME/codewhale"
66 return
67 fi
68 if [ -n "${HOME:-}" ]; then
69 printf '%s\n' "$HOME/.cache/codewhale"
70 return
71 fi
72 printf '%s\n' "${TMPDIR:-/tmp}/codewhale-cache"
73 }
74
75 codewhale_dev_cache_repo_root() {
76 if [ -n "${CODEWHALE_DEV_CACHE_REPO_ROOT:-}" ]; then
77 printf '%s\n' "$CODEWHALE_DEV_CACHE_REPO_ROOT"
78 return
79 fi
80 pwd
81 }
82
83 codewhale_dev_cache_rustc_commit() {
84 if [ -n "${CODEWHALE_RUSTC_COMMIT:-}" ]; then
85 printf '%s\n' "$CODEWHALE_RUSTC_COMMIT"
86 return
87 fi
88 # rustc -vV is cheap and does not start a compile.
89 _cw_out=$(rustc -vV 2>/dev/null || true)
90 _cw_commit=$(printf '%s\n' "$_cw_out" | awk '/^commit-hash:/{print $2; exit}')
91 if [ -n "$_cw_commit" ]; then
92 printf '%s\n' "$_cw_commit"
93 else
94 printf '%s\n' "unknown"
95 fi
96 }
97
98 codewhale_dev_cache_cargo_version_mm() {
99 # cargo 1.97.0 -> 1097. Empty if cargo is missing or unparsable.
100 cargo --version 2>/dev/null | awk '{
101 split($2, a, ".")
102 if (a[1] ~ /^[0-9]+$/ && a[2] ~ /^[0-9]+$/)
103 printf "%d\n", a[1]*1000+a[2]
104 }'
105 }
106
107 # Stable 16-hex-char fingerprint of a path. Used only for the pre-1.91
108 # CARGO_TARGET_DIR fallback; Cargo 1.91+ expands {workspace-path-hash}.
109 codewhale_dev_cache_path_fingerprint() {
110 if command -v sha256sum >/dev/null 2>&1; then
111 printf '%s' "$1" | sha256sum | awk '{print substr($1,1,16)}'
112 elif command -v shasum >/dev/null 2>&1; then
113 printf '%s' "$1" | shasum -a 256 | awk '{print substr($1,1,16)}'
114 elif command -v openssl >/dev/null 2>&1; then
115 printf '%s' "$1" | openssl dgst -sha256 | awk '{print substr($NF,1,16)}'
116 else
117 printf '%s' "$1" | cksum | awk '{printf "%s%08s\n", $1, $2}' | tr ' ' '0'
118 fi
119 }
120
121 codewhale_dev_cache_incremental_off() {
122 case ${CARGO_INCREMENTAL:-} in
123 0|false|no|off) return 0 ;;
124 *) return 1 ;;
125 esac
126 }
127
128 codewhale_dev_cache_incremental_on() {
129 case ${CARGO_INCREMENTAL:-} in
130 1|true|yes|on) return 0 ;;
131 *) return 1 ;;
132 esac
133 }
134
135 codewhale_dev_cache_sccache_bin() {
136 command -v sccache 2>/dev/null || true
137 }
138
139 codewhale_dev_cache_log() {
140 if [ "${CODEWHALE_DEV_CACHE_QUIET:-}" = 1 ]; then
141 return 0
142 fi
143 printf 'dev-cache: %s\n' "$*" >&2
144 }
145
146 # Free space (in GiB, integer, truncated) on the filesystem holding $1, or
147 # empty when it cannot be determined. `df -Pk` is the POSIX-portable form and
148 # behaves the same on macOS and Linux.
149 codewhale_dev_cache_free_gib() {
150 _cw_probe=$1
151 # Walk up to the nearest existing ancestor: the cache root usually does not
152 # exist yet on a first run, and df needs a real path.
153 while [ -n "$_cw_probe" ] && [ ! -d "$_cw_probe" ]; do
154 _cw_parent=$(dirname -- "$_cw_probe")
155 [ "$_cw_parent" = "$_cw_probe" ] && break
156 _cw_probe=$_cw_parent
157 done
158 [ -d "$_cw_probe" ] || return 0
159 df -Pk -- "$_cw_probe" 2>/dev/null | awk 'NR==2 { print int($4 / 1048576) }'
160 }
161
162 # A cold workspace build-dir is ~6 GB, and cargo writes it incrementally, so a
163 # volume that runs out mid-build leaves a half-written cache and a failed run.
164 # Worse, on the host where this was added the low volume was also $TMPDIR's:
165 # the shell tool's spill `tempfile` fails on a full disk, which is the exact
166 # wedge #5465 was filed for. Warn before the build starts rather than after it
167 # dies, and name the override that fixes it.
168 CODEWHALE_DEV_CACHE_MIN_FREE_GIB=${CODEWHALE_DEV_CACHE_MIN_FREE_GIB:-15}
169
170 codewhale_dev_cache_warn_low_space() {
171 _cw_root=$1
172 _cw_free=$(codewhale_dev_cache_free_gib "$_cw_root")
173 [ -n "$_cw_free" ] || return 0
174 [ "$_cw_free" -lt "$CODEWHALE_DEV_CACHE_MIN_FREE_GIB" ] || return 0
175 codewhale_dev_cache_log \
176 "WARNING: only ${_cw_free}GiB free on the volume holding ${_cw_root}; a cold build-dir is ~6GB."
177 codewhale_dev_cache_log \
178 " Set CODEWHALE_CACHE_ROOT=<dir on a roomier volume> (threshold: CODEWHALE_DEV_CACHE_MIN_FREE_GIB)."
179 }
180
181 # Apply the topology to the current shell. Idempotent. Never overrides an
182 # already-set CARGO_TARGET_DIR, CARGO_BUILD_BUILD_DIR, RUSTC_WRAPPER, or
183 # SCCACHE_DIR.
184 codewhale_dev_cache_apply() {
185 CODEWHALE_DEV_CACHE_MODE=${CODEWHALE_DEV_CACHE_MODE:-}
186 # Re-entry after a previous apply in this shell: keep the first decision
187 # unless the caller unset the mode (tests do that between cases).
188 if [ -n "${CODEWHALE_DEV_CACHE_APPLIED:-}" ] && [ -n "${CODEWHALE_DEV_CACHE_MODE:-}" ]; then
189 return 0
190 fi
191
192 _cw_want=${CODEWHALE_DEV_CACHE:-auto}
193 _cw_root=$(codewhale_dev_cache_root)
194 _cw_sccache_want=${CODEWHALE_SCCACHE:-auto}
195 _cw_commit=$(codewhale_dev_cache_rustc_commit)
196 CODEWHALE_DEV_CACHE_SCCACHE_DIR=${SCCACHE_DIR:-${_cw_root}/sccache/${_cw_commit}}
197 CODEWHALE_DEV_CACHE_BUILD_DIR=${_cw_root}/build/'{workspace-path-hash}'
198 CODEWHALE_DEV_CACHE_LEGACY_TARGET_DIR=${_cw_root}/target/$(codewhale_dev_cache_path_fingerprint "$(codewhale_dev_cache_repo_root)")
199
200 if codewhale_dev_cache_falsey "$_cw_want"; then
201 CODEWHALE_DEV_CACHE_MODE=disabled
202 CODEWHALE_DEV_CACHE_SCCACHE=disabled
203 CODEWHALE_DEV_CACHE_APPLIED=1
204 export CODEWHALE_DEV_CACHE_MODE CODEWHALE_DEV_CACHE_SCCACHE
205 codewhale_dev_cache_log "disabled (CODEWHALE_DEV_CACHE=${_cw_want})"
206 return 0
207 fi
208
209 if [ -n "${CARGO_TARGET_DIR:-}" ]; then
210 CODEWHALE_DEV_CACHE_MODE=inherited-target-dir
211 elif [ -n "${CARGO_BUILD_BUILD_DIR:-}" ]; then
212 CODEWHALE_DEV_CACHE_MODE=inherited-build-dir
213 elif [ "$_cw_want" = local ]; then
214 # Explicit stay-in-./target. A stub target/ created by a previous
215 # isolated build-dir run is not a reason to abandon isolation: Cargo
216 # still writes CACHEDIR.TAG (and sometimes finals) under ./target
217 # when build-dir is split, and treating that as "warm" made the
218 # second command recompile everything into a second tree.
219 CODEWHALE_DEV_CACHE_MODE=existing-target
220 else
221 CODEWHALE_DEV_CACHE_MODE=isolated-build-dir
222 fi
223
224 case $CODEWHALE_DEV_CACHE_MODE in
225 force-isolated|isolated-build-dir)
226 _cw_mm=$(codewhale_dev_cache_cargo_version_mm)
227 if [ -n "$_cw_mm" ] && [ "$_cw_mm" -lt 1091 ]; then
228 # Cargo 1.91 introduced build.build-dir. Older cargo only isolates
229 # via CARGO_TARGET_DIR; keep that fallback per-workspace.
230 if [ -z "${CARGO_TARGET_DIR:-}" ]; then
231 CARGO_TARGET_DIR=$CODEWHALE_DEV_CACHE_LEGACY_TARGET_DIR
232 export CARGO_TARGET_DIR
233 fi
234 CODEWHALE_DEV_CACHE_MODE=${CODEWHALE_DEV_CACHE_MODE}-legacy-target-dir
235 else
236 CARGO_BUILD_BUILD_DIR=$CODEWHALE_DEV_CACHE_BUILD_DIR
237 export CARGO_BUILD_BUILD_DIR
238 mkdir -p "$_cw_root/build" 2>/dev/null || true
239 fi
240 ;;
241 esac
242
243 # sccache: never wrap incremental rustc. Missing binary is a fallback,
244 # not an error.
245 if codewhale_dev_cache_falsey "$_cw_sccache_want"; then
246 CODEWHALE_DEV_CACHE_SCCACHE=disabled
247 elif [ -n "${RUSTC_WRAPPER:-}" ]; then
248 case $RUSTC_WRAPPER in
249 *sccache*) CODEWHALE_DEV_CACHE_SCCACHE=external-sccache ;;
250 *) CODEWHALE_DEV_CACHE_SCCACHE=external-wrapper ;;
251 esac
252 else
253 if codewhale_dev_cache_truthy "$_cw_sccache_want" && ! codewhale_dev_cache_incremental_on; then
254 # Request the measured new-worktree / CI-like topology.
255 if [ -z "${CARGO_INCREMENTAL:-}" ]; then
256 CARGO_INCREMENTAL=0
257 export CARGO_INCREMENTAL
258 fi
259 fi
260 if ! codewhale_dev_cache_incremental_off; then
261 CODEWHALE_DEV_CACHE_SCCACHE=skipped-incremental
262 else
263 _cw_bin=$(codewhale_dev_cache_sccache_bin)
264 if [ -z "$_cw_bin" ]; then
265 CODEWHALE_DEV_CACHE_SCCACHE=not-found
266 else
267 RUSTC_WRAPPER=$_cw_bin
268 export RUSTC_WRAPPER
269 if [ -z "${SCCACHE_DIR:-}" ]; then
270 SCCACHE_DIR=$CODEWHALE_DEV_CACHE_SCCACHE_DIR
271 export SCCACHE_DIR
272 fi
273 mkdir -p "$SCCACHE_DIR" 2>/dev/null || true
274 CODEWHALE_DEV_CACHE_SCCACHE=enabled
275 fi
276 fi
277 fi
278
279 CODEWHALE_DEV_CACHE_APPLIED=1
280 export CODEWHALE_DEV_CACHE_MODE CODEWHALE_DEV_CACHE_SCCACHE
281 export CODEWHALE_DEV_CACHE_SCCACHE_DIR
282
283 _cw_line="mode=${CODEWHALE_DEV_CACHE_MODE} sccache=${CODEWHALE_DEV_CACHE_SCCACHE}"
284 case $CODEWHALE_DEV_CACHE_MODE in
285 isolated-build-dir*|force-isolated*)
286 _cw_line="${_cw_line} build-dir=${CARGO_BUILD_BUILD_DIR:-${CARGO_TARGET_DIR:-unset}}"
287 ;;
288 inherited-target-dir)
289 _cw_line="${_cw_line} CARGO_TARGET_DIR=${CARGO_TARGET_DIR}"
290 ;;
291 inherited-build-dir)
292 _cw_line="${_cw_line} CARGO_BUILD_BUILD_DIR=${CARGO_BUILD_BUILD_DIR}"
293 ;;
294 existing-target)
295 _cw_line="${_cw_line} using ./target (CODEWHALE_DEV_CACHE=local)"
296 ;;
297 esac
298 if [ "${CODEWHALE_DEV_CACHE_SCCACHE}" = enabled ]; then
299 _cw_line="${_cw_line} SCCACHE_DIR=${SCCACHE_DIR} rustc=${_cw_commit}"
300 fi
301 codewhale_dev_cache_log "$_cw_line"
302 case $CODEWHALE_DEV_CACHE_MODE in
303 disabled|existing-target) ;;
304 *) codewhale_dev_cache_warn_low_space "$(codewhale_dev_cache_root)" ;;
305 esac
306 }
307
308 codewhale_dev_cache_status() {
309 _cw_tmpl=${CODEWHALE_DEV_CACHE_BUILD_DIR:-}
310 if [ -z "$_cw_tmpl" ]; then
311 _cw_tmpl=$(codewhale_dev_cache_root)/build/'{workspace-path-hash}'
312 fi
313 printf '%s\n' \
314 "cache_root=$(codewhale_dev_cache_root)" \
315 "repo_root=$(codewhale_dev_cache_repo_root)" \
316 "mode=${CODEWHALE_DEV_CACHE_MODE:-unset}" \
317 "sccache=${CODEWHALE_DEV_CACHE_SCCACHE:-unset}" \
318 "build_dir_template=${_cw_tmpl}" \
319 "CARGO_BUILD_BUILD_DIR=${CARGO_BUILD_BUILD_DIR:-<unset>}" \
320 "CARGO_TARGET_DIR=${CARGO_TARGET_DIR:-<unset>}" \
321 "CARGO_INCREMENTAL=${CARGO_INCREMENTAL:-<unset>}" \
322 "RUSTC_WRAPPER=${RUSTC_WRAPPER:-<unset>}" \
323 "SCCACHE_DIR=${SCCACHE_DIR:-<unset>}" \
324 "rustc_commit=$(codewhale_dev_cache_rustc_commit)" \
325 "sccache_bin=$(codewhale_dev_cache_sccache_bin)"
326 }
327
328 # Extra cargo CLI flags that force template expansion of build-dir.
329 # CARGO_BUILD_BUILD_DIR is still exported for nested tools; --config is
330 # what Cargo documents as supporting {workspace-path-hash}.
331 codewhale_dev_cache_exec_cargo() {
332 case ${CODEWHALE_DEV_CACHE_MODE:-} in
333 isolated-build-dir|force-isolated)
334 if [ -n "${CARGO_BUILD_BUILD_DIR:-}" ]; then
335 set -- --config "build.build-dir = \"${CARGO_BUILD_BUILD_DIR}\"" "$@"
336 fi
337 ;;
338 esac
339 # One Cargo build per machine: separate target dirs do not make concurrent
340 # builds safe on a memory-constrained host (scripts/build-lock.py).
341 _cw_lock_script=${repo_root:-.}/scripts/build-lock.py
342 if ! codewhale_dev_cache_falsey "${CODEWHALE_BUILD_LOCK:-1}" \
343 && [ -z "${CODEWHALE_BUILD_LOCK_HELD:-}" ] \
344 && [ -f "$_cw_lock_script" ] \
345 && command -v python3 >/dev/null 2>&1; then
346 exec python3 "$_cw_lock_script" "${CODEWHALE_BUILD_LOCK_FILE:-$(codewhale_dev_cache_root)/build.lock}" -- cargo "$@"
347 fi
348 exec cargo "$@"
349 }
350
351 codewhale_dev_cache_print_exports() {
352 [ -n "${CARGO_BUILD_BUILD_DIR:-}" ] && printf 'export CARGO_BUILD_BUILD_DIR=%s\n' "$CARGO_BUILD_BUILD_DIR"
353 [ -n "${CARGO_TARGET_DIR:-}" ] && printf 'export CARGO_TARGET_DIR=%s\n' "$CARGO_TARGET_DIR"
354 [ -n "${CARGO_INCREMENTAL:-}" ] && printf 'export CARGO_INCREMENTAL=%s\n' "$CARGO_INCREMENTAL"
355 [ -n "${RUSTC_WRAPPER:-}" ] && printf 'export RUSTC_WRAPPER=%s\n' "$RUSTC_WRAPPER"
356 [ -n "${SCCACHE_DIR:-}" ] && printf 'export SCCACHE_DIR=%s\n' "$SCCACHE_DIR"
357 [ -n "${CODEWHALE_DEV_CACHE_MODE:-}" ] && printf 'export CODEWHALE_DEV_CACHE_MODE=%s\n' "$CODEWHALE_DEV_CACHE_MODE"
358 [ -n "${CODEWHALE_DEV_CACHE_SCCACHE:-}" ] && printf 'export CODEWHALE_DEV_CACHE_SCCACHE=%s\n' "$CODEWHALE_DEV_CACHE_SCCACHE"
359 }
360
361 codewhale_dev_cache_self_check() {
362 _cw_fail=0
363 _cw_root=$(codewhale_dev_cache_root)
364 _cw_home=${HOME:-}
365 _cw_xdg=${XDG_CACHE_HOME:-}
366
367 if [ -z "${CODEWHALE_CACHE_ROOT:-}" ]; then
368 if [ -n "$_cw_xdg" ]; then
369 case $_cw_root in
370 "$_cw_xdg"/codewhale) ;;
371 *)
372 printf 'self-check: expected XDG default %s/codewhale, got %s\n' "$_cw_xdg" "$_cw_root" >&2
373 _cw_fail=1
374 ;;
375 esac
376 elif [ -n "$_cw_home" ]; then
377 case $_cw_root in
378 "$_cw_home"/.cache/codewhale) ;;
379 *)
380 printf 'self-check: expected HOME default %s/.cache/codewhale, got %s\n' "$_cw_home" "$_cw_root" >&2
381 _cw_fail=1
382 ;;
383 esac
384 fi
385 fi
386
387 case ${CARGO_BUILD_BUILD_DIR:-} in
388 *'{workspace-path-hash}'*)
389 ;;
390 '')
391 case ${CODEWHALE_DEV_CACHE_MODE:-} in
392 isolated-build-dir|force-isolated)
393 printf 'self-check: isolated mode did not set CARGO_BUILD_BUILD_DIR\n' >&2
394 _cw_fail=1
395 ;;
396 esac
397 ;;
398 *)
399 case ${CODEWHALE_DEV_CACHE_MODE:-} in
400 isolated-build-dir|force-isolated)
401 printf 'self-check: isolated build-dir lacks {workspace-path-hash}: %s\n' "$CARGO_BUILD_BUILD_DIR" >&2
402 _cw_fail=1
403 ;;
404 esac
405 ;;
406 esac
407
408 if [ "${CODEWHALE_DEV_CACHE_SCCACHE:-}" = enabled ]; then
409 if ! codewhale_dev_cache_incremental_off; then
410 printf 'self-check: sccache enabled while incremental is not off\n' >&2
411 _cw_fail=1
412 fi
413 case ${RUSTC_WRAPPER:-} in
414 *sccache*) ;;
415 *)
416 printf 'self-check: sccache enabled but RUSTC_WRAPPER=%s\n' "${RUSTC_WRAPPER:-<unset>}" >&2
417 _cw_fail=1
418 ;;
419 esac
420 fi
421
422 if [ "${CODEWHALE_DEV_CACHE_SCCACHE:-}" = not-found ]; then
423 if [ -n "${RUSTC_WRAPPER:-}" ]; then
424 printf 'self-check: missing sccache must not set RUSTC_WRAPPER (%s)\n' "$RUSTC_WRAPPER" >&2
425 _cw_fail=1
426 fi
427 fi
428
429 if [ "$_cw_fail" -eq 0 ]; then
430 printf 'dev-cache: self-check ok\n'
431 return 0
432 fi
433 printf 'dev-cache: self-check FAILED\n' >&2
434 return 1
435 }
436
437 codewhale_dev_cache_usage() {
438 cat <<'EOF'
439 usage: scripts/dev-cache.sh --status|--self-check|--print-exports|--help
440
441 Portable opt-in Cargo cache topology. Source this file and call
442 codewhale_dev_cache_apply from scripts/dev-test.sh / scripts/dev-cargo.sh.
443
444 --status apply (if needed) and print key=value status
445 --self-check apply and check portable defaults / sccache fallback
446 --print-exports apply and print export statements for eval
447 --help this message
448
449 Environment overrides are documented at the top of this file. Defaults
450 never contain a machine-specific absolute path.
451 EOF
452 }
453
454 # CLI only when this file is the executed program, not when sourced.
455 case ${0##*/} in
456 dev-cache.sh)
457 set -eu
458 _cw_cmd=${1:---status}
459 case $_cw_cmd in
460 --status|-s)
461 codewhale_dev_cache_apply
462 codewhale_dev_cache_status
463 ;;
464 --self-check)
465 codewhale_dev_cache_apply
466 codewhale_dev_cache_status
467 codewhale_dev_cache_self_check
468 ;;
469 --print-exports)
470 codewhale_dev_cache_apply
471 codewhale_dev_cache_print_exports
472 ;;
473 --help|-h)
474 codewhale_dev_cache_usage
475 ;;
476 *)
477 codewhale_dev_cache_usage >&2
478 exit 2
479 ;;
480 esac
481 ;;
482 esac
483
483 lines BASH