Coverage for src/footman/_fetch.py: 94%
147 statements
« prev ^ index » next coverage.py v7.15.2, created at 2026-07-24 13:38 +0000
1"""Download files into footman's cache — `fetch()`.
3Build tasks fetch things: a toolchain tarball, a schema, a fixture. Doing
4it well means caching by URL, revalidating instead of re-downloading,
5verifying what arrived, and reporting progress — and doing it *here*
6means it composes with everything already built:
8- the cached copy lives under `footman_cache_dir()`, so
9 `FOOTMAN_CACHE_DIR` relocates it and the cache collector tends it;
10- every fetch records a `Result`, so `--dry-run` prints without
11 downloading, `recording()` asserts on it in tests, `--json` carries
12 it, and the step lines show it in the same aligned grid as `run()`;
13- byte counts feed `progress()`, so a download drives the live bar.
15**Backends.** The default is stdlib `urllib` — always present, zero
16dependencies, deterministic, and the only backend that can report bytes
17as they arrive. `curl` (shipped in Windows' System32 since build 17063,
18and on every POSIX box) is the escape hatch for corporate proxies and
19TLS stores that Python's defaults can't see; `httpx` and `requests` are
20used only when explicitly named. Choose per call, or set
21`[fetch] backend` in any config file — a machine behind a proxy sets it
22once in `~/.config/footman/config.toml` and every project follows.
24Deliberately *not* automatic: a fetch that silently picks a different
25engine depending on what happens to be importable would change its TLS
26trust store and proxy semantics when an unrelated dependency appears.
27`backend = "auto"` exists for people who want that, spelled out as a
28choice rather than a surprise.
30This is for build artifacts, not a general HTTP client. Anything exotic
31belongs in `tools.curl(...)`, which is right there.
32"""
34from __future__ import annotations
36import hashlib
37import json
38import shutil
39import time
40import urllib.error
41import urllib.request
42from pathlib import Path
43from typing import Any
45from footman import _paths, context
47BACKENDS = ("urllib", "curl", "httpx", "requests", "auto")
48_AUTO_ORDER = ("httpx", "requests", "urllib", "curl")
49CHUNK = 64 * 1024
52class FetchError(Exception):
53 """A download failed, or arrived wrong (checksum, missing backend)."""
56def cache_dir() -> Path:
57 """Where fetched files live: a `fetch/` room in footman's own cache."""
58 return _paths.footman_cache_dir() / "fetch"
61def _key(url: str) -> str:
62 return hashlib.sha256(url.encode("utf-8")).hexdigest()[:16]
65def _paths_for(url: str) -> tuple[Path, Path]:
66 """The cached body and its metadata sidecar (ETag, Last-Modified)."""
67 stem = cache_dir() / _key(url)
68 return stem.with_suffix(".bin"), stem.with_suffix(".meta.json")
71def _load_meta(path: Path) -> dict[str, Any]:
72 try:
73 data = json.loads(path.read_text("utf-8"))
74 except (OSError, ValueError):
75 return {}
76 return data if isinstance(data, dict) else {}
79def _digest(path: Path) -> str:
80 sha = hashlib.sha256()
81 with open(path, "rb") as fh:
82 for block in iter(lambda: fh.read(CHUNK), b""):
83 sha.update(block)
84 return sha.hexdigest()
87def _resolve_backend(name: str) -> str:
88 """The backend to use, refusing a named-but-missing one out loud."""
89 if name == "auto":
90 for candidate in _AUTO_ORDER: 90 ↛ 93line 90 didn't jump to line 93 because the loop on line 90 didn't complete
91 if _available(candidate):
92 return candidate
93 raise FetchError("fetch: no usable backend (not even urllib?)")
94 if name not in BACKENDS:
95 options = ", ".join(BACKENDS)
96 raise FetchError(f"fetch: unknown backend {name!r} — choose one of {options}")
97 if not _available(name):
98 if name == "curl": 98 ↛ 99line 98 didn't jump to line 99 because the condition on line 98 was never true
99 raise FetchError("fetch: backend 'curl' is not on PATH")
100 raise FetchError(
101 f"fetch: backend {name!r} is not installed — `pip install {name}`, "
102 f"or leave [fetch] backend unset to use the stdlib"
103 )
104 return name
107def _available(name: str) -> bool:
108 if name == "urllib":
109 return True
110 if name == "curl":
111 return shutil.which("curl") is not None
112 import importlib.util
114 return importlib.util.find_spec(name) is not None
117def _download(backend: str, url: str, dest: Path, meta: dict[str, Any]) -> dict:
118 """Fetch *url* into *dest*; return the new metadata (empty = unchanged)."""
119 if backend == "curl":
120 return _download_curl(url, dest, meta)
121 if backend in ("httpx", "requests"):
122 return _download_lib(backend, url, dest, meta)
123 return _download_urllib(url, dest, meta)
126def _conditional_headers(meta: dict[str, Any]) -> dict[str, str]:
127 headers = {}
128 if etag := meta.get("etag"):
129 headers["If-None-Match"] = str(etag)
130 if modified := meta.get("last_modified"): 130 ↛ 131line 130 didn't jump to line 131 because the condition on line 130 was never true
131 headers["If-Modified-Since"] = str(modified)
132 return headers
135def _download_urllib(url: str, dest: Path, meta: dict[str, Any]) -> dict:
136 request = urllib.request.Request(url, headers=_conditional_headers(meta))
137 try:
138 with urllib.request.urlopen(request) as response:
139 total = int(response.headers.get("Content-Length") or 0)
140 received = 0
141 with open(dest, "wb") as fh:
142 while chunk := response.read(CHUNK):
143 fh.write(chunk)
144 received += len(chunk)
145 if total: 145 ↛ 142line 145 didn't jump to line 142 because the condition on line 145 was always true
146 context.progress(received, total)
147 if total: 147 ↛ 149line 147 didn't jump to line 149 because the condition on line 147 was always true
148 context.progress(0, 0) # done reporting: back to the estimate
149 return {
150 "etag": response.headers.get("ETag"),
151 "last_modified": response.headers.get("Last-Modified"),
152 }
153 except urllib.error.HTTPError as exc:
154 if exc.code == 304: # not modified: the cached copy stands
155 return {}
156 raise FetchError(f"fetch: {url} — HTTP {exc.code} {exc.reason}") from exc
157 except urllib.error.URLError as exc:
158 raise FetchError(
159 f"fetch: {url} — {exc.reason}. If this machine needs the system "
160 f'curl (a corporate proxy or TLS store), set `backend = "curl"` '
161 f"under [fetch] — in this project, or once for every project in "
162 f"{_paths.footman_config_file()}"
163 ) from exc
166def _download_curl(url: str, dest: Path, meta: dict[str, Any]) -> dict:
167 import subprocess
169 argv = ["curl", "-fsSL", "--retry", "2", "-o", str(dest), url]
170 for header, value in _conditional_headers(meta).items(): 170 ↛ 171line 170 didn't jump to line 171 because the loop on line 170 never started
171 argv += ["-H", f"{header}: {value}"]
172 done = subprocess.run(argv, capture_output=True, text=True)
173 if done.returncode != 0:
174 raise FetchError(f"fetch: {url} — curl: {done.stderr.strip()}")
175 return {} # curl's revalidation story is its own; re-fetch is honest
178def _download_lib(name: str, url: str, dest: Path, meta: dict[str, Any]) -> dict:
179 import importlib
181 client = importlib.import_module(name)
182 response = (
183 client.get( # type: ignore[attr-defined]
184 url, headers=_conditional_headers(meta), follow_redirects=True
185 )
186 if name == "httpx"
187 else client.get( # type: ignore[attr-defined]
188 url, headers=_conditional_headers(meta), allow_redirects=True
189 )
190 )
191 if response.status_code == 304:
192 return {}
193 if response.status_code >= 400:
194 raise FetchError(f"fetch: {url} — HTTP {response.status_code}")
195 dest.write_bytes(response.content)
196 return {
197 "etag": response.headers.get("ETag"),
198 "last_modified": response.headers.get("Last-Modified"),
199 }
202def fetch(
203 url: str,
204 *,
205 into: Path | str | None = None,
206 sha256: str = "",
207 backend: str = "",
208 refresh: bool = False,
209) -> Path:
210 """Download *url* (cached), returning the path to the local file.
212 A second call for the same URL revalidates with the server (ETag /
213 Last-Modified) rather than re-downloading; a `304 Not Modified`
214 costs one round trip and keeps "cached" honest. Pass *refresh* to
215 skip revalidation and fetch unconditionally.
217 ```python
218 @task
219 def deps():
220 "Fetch the toolchain."
221 archive = fetch(TOOLCHAIN_URL, sha256="9f86d0…")
222 tools.tar("-xzf", archive, "-C", "vendor")
223 ```
225 *into* copies the cached file to a path of your choosing (and
226 returns that path). *sha256* verifies what arrived and refuses a
227 mismatch — the way to make a build reproducible. *backend* overrides
228 the configured one for this call.
230 Under `--dry-run` nothing is downloaded: the step is recorded and
231 the would-be cache path returned, so a plan can be inspected safely.
232 """
233 ctx = context.current()
234 label = f"fetch {url}"
235 body, sidecar = _paths_for(url)
236 destination = Path(into) if into is not None else body
238 if ctx.dry_run:
239 ctx.steps.append(context.Result(0, command=label, raw=label))
240 if not ctx.quiet: 240 ↛ 242line 240 didn't jump to line 242 because the condition on line 240 was always true
241 print(f"$ {label}")
242 return destination
244 chosen = _resolve_backend(backend or _configured_backend(ctx))
245 started = time.perf_counter()
246 cache_dir().mkdir(parents=True, exist_ok=True)
247 meta = {} if refresh else _load_meta(sidecar)
248 try:
249 fresh = _download(chosen, url, body, meta if body.exists() else {})
250 except FetchError:
251 if body.exists(): # a cached copy beats a failed refresh
252 _record(ctx, label, started, cached=True)
253 return _deliver(body, destination, sha256, url)
254 raise
255 if fresh:
256 sidecar.write_text(json.dumps(fresh), encoding="utf-8")
257 _record(ctx, label, started, cached=not fresh)
258 return _deliver(body, destination, sha256, url)
261def _configured_backend(ctx: context.Context) -> str:
262 """`[fetch] backend` from the config ladder, defaulting to urllib."""
263 configured = getattr(ctx, "fetch_backend", "") or ""
264 return str(configured) or "urllib"
267def _deliver(body: Path, destination: Path, sha256: str, url: str) -> Path:
268 if sha256:
269 actual = _digest(body)
270 if actual != sha256.lower():
271 raise FetchError(
272 f"fetch: {url} — sha256 mismatch\n expected {sha256.lower()}\n"
273 f" received {actual}"
274 )
275 if destination != body:
276 destination.parent.mkdir(parents=True, exist_ok=True)
277 shutil.copyfile(body, destination)
278 return destination
281def _record(ctx: context.Context, label: str, started: float, *, cached: bool) -> None:
282 """A fetch is a step: same grid, same --json entry, same recording()."""
283 note = "cached" if cached else ""
284 ctx.steps.append(
285 context.Result(
286 0,
287 command=label,
288 stdout=note,
289 duration=time.perf_counter() - started,
290 raw=label,
291 )
292 )