hugFace 下载#

1
#!/usr/bin/env bash
2
RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; DIM='\033[2m'; BOLD='\033[1m'; NC='\033[0m' # No Color
3

4
trap 'printf "\n%bInterrupted. Re-run to resume.%b\n" "$YELLOW" "$NC"; exit 130' INT
5

6
# Format a byte count as a human-readable string (decimal units, matching the HF UI).
7
human() {
8
    awk -v b="${1:-0}" 'BEGIN{u="B KB MB GB TB PB";n=split(u,a," ");i=1;
9
        while(b>=1000&&i<n){b/=1000;i++} printf (i==1?"%d%s":"%.2f%s"),b,a[i]}'
10
}
11

12
display_help() {
13
    cat << EOF
14
Usage:
15
  hfd <REPO_ID> [--include include_pattern1 include_pattern2 ...] [--exclude exclude_pattern1 exclude_pattern2 ...] [--hf_username username] [--hf_token token] [--tool aria2c|wget] [-x threads] [-j jobs] [--dataset] [--local-dir path] [--revision rev]
16

17
Description:
18
  Downloads a model or dataset from Hugging Face using the provided repo ID.
19

20
Arguments:
21
  REPO_ID         The Hugging Face repo ID (Required)
22
                  Format: 'org_name/repo_name' or legacy format (e.g., gpt2)
23
Options:
24
  include/exclude_pattern The patterns to match against file path, supports wildcard characters.
25
                  e.g., '--exclude *.safetensor *.md', '--include vae/*'.
26
  --include       (Optional) Patterns to include files for downloading (supports multiple patterns).
27
  --exclude       (Optional) Patterns to exclude files from downloading (supports multiple patterns).
28
  --hf_username   (Optional) Hugging Face username for authentication (not email).
29
  --hf_token      (Optional) Hugging Face token for authentication.
30
  --tool          (Optional) Download tool to use: aria2c (default) or wget.
31
  -x              (Optional) Number of download threads for aria2c (default: 4).
32
  -j              (Optional) Number of concurrent downloads for aria2c (default: 5).
33
  --dataset       (Optional) Flag to indicate downloading a dataset.
34
  --local-dir     (Optional) Directory path to store the downloaded data.
35
                             Defaults to the current directory with a subdirectory named 'repo_name'
36
                             if REPO_ID is composed of 'org_name/repo_name'.
37
  --revision      (Optional) Model/Dataset revision to download (default: main).
38

39
Example:
40
  hfd gpt2
41
  hfd bigscience/bloom-560m --exclude *.safetensors
42
  hfd meta-llama/Llama-2-7b --hf_username myuser --hf_token mytoken -x 4
43
  hfd lavita/medical-qa-shared-task-v1-toy --dataset
44
  hfd bartowski/Phi-3.5-mini-instruct-exl2 --revision 5_0
45
EOF
46
    exit 1
47
}
48

49
[[ -z "$1" || "$1" =~ ^-h || "$1" =~ ^--help ]] && display_help
50

51
REPO_ID=$1
52
shift
53

54
# Default values
55
TOOL="aria2c"
56
THREADS=4
57
CONCURRENT=5
58
HF_ENDPOINT=${HF_ENDPOINT:-"https://huggingface.co"}
59
INCLUDE_PATTERNS=()
60
EXCLUDE_PATTERNS=()
61
REVISION="main"
62

63
validate_number() {
64
    [[ "$2" =~ ^[1-9][0-9]*$ && "$2" -le "$3" ]] || { printf "%b[Error] %s must be 1-%s%b\n" "$RED" "$1" "$3" "$NC"; exit 1; }
65
}
66

67
# Argument parsing
68
while [[ $# -gt 0 ]]; do
69
    case $1 in
70
        --include) shift; while [[ $# -gt 0 && ! ($1 =~ ^--) && ! ($1 =~ ^-[^-]) ]]; do INCLUDE_PATTERNS+=("$1"); shift; done ;;
71
        --exclude) shift; while [[ $# -gt 0 && ! ($1 =~ ^--) && ! ($1 =~ ^-[^-]) ]]; do EXCLUDE_PATTERNS+=("$1"); shift; done ;;
72
        --hf_username) HF_USERNAME="$2"; shift 2 ;;
73
        --hf_token) HF_TOKEN="$2"; shift 2 ;;
74
        --tool)
75
            [[ "$2" == aria2c || "$2" == wget ]] || { printf "%b[Error] Invalid tool. Use 'aria2c' or 'wget'.%b\n" "$RED" "$NC"; exit 1; }
76
            TOOL="$2"; shift 2 ;;
77
        -x) validate_number "threads (-x)" "$2" 10; THREADS="$2"; shift 2 ;;
78
        -j) validate_number "concurrent downloads (-j)" "$2" 10; CONCURRENT="$2"; shift 2 ;;
79
        --dataset) DATASET=1; shift ;;
80
        --local-dir) LOCAL_DIR="$2"; shift 2 ;;
81
        --revision) REVISION="$2"; shift 2 ;;
82
        *) display_help ;;
83
    esac
84
done
85

86
# A fingerprint of the options that affect the file list; a change forces regeneration.
87
generate_command_string() {
88
    printf 'REPO_ID=%s TOOL=%s INCLUDE=%s EXCLUDE=%s DATASET=%s HF_USERNAME=%s HF_TOKEN=%s HF_ENDPOINT=%s REVISION=%s' \
89
        "$REPO_ID" "$TOOL" "${INCLUDE_PATTERNS[*]}" "${EXCLUDE_PATTERNS[*]}" "${DATASET:-0}" \
90
        "${HF_USERNAME:-}" "${HF_TOKEN:-}" "${HF_ENDPOINT:-}" "$REVISION"
91
}
92

93
check_command() {
94
    if ! command -v "$1" &>/dev/null; then
95
        printf "%b%s is not installed. Please install it first.%b\n" "$RED" "$1" "$NC"
96
        exit 1
97
    fi
98
}
99

100
check_command curl; check_command "$TOOL"
101

102
LOCAL_DIR="${LOCAL_DIR:-${REPO_ID#*/}}"
103
mkdir -p "$LOCAL_DIR/.hfd"
104

105
REPO_API_PATH="models/$REPO_ID"; DOWNLOAD_API_PATH="$REPO_ID"
106
[[ "$DATASET" == 1 ]] && { REPO_API_PATH="datasets/$REPO_ID"; DOWNLOAD_API_PATH=$REPO_API_PATH; }
107

108
# wget --cut-dirs strips "<download_api_path>/resolve/<revision>/"; a fixed value wrongly
109
# assumes the org/name form and eats a directory level for legacy (single-name) repo ids.
110
CUT_DIRS=$(( $(printf '%s' "$DOWNLOAD_API_PATH" | tr -cd '/' | wc -c) + 3 ))
111

112
# Metadata API URL (used for the gated/auth check); append revision when not main.
113
METADATA_API_PATH="$REPO_API_PATH"
114
[[ "$REVISION" != "main" ]] && METADATA_API_PATH="$METADATA_API_PATH/revision/$REVISION"
115
API_URL="$HF_ENDPOINT/api/$METADATA_API_PATH?blobs=true"
116

117
METADATA_FILE="$LOCAL_DIR/.hfd/repo_metadata.json"
118

119
fetch_and_save_metadata() {
120
    status_code=$(curl -L -s -w "%{http_code}" -o "$METADATA_FILE" ${HF_TOKEN:+-H "Authorization: Bearer $HF_TOKEN"} "$API_URL")
121
    RESPONSE=$(cat "$METADATA_FILE")
122
    if [ "$status_code" -eq 200 ]; then
123
        printf "%s\n" "$RESPONSE"
124
    else
125
        printf "%b[Error] Failed to fetch metadata from %s. HTTP status code: %s.%b\n%s\n" "$RED" "$API_URL" "$status_code" "$NC" "$RESPONSE" >&2
126
        rm -f "$METADATA_FILE"
127
        exit 1
128
    fi
129
}
130

131
# Exit early if the repo is gated/private but no credentials were supplied.
132
check_authentication() {
133
    local gated
134
    if command -v jq &>/dev/null; then
135
        gated=$(printf '%s' "$1" | jq -r '.gated // false')
136
    else
137
        printf '%s' "$1" | grep -q '"gated":[^f]' && gated=true || gated=false
138
    fi
139
    if [[ "$gated" != "false" && ( -z "$HF_TOKEN" || -z "$HF_USERNAME" ) ]]; then
140
        printf "%bThe repository requires authentication, but --hf_username and --hf_token is not passed. Please get token from https://huggingface.co/settings/tokens.\nExiting.\n%b" "$RED" "$NC"
141
        exit 1
142
    fi
143
}
144

145
printf "%b%s%b (%s)\n" "$BOLD" "$REPO_ID" "$NC" "$REVISION"
146
if [[ ! -f "$METADATA_FILE" ]]; then
147
    printf "%bFetching metadata...%b\n" "$DIM" "$NC"
148
    RESPONSE=$(fetch_and_save_metadata) || exit 1
149
else
150
    RESPONSE=$(cat "$METADATA_FILE")
151
fi
152
check_authentication "$RESPONSE"
153

154
# Total bytes of the revision (cheap, branch-specific); empty if the endpoint lacks this API.
155
fetch_treesize() {
156
    local json
157
    json=$(curl -sSL ${HF_TOKEN:+-H "Authorization: Bearer $HF_TOKEN"} "$HF_ENDPOINT/api/$REPO_API_PATH/treesize/$REVISION") || return
158
    if command -v jq &>/dev/null; then
159
        printf '%s' "$json" | jq -r '.size // empty' 2>/dev/null
160
    else
161
        printf '%s' "$json" | grep -o '"size":[0-9]*' | head -1 | grep -o '[0-9]*'
162
    fi
163
}
164
# Reuse the cached list only if the command is unchanged AND the manifest is intact (its line
165
# count equals repo_info's total). An interrupted/failed walk leaves no repo_info, so it re-lists.
166
should_regenerate_filelist() {
167
    local cmd="$LOCAL_DIR/.hfd/last_download_command" mf="$LOCAL_DIR/.hfd/manifest" info="$LOCAL_DIR/.hfd/repo_info"
168
    [[ -f "$mf" && -f "$info" && "$(generate_command_string)" == "$(cat "$cmd" 2>/dev/null)" \
169
       && "$(wc -l < "$mf")" == "$(cut -d' ' -f1 "$info" 2>/dev/null)" ]] && return 1
170
    return 0
171
}
172

173
fileslist_file=".hfd/${TOOL}_urls.txt"
174

175
# Convert a list of wildcard patterns into a single alternation regex.
176
patterns_to_regex() {
177
    (($#)) || return 0
178
    printf '%s\n' "$@" | sed 's/\./\\./g; s/\*/.*/g' | paste -sd '|' -
179
}
180

181
# Emit "size<TAB>path" for every file in a tree page (jq, or a grep/awk fallback).
182
emit_page_files() {
183
    if command -v jq &>/dev/null; then
184
        jq -r '.[] | select(.type=="file") | "\(.size)\t\(.path)"' "$1"
185
    else
186
        # Strip the nested lfs{} object (it has its own "size") so type/size/path align per entry.
187
        sed 's/"lfs":{[^}]*}//g' "$1" \
188
            | grep -oE '"type":"[^"]*"|"size":[0-9]+|"path":"[^"]*"' \
189
            | sed 's/"type":"//; s/"size"://; s/"path":"//; s/"$//' \
190
            | awk 'NR%3==1{t=$0} NR%3==2{s=$0} NR%3==0{if(t=="file")print s"\t"$0}'
191
    fi
192
}
193

194
# Emit "size<TAB>path" for every file from the cached metadata siblings[] (needs jq).
195
emit_siblings() {
196
    jq -r '(.siblings // [])[] | "\(.size // 0)\t\(.rfilename)"' "$METADATA_FILE"
197
}
198

199
# True when siblings[] is the complete list: its sizes sum to treesize (so it wasn't truncated, as
200
# it is for very large repos). Lets us skip the slow tree walk for typical repos via one API call.
201
siblings_complete() {
202
    command -v jq &>/dev/null && [[ "$REPO_SIZE" =~ ^[0-9]+$ ]] || return 1
203
    local n s
204
    read -r n s < <(jq -r '(.siblings // []) as $s | "\($s|length) \([$s[].size // 0]|add // 0)"' "$METADATA_FILE" 2>/dev/null)
205
    (( ${n:-0} > 0 )) && [[ "${s:-0}" == "$REPO_SIZE" ]]
206
}
207

208
# Keep only "size<TAB>path" stdin lines matching include/exclude (the *_REGEX globals).
209
filter_size_path() {
210
    local size path
211
    while IFS=$'\t' read -r size path; do
212
        [[ -z "$path" ]] && continue
213
        [[ -n "$INCLUDE_REGEX" && ! "$path" =~ $INCLUDE_REGEX ]] && continue
214
        [[ -n "$EXCLUDE_REGEX" && "$path" =~ $EXCLUDE_REGEX ]] && continue
215
        printf '%s\t%s\n' "${size:-0}" "$path"
216
    done
217
}
218

219
# Resumably walk the recursive tree into .hfd/manifest.partial (filtered size<TAB>path). After each
220
# page, checkpoint .hfd/list_state (fingerprint / entries scanned / next cursor) so an interrupted or
221
# failed walk continues from there instead of restarting. Progress (entries scanned) goes to stderr.
222
walk_tree() {
223
    local state="$LOCAL_DIR/.hfd/list_state" partial="$LOCAL_DIR/.hfd/manifest.partial"
224
    local page="$LOCAL_DIR/.hfd/tree_page.json" fp url scanned=0 headers n saved_fp=""
225
    fp=$(generate_command_string)
226
    [[ -f "$state" && -f "$partial" ]] && { read -r saved_fp; read -r scanned; read -r url; } < "$state"
227
    if [[ "$saved_fp" != "$fp" ]]; then   # no checkpoint, or it belongs to a different command
228
        url="$HF_ENDPOINT/api/$REPO_API_PATH/tree/$REVISION?recursive=true&expand=false"
229
        scanned=0; : > "$partial"
230
    elif [[ -s "$partial" && -n "$(tail -c1 "$partial")" ]]; then
231
        sed -i '$d' "$partial"   # drop a torn final line from a kill mid-append (the page re-fetches)
232
    fi
233
    while [[ -n "$url" ]]; do
234
        headers=$(curl -sSL ${HF_TOKEN:+-H "Authorization: Bearer $HF_TOKEN"} -D - -o "$page" "$url") || return 1
235
        if command -v jq &>/dev/null; then n=$(jq 'length' "$page" 2>/dev/null); else n=$(grep -o '"type":"' "$page" | wc -l); fi
236
        emit_page_files "$page" | filter_size_path >> "$partial"
237
        scanned=$(( scanned + ${n:-0} ))
238
        printf '\r\033[K%bListing files... %d scanned%b' "$DIM" "$scanned" "$NC" >&2
239
        url=$(printf '%s' "$headers" | tr -d '\r' | grep -i '^link:' \
240
            | grep -o '<[^>]*>;[[:space:]]*rel="next"' | sed -n 's/^<\([^>]*\)>.*/\1/p')
241
        printf '%s\n%s\n%s\n' "$fp" "$scanned" "$url" > "$state"   # checkpoint after the page is in
242
    done
243
    return 0
244
}
245

246
# Dedup .hfd/manifest.partial by path (a resumed walk may re-list its boundary page) into the final
247
# manifest, and write "<count> <bytes>" totals to filelist_stats.
248
finalize_filelist() {
249
    local mf="$LOCAL_DIR/.hfd/manifest"
250
    : > "$mf"
251
    awk -F'\t' -v mf="$mf" '!seen[$2]++ { print >> mf; n++; s+=$1 } END { print (n+0)" "(s+0) }' \
252
        "$LOCAL_DIR/.hfd/manifest.partial" > "$LOCAL_DIR/.hfd/filelist_stats"
253
}
254

255
# Build tool input from the manifest; set NEED_COUNT (files still to fetch). One find pass diffs
256
# the tree, keeping files missing or the wrong size (.aria2 sidecar = in-progress, kept): cheap
257
# for huge repos, correct after a manual delete. Run from the local dir (manifest paths relative).
258
build_download_list() {
259
    local needed=.hfd/needed size path dir cur
260
    if [[ ! -s .hfd/manifest ]]; then NEED_COUNT=0; : > "$fileslist_file"; return; fi
261
    awk -F'\t' '
262
        FNR==NR { want[$2]=$1; ord[++n]=$2; next }
263
        { p=$2; sub(/^\.\//,"",p);
264
          if (p ~ /\.aria2$/) { sub(/\.aria2$/,"",p); part[p]=1 } else have[p]=$1 }
265
        END { for (i=1;i<=n;i++) { q=ord[i];
266
                if ((q in have) && have[q]==want[q] && !(q in part)) continue;
267
                print want[q] "\t" q } }
268
    ' .hfd/manifest <(find . -type f ! -path './.hfd/*' -printf '%s\t%p\n' 2>/dev/null) > "$needed"
269
    NEED_COUNT=$(wc -l < "$needed")
270
    # Per needed file: drop a wrong-size copy -c/--continue can't fix in place (aria2c with no
271
    # .aria2 control file, or a wget file larger than expected) for a fresh refetch; emit the record.
272
    while IFS=$'\t' read -r size path; do
273
        [[ -z "$path" ]] && continue
274
        if [[ "$TOOL" == "aria2c" ]]; then
275
            [[ -e "$path" && ! -e "$path.aria2" ]] && rm -f "$path"
276
            dir="${path%/*}"; [[ "$dir" == "$path" ]] && dir=""
277
            printf '%s/%s/resolve/%s/%s\n dir=%s\n out=%s\n' "$HF_ENDPOINT" "$DOWNLOAD_API_PATH" "$REVISION" "$path" "$dir" "${path##*/}"
278
            [[ -n "$HF_TOKEN" ]] && printf ' header=Authorization: Bearer %s\n' "$HF_TOKEN"
279
            printf '\n'
280
        else
281
            cur=$(stat -c%s "$path" 2>/dev/null || echo 0); (( cur > size )) && rm -f "$path"
282
            printf '%s/%s/resolve/%s/%s\n' "$HF_ENDPOINT" "$DOWNLOAD_API_PATH" "$REVISION" "$path"
283
        fi
284
    done < "$needed" > "$fileslist_file"
285
    rm -f "$needed"
286
}
287

288
if should_regenerate_filelist; then
289
    command -v jq &>/dev/null || printf "%b[Warning] jq not installed, using grep/awk for json parsing (slower). Consider installing jq.%b\n" "$YELLOW" "$NC"
290

291
    # Total size up front (before the long walk); label the call so its few-second wait isn't silent.
292
    printf "%bFetching repository info...%b" "$DIM" "$NC"
293
    REPO_SIZE=$(fetch_treesize); printf '\r\033[K'
294
    [[ "$REPO_SIZE" =~ ^[0-9]+$ ]] && printf "%bRepository size: %s%b\n" "$DIM" "$(human "$REPO_SIZE")" "$NC"
295
    INCLUDE_REGEX=$(patterns_to_regex "${INCLUDE_PATTERNS[@]}")
296
    EXCLUDE_REGEX=$(patterns_to_regex "${EXCLUDE_PATTERNS[@]}")
297
    printf "%bListing files...%b" "$DIM" "$NC"
298
    # Fast path: typical repos return their whole file list in the metadata; only big repos
299
    # (truncated siblings) need the paginated, resumable tree walk.
300
    if siblings_complete; then
301
        rm -f "$LOCAL_DIR/.hfd/list_state"
302
        emit_siblings | filter_size_path > "$LOCAL_DIR/.hfd/manifest.partial"
303
        gen_status=${PIPESTATUS[0]}
304
    else
305
        walk_tree; gen_status=$?
306
    fi
307
    # A failed walk keeps its checkpoint (list_state + manifest.partial), so a re-run resumes.
308
    (( gen_status == 0 )) || { printf "\n%b[Error] Failed to list repository files. Re-run to resume.%b\n" "$RED" "$NC" >&2; exit 1; }
309
    finalize_filelist
310
    read -r TOTAL_FILES SUM_SIZE < "$LOCAL_DIR/.hfd/filelist_stats"
311
    printf '\r\033[K%bListed %d files (%s)%b\n' "$DIM" "$TOTAL_FILES" "$(human "$SUM_SIZE")" "$NC"
312
    # Whole-revision total from treesize; the summed filtered size when include/exclude narrows it.
313
    TOTAL_SIZE=$REPO_SIZE
314
    [[ -n "$INCLUDE_REGEX$EXCLUDE_REGEX" || ! "$REPO_SIZE" =~ ^[0-9]+$ ]] && TOTAL_SIZE=$SUM_SIZE
315
    printf '%s %s\n' "$TOTAL_FILES" "$TOTAL_SIZE" > "$LOCAL_DIR/.hfd/repo_info"
316
    # Fingerprint written last: marks the list complete (an interrupted walk has none, so it re-lists).
317
    generate_command_string > "$LOCAL_DIR/.hfd/last_download_command"
318
    rm -f "$LOCAL_DIR/.hfd/manifest.partial" "$LOCAL_DIR/.hfd/list_state" "$LOCAL_DIR/.hfd/tree_page.json"
319
fi
320

321
cd "$LOCAL_DIR" || exit 1
322

323
# Render one in-place status line; speed = byte delta since last call (PREV_B/PREV_T).
324
# Fields are fixed-width so columns don't jitter as values grow a digit; ETA is last.
325
PREV_B=0; PREV_T=0
326
render_progress() {
327
    local now=$1 dfiles=$2 dt sp pct eta e
328
    dt=$(( SECONDS - PREV_T )); ((dt<1)) && dt=1
329
    sp=$(( (now - PREV_B) / dt )); ((sp<0)) && sp=0; PREV_B=$now; PREV_T=$SECONDS
330
    pct=0; ((TOTAL_SIZE>0)) && pct=$(( now * 100 / TOTAL_SIZE )); ((pct>100)) && pct=100
331
    # ETA only once speed is meaningful; clamp absurd early estimates to keep it tidy.
332
    eta="--:--"; ((sp>0 && TOTAL_SIZE>now)) && { e=$(( (TOTAL_SIZE-now)/sp )); ((e<360000)) && eta=$(printf '%02d:%02d' $((e/60)) $((e%60))); }
333
    printf '\r\033[K%b[%3d%%]%b %*d/%d files | %9s/%9s | %9s/s | ETA %s' \
334
        "$GREEN" "$pct" "$NC" "${#TOTAL_FILES}" "$dfiles" "$TOTAL_FILES" \
335
        "$(human "$now")" "$(human "$TOTAL_SIZE")" "$(human "$sp")" "$eta"
336
}
337

338
# Progress without scanning finished files: manifest totals + a stat of only in-flight files.
339
monitor_progress() {
340
    local interval=$1 base=$1 stop_hint="Stopping download, please wait..."; PREV_T=$SECONDS
341
    # The main INT trap is deferred until the foreground download returns, so acknowledge
342
    # Ctrl+C here (fires at once in the subshell) and note the force-stop that already works.
343
    [[ "$TOOL" == "aria2c" ]] && stop_hint="Stopping download, please wait... (press Ctrl+C again to force stop)"
344
    trap 'printf "\n%b%s%b\n" "$YELLOW" "$stop_hint" "$NC"; exit 0' INT
345
    if [[ "$TOOL" == "wget" ]]; then
346
        # wget downloads in manifest order, so a forward cursor needs to stat only the
347
        # current file; everything before it is done and counted by its known size.
348
        local -a MS MP; local s p
349
        while IFS=$'\t' read -r s p; do MS+=("$s"); MP+=("$p"); done < .hfd/manifest
350
        local idx=0 done_b=0 cur
351
        while :; do
352
            sleep "$interval"
353
            while (( idx < ${#MP[@]} )); do
354
                cur=$(stat -c%s "${MP[idx]}" 2>/dev/null) || break
355
                (( cur >= MS[idx] )) || break
356
                done_b=$(( done_b + MS[idx] )); idx=$((idx+1))
357
            done
358
            cur=0; (( idx < ${#MP[@]} )) && cur=$(stat -c%s "${MP[idx]}" 2>/dev/null || echo 0)
359
            render_progress $(( done_b + cur )) "$idx"
360
        done
361
    else
362
        # Parallel downloads finish out of order, so a cursor would miss files completing between
363
        # ticks. Each tick sums block usage (sparse-aware) of the manifest's files only — ignoring
364
        # stale files left from an earlier version of the repo — minus in-progress (.aria2) ones.
365
        local now files t0 walk
366
        while :; do
367
            sleep "$interval"
368
            t0=$SECONDS
369
            read -r now files < <(awk -F'\t' '
370
                FNR==NR { want[$2]=1; next }
371
                { p=$2; sub(/^\.\//,"",p)
372
                  if (p ~ /\.aria2$/) { sub(/\.aria2$/,"",p); if (p in want) a++; next }
373
                  if (p in want) { b+=$1; d++ } }
374
                END { print b*512, d-a }
375
            ' .hfd/manifest <(find . -type f ! -path './.hfd/*' -printf '%b\t%p\n' 2>/dev/null))
376
            render_progress "$now" "$files"
377
            # Self-throttle: if the walk took longer than the base interval, sleep that long
378
            # next time so scanning stays under ~half the wall-clock at any repo size.
379
            walk=$(( SECONDS - t0 )); interval=$base; (( walk > base )) && interval=$walk
380
        done
381
    fi
382
}
383

384
# Totals were persisted at file-list generation; reload for resumed runs.
385
[[ -f .hfd/repo_info ]] && read -r TOTAL_FILES TOTAL_SIZE < .hfd/repo_info
386
TOTAL_FILES=${TOTAL_FILES:-0}; TOTAL_SIZE=${TOTAL_SIZE:-0}
387
FILE_NOUN="files"; ((TOTAL_FILES==1)) && FILE_NOUN="file"
388

389
# Diff the local tree against the manifest to find what's missing, then short-circuit if done.
390
((TOTAL_FILES>50000)) && printf "%bChecking local files...%b\n" "$DIM" "$NC"
391
build_download_list
392
if (( NEED_COUNT == 0 )); then
393
    printf "%bUp to date. %s %s in %s%b\n" "$GREEN" "$TOTAL_FILES" "$FILE_NOUN" "$LOCAL_DIR" "$NC"
394
    exit 0
395
fi
396

397
# "Resuming" if any data is already on disk (completed files or a partial), else a fresh start.
398
verb="Downloading"; [[ -n "$(find . -type f ! -path './.hfd/*' -print -quit 2>/dev/null)" ]] && verb="Resuming"
399
printf "%s %s %s to %s  ·  Ctrl+C to stop, re-run to resume\n" "$verb" "$TOTAL_FILES" "$FILE_NOUN" "$LOCAL_DIR"
400

401
# Silence native per-file output (logged to .hfd/download.log) so the monitor owns one
402
# clean line. Refresh every 1s, backing off for huge repos where the walk gets expensive.
403
interval=1; ((TOTAL_FILES>50000)) && interval=5
404
monitor_progress "$interval" &
405
MON_PID=$!
406
trap 'kill "$MON_PID" 2>/dev/null' EXIT
407

408
if [[ "$TOOL" == "aria2c" ]]; then
409
    aria2c --quiet=true --log=.hfd/download.log --log-level=error --file-allocation=none \
410
        -x "$THREADS" -j "$CONCURRENT" -s "$THREADS" -k 1M -c -i "$fileslist_file" >/dev/null
411
    status=$?
412
else
413
    wget -x -nH --cut-dirs="$CUT_DIRS" ${HF_TOKEN:+--header="Authorization: Bearer $HF_TOKEN"} \
414
        --input-file="$fileslist_file" --continue -nv -o .hfd/download.log
415
    status=$?
416
fi
417

418
# Clear the live progress line in place; the final status line takes its spot (no blank line).
419
kill "$MON_PID" 2>/dev/null; wait "$MON_PID" 2>/dev/null; printf '\r\033[K'
420

421
if [[ $status -eq 0 ]]; then
422
    printf "%bDone. %s %s, %s in %s%b\n" "$GREEN" "$TOTAL_FILES" "$FILE_NOUN" "$(human "$TOTAL_SIZE")" "$LOCAL_DIR" "$NC"
423
else
424
    printf "%bDownload incomplete. Re-run to resume. Log: %s%b\n" "$RED" "$PWD/.hfd/download.log" "$NC"
425
    exit 1
426
fi