Generate the built-in package list from the official index
The ##built-in section of each package list was copied by hand from the Firmware Selector for every new release. It is now generated from the target's profiles.json (default_packages + device_packages, plus the two packages the Selector adds on top), which reproduces the hand-written list byte for byte, order included. New lib-openwrt-upstream.sh holds the shared upstream access: URL construction, cached fetches, version detection, device search over .overview.json and profiles.json parsing. It needs python3, not jq. New gen-package-list.sh writes config_<ver>/packages_<dev>_<ver>.txt, replacing only the generated block so hand-written extras and the ##module section survive; --stdout and --apply skip the file entirely. download-config.sh: - Look the device up in the official index instead of a hardcoded table of three devices, so any supported device can be selected by name. - Build the firmware URLs from profiles.json images[]. The old code pasted the target into the filename with its '/' intact, so the URL was always wrong and the check always warned. - Stage the download in a temp file and validate it before touching .config, and keep a backup. A 404 used to truncate .config to nothing. - Fix version detection: `git describe --tags --abbrev=0` returns a tag whenever any tag is reachable, so the entire fallback chain below it was unreachable and a branch tip silently produced the wrong release URL. - Build only the selected device instead of all 35 in the target, and turn off the buildbot flags (ALL_KMODS, ALL_NONSHARED, SDK, IB, MAKE_TOOLCHAIN, COLLECT_KERNEL_DEBUG, AUTOREMOVE). Verified not to change the firmware: the set of packages built into the image is identical either way (193 packages), while the module packages to build drop from 1248 to 0. --keep-buildbot-flags and --all-profiles restore the old behaviour. - Assert the device symbol survived make defconfig, so a tree that does not match the release fails loudly instead of silently building the default. add-openwrt-packages.sh: - Rewrite existing CONFIG_PACKAGE_ lines in place. Repeated runs used to append duplicates and grow .config every time. - Support removal via a -pkg prefix or a ##remove section. - Fix the parser dropping a lone '#' or an unknown ##header into the package list, where it was reported as a missing package named '#'. - Read .config once into a map instead of grepping it per package, and compare package names exactly rather than as regexes. - After make defconfig, report any package whose final state differs from what was asked for, which is the only way to see a removal that a dependency pulled back in. Delete add-openwrt-packages_old.sh; nothing referenced it and it was a strict subset of the current script. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
431
lib-openwrt-upstream.sh
Normal file
431
lib-openwrt-upstream.sh
Normal file
@@ -0,0 +1,431 @@
|
||||
#!/bin/bash
|
||||
# Shared helpers for talking to downloads.openwrt.org.
|
||||
#
|
||||
# Sourced by download-config.sh and gen-package-list.sh. Not executable on its
|
||||
# own. Callers are expected to set `set -euo pipefail`; this file does not.
|
||||
#
|
||||
# Author: Zhe Yuan
|
||||
|
||||
[ -n "${_OWRT_LIB_LOADED:-}" ] && return 0
|
||||
_OWRT_LIB_LOADED=1
|
||||
|
||||
BASE_URL="${BASE_URL:-https://downloads.openwrt.org}"
|
||||
CACHE_DIR="${CACHE_DIR:-${XDG_CACHE_HOME:-$HOME/.cache}/openwrt-build-helper}"
|
||||
CACHE_TTL="${CACHE_TTL:-86400}"
|
||||
|
||||
# What the Firmware Selector adds on top of the target's own package set.
|
||||
# This is a UI convention, not an API, so keep it overridable.
|
||||
SELECTOR_EXTRAS="${SELECTOR_EXTRAS:-luci luci-app-attendedsysupgrade}"
|
||||
|
||||
# Devices offered when an interactive search is left empty. Only a convenience
|
||||
# shortcut -- the target still comes from .overview.json, never hardcoded.
|
||||
FAVOURITE_DEVICES="${FAVOURITE_DEVICES:-linksys_mx8500 cudy_tr3000-v1 cudy_tr3000-256mb-v1}"
|
||||
|
||||
REFRESH="${REFRESH:-false}"
|
||||
DRY_RUN="${DRY_RUN:-false}"
|
||||
ASSUME_YES="${ASSUME_YES:-false}"
|
||||
|
||||
# --- Output helpers -------------------------------------------------------
|
||||
# All logging goes to stderr so that functions returning a value on stdout
|
||||
# (detect_version, profile_packages, ...) can be captured safely.
|
||||
info() { echo "[INFO] $*" >&2; }
|
||||
warn() { echo "[WARN] $*" >&2; }
|
||||
err() { echo "[ERROR] $*" >&2; }
|
||||
|
||||
# Wrap every mutating command so --dry-run prints instead of running.
|
||||
run() {
|
||||
if [ "$DRY_RUN" = true ]; then
|
||||
printf '[DRY-RUN]'
|
||||
printf ' %q' "$@"
|
||||
echo
|
||||
return 0
|
||||
fi
|
||||
"$@"
|
||||
}
|
||||
|
||||
confirm() {
|
||||
# confirm "<question>" <default: yes|no>
|
||||
local question="$1" default="$2" reply=""
|
||||
|
||||
[ "$ASSUME_YES" = true ] && return 0
|
||||
|
||||
if [ "$default" = "yes" ]; then
|
||||
read -r -p "$question [Y/n]: " reply || reply=""
|
||||
case "${reply:-}" in
|
||||
[nN] | [nN][oO]) return 1 ;;
|
||||
*) return 0 ;;
|
||||
esac
|
||||
else
|
||||
read -r -p "$question [y/N]: " reply || reply=""
|
||||
case "${reply:-}" in
|
||||
[yY] | [yY][eE][sS]) return 0 ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
fi
|
||||
}
|
||||
|
||||
require_python3() {
|
||||
if ! command -v python3 >/dev/null 2>&1; then
|
||||
err "python3 is required to parse the upstream JSON indexes."
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
require_openwrt_root() {
|
||||
if [ ! -f feeds.conf.default ] || [ ! -f Makefile ] || [ ! -d scripts ]; then
|
||||
err "Must run inside the OpenWrt root directory."
|
||||
err "From the OpenWrt root, run it as: ./helper/$(basename "$0")"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# --- URL construction -----------------------------------------------------
|
||||
# Every upstream URL is built from this one function.
|
||||
release_path() {
|
||||
local ver="$1"
|
||||
if [ "$ver" = "SNAPSHOT" ]; then
|
||||
echo "snapshots"
|
||||
else
|
||||
echo "releases/$ver"
|
||||
fi
|
||||
}
|
||||
|
||||
url_overview() { echo "$BASE_URL/$(release_path "$1")/.overview.json"; }
|
||||
url_profiles() { echo "$BASE_URL/$(release_path "$1")/targets/$2/profiles.json"; }
|
||||
url_buildinfo() { echo "$BASE_URL/$(release_path "$1")/targets/$2/config.buildinfo"; }
|
||||
url_image() { echo "$BASE_URL/$(release_path "$1")/targets/$2/$3"; }
|
||||
|
||||
selector_url() {
|
||||
local ver="$1" target="$2" id="$3"
|
||||
echo "https://firmware-selector.openwrt.org/?version=$ver&target=${target//\//%2F}&id=$id"
|
||||
}
|
||||
|
||||
# --- Fetching -------------------------------------------------------------
|
||||
# `curl -f` is the key flag: a 404 exits non-zero and never writes a body, so
|
||||
# a failed download can never truncate the destination.
|
||||
fetch_url() {
|
||||
local url="$1" dest="$2"
|
||||
|
||||
if ! curl -fsSL --retry 3 --retry-delay 2 --max-time 60 -o "$dest.part" "$url"; then
|
||||
rm -f "$dest.part"
|
||||
return 1
|
||||
fi
|
||||
mv -f "$dest.part" "$dest"
|
||||
}
|
||||
|
||||
fetch_cached() {
|
||||
local url="$1" cache="$2" age=0
|
||||
|
||||
mkdir -p "$(dirname "$cache")"
|
||||
|
||||
if [ "$REFRESH" != true ] && [ -s "$cache" ]; then
|
||||
age=$(( $(date +%s) - $(stat -c %Y "$cache" 2>/dev/null || echo 0) ))
|
||||
if [ "$age" -lt "$CACHE_TTL" ]; then
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
|
||||
if fetch_url "$url" "$cache"; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
# A stale cache beats no data when the network is down.
|
||||
if [ -s "$cache" ]; then
|
||||
warn "Could not refresh $url; using the cached copy."
|
||||
return 0
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
|
||||
version_exists() {
|
||||
curl -fsI --max-time 30 "$(url_overview "$1")" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
# --- Version detection ----------------------------------------------------
|
||||
# Replaces `git describe --tags --abbrev=0`, which returns a tag whenever ANY
|
||||
# tag is reachable -- so the old script's whole fallback chain was dead code
|
||||
# and a branch tip past a tag silently produced the wrong release URL.
|
||||
detect_version() {
|
||||
local ver="" branch="" tag="" drift=0
|
||||
|
||||
if [ -n "${VERSION_OVERRIDE:-}" ]; then
|
||||
ver="${VERSION_OVERRIDE#v}"
|
||||
info "Version from the command line: $ver"
|
||||
echo "$ver"
|
||||
return 0
|
||||
fi
|
||||
|
||||
if ! git rev-parse --is-inside-work-tree >/dev/null 2>&1; then
|
||||
err "Not inside a Git repository and no --version given."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if tag="$(git describe --tags --exact-match HEAD 2>/dev/null)"; then
|
||||
ver="${tag#v}"
|
||||
info "HEAD is exactly at tag $tag; using version $ver."
|
||||
echo "$ver"
|
||||
return 0
|
||||
fi
|
||||
|
||||
if branch="$(git symbolic-ref --quiet --short HEAD 2>/dev/null)"; then
|
||||
case "$branch" in
|
||||
main | master)
|
||||
info "On branch '$branch'; using SNAPSHOT."
|
||||
echo "SNAPSHOT"
|
||||
return 0
|
||||
;;
|
||||
openwrt-* | [0-9]*)
|
||||
local series="${branch#openwrt-}"
|
||||
# --merged HEAD keeps tags from unrelated branches out of the picture.
|
||||
tag="$(git tag --merged HEAD -l "v${series}.*" 2>/dev/null |
|
||||
grep -v -- '-rc' | sort -V | tail -n 1)"
|
||||
if [ -n "$tag" ]; then
|
||||
drift="$(git rev-list --count "$tag..HEAD" 2>/dev/null || echo 0)"
|
||||
if [ "$drift" -gt 0 ]; then
|
||||
warn "HEAD is $drift commit(s) past $tag."
|
||||
warn "The downloaded config describes the release, not your tree."
|
||||
fi
|
||||
ver="${tag#v}"
|
||||
info "Branch '$branch' -> newest merged tag $tag; using version $ver."
|
||||
echo "$ver"
|
||||
return 0
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
|
||||
if tag="$(git describe --tags --long 2>/dev/null)"; then
|
||||
tag="${tag%-*-g*}"
|
||||
drift="$(git rev-list --count "$tag..HEAD" 2>/dev/null || echo 0)"
|
||||
warn "HEAD is not at a tag; nearest is $tag ($drift commit(s) behind HEAD)."
|
||||
if [ "$ASSUME_YES" != true ] &&
|
||||
! confirm "Use version ${tag#v}?" no; then
|
||||
err "Aborted; pass --version explicitly."
|
||||
exit 1
|
||||
fi
|
||||
ver="${tag#v}"
|
||||
info "Using version $ver."
|
||||
echo "$ver"
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [ "$ASSUME_YES" = true ] || [ ! -t 0 ]; then
|
||||
err "Cannot determine the OpenWrt version; pass --version <ver> or --snapshot."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
read -r -p "Enter version (e.g. 25.12.5 or SNAPSHOT): " ver || ver=""
|
||||
if [ -z "$ver" ]; then
|
||||
err "No version given."
|
||||
exit 1
|
||||
fi
|
||||
echo "${ver#v}"
|
||||
}
|
||||
|
||||
# --- Device index ---------------------------------------------------------
|
||||
fetch_overview() {
|
||||
local ver="$1"
|
||||
# Separate statement on purpose: `local a=$1 b=$a` expands every word before
|
||||
# assigning, so $a would be unbound under `set -u`.
|
||||
local cache="$CACHE_DIR/overview-$ver.json"
|
||||
|
||||
# Snapshots move daily, so never serve them from a day-old cache.
|
||||
[ "$ver" = "SNAPSHOT" ] && REFRESH=true
|
||||
|
||||
if ! fetch_cached "$(url_overview "$ver")" "$cache"; then
|
||||
err "Cannot fetch the device index for version '$ver'."
|
||||
err " $(url_overview "$ver")"
|
||||
exit 1
|
||||
fi
|
||||
echo "$cache"
|
||||
}
|
||||
|
||||
# Emits TSV: id <tab> target <tab> label
|
||||
_overview_py() {
|
||||
python3 - "$@" <<'PY'
|
||||
import json, sys, difflib
|
||||
|
||||
path, mode, arg = sys.argv[1], sys.argv[2], sys.argv[3]
|
||||
data = json.load(open(path, encoding="utf-8"))
|
||||
profiles = data.get("profiles") or []
|
||||
|
||||
|
||||
def label(p):
|
||||
parts = []
|
||||
for t in p.get("titles") or []:
|
||||
if not isinstance(t, dict):
|
||||
continue
|
||||
# Upstream entries come in three shapes: {title}, {vendor,model},
|
||||
# and {vendor,model,variant}.
|
||||
if t.get("title"):
|
||||
parts.append(t["title"])
|
||||
else:
|
||||
parts.append(" ".join(
|
||||
str(x) for x in (t.get("vendor"), t.get("model"), t.get("variant")) if x
|
||||
))
|
||||
parts = [p for p in parts if p]
|
||||
return " / ".join(parts) or p.get("id", "")
|
||||
|
||||
|
||||
def emit(rows):
|
||||
for p in rows:
|
||||
print("%s\t%s\t%s" % (p.get("id", ""), p.get("target", ""), label(p)))
|
||||
|
||||
|
||||
if mode == "id":
|
||||
rows = [p for p in profiles if p.get("id") == arg]
|
||||
if not rows:
|
||||
names = [p.get("id", "") for p in profiles]
|
||||
close = difflib.get_close_matches(arg, names, n=5, cutoff=0.5)
|
||||
sys.stderr.write("no device with id '%s'\n" % arg)
|
||||
if close:
|
||||
sys.stderr.write("closest matches: %s\n" % ", ".join(close))
|
||||
sys.exit(2)
|
||||
emit(rows)
|
||||
elif mode == "search":
|
||||
terms = [t.lower() for t in arg.split() if t]
|
||||
rows = []
|
||||
for p in profiles:
|
||||
hay = (p.get("id", "") + " " + label(p)).lower()
|
||||
if all(t in hay for t in terms):
|
||||
rows.append(p)
|
||||
rows.sort(key=lambda p: p.get("id", ""))
|
||||
emit(rows)
|
||||
elif mode == "list":
|
||||
wanted = arg.split()
|
||||
by_id = {p.get("id"): p for p in profiles}
|
||||
emit([by_id[i] for i in wanted if i in by_id])
|
||||
PY
|
||||
}
|
||||
|
||||
# Sets DEV_ID, DEV_TARGET, DEV_TSYM, DEV_LABEL
|
||||
select_device() {
|
||||
local ver="$1" json="" rows="" n=0 choice="" tries=0 query=""
|
||||
|
||||
json="$(fetch_overview "$ver")"
|
||||
|
||||
if [ -n "${DEVICE_ID:-}" ]; then
|
||||
if ! rows="$(_overview_py "$json" id "$DEVICE_ID" 2>&1)"; then
|
||||
err "$rows"
|
||||
exit 1
|
||||
fi
|
||||
if [ -n "${DEVICE_TARGET:-}" ]; then
|
||||
rows="$(printf '%s\n' "$rows" | awk -F'\t' -v t="$DEVICE_TARGET" '$2 == t')"
|
||||
fi
|
||||
n="$(printf '%s\n' "$rows" | grep -c . || true)"
|
||||
if [ "$n" -eq 0 ]; then
|
||||
err "Device '$DEVICE_ID' exists but not for target '${DEVICE_TARGET:-}'."
|
||||
exit 1
|
||||
fi
|
||||
if [ "$n" -gt 1 ]; then
|
||||
err "Device id '$DEVICE_ID' is present in more than one target:"
|
||||
printf '%s\n' "$rows" | awk -F'\t' '{print " " $2 " (" $3 ")"}' >&2
|
||||
err "Disambiguate with --target <target>."
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
if [ "$ASSUME_YES" = true ] || [ ! -t 0 ]; then
|
||||
err "No device selected. Pass --device <id> for non-interactive use."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
while [ "$tries" -lt 3 ]; do
|
||||
tries=$((tries + 1))
|
||||
read -r -p "Search device (vendor, model or id; empty for favourites): " query || query=""
|
||||
|
||||
if [ -z "${query:-}" ]; then
|
||||
rows="$(_overview_py "$json" list "$FAVOURITE_DEVICES")"
|
||||
else
|
||||
rows="$(_overview_py "$json" search "$query")"
|
||||
fi
|
||||
|
||||
n="$(printf '%s\n' "$rows" | grep -c . || true)"
|
||||
if [ "$n" -eq 0 ]; then
|
||||
warn "No device matched '$query'."
|
||||
continue
|
||||
fi
|
||||
if [ "$n" -gt 25 ]; then
|
||||
warn "$n devices matched; please narrow the search."
|
||||
continue
|
||||
fi
|
||||
|
||||
echo
|
||||
printf '%s\n' "$rows" | awk -F'\t' '{printf " %2d) %-32s %-22s %s\n", NR, $1, $2, $3}'
|
||||
echo
|
||||
read -r -p "Select device number [1-$n]: " choice || choice=""
|
||||
|
||||
# 1-based plus an explicit numeric test: an unset/non-numeric index in
|
||||
# an array subscript is an arithmetic context that silently yields 0.
|
||||
if [[ "${choice:-}" =~ ^[0-9]+$ ]] && [ "$choice" -ge 1 ] && [ "$choice" -le "$n" ]; then
|
||||
rows="$(printf '%s\n' "$rows" | sed -n "${choice}p")"
|
||||
break
|
||||
fi
|
||||
warn "Invalid selection."
|
||||
rows=""
|
||||
done
|
||||
|
||||
if [ -z "$rows" ]; then
|
||||
err "No device selected."
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
IFS=$'\t' read -r DEV_ID DEV_TARGET DEV_LABEL <<<"$rows"
|
||||
DEV_TSYM="${DEV_TARGET//\//_}"
|
||||
|
||||
if [ -z "$DEV_ID" ] || [ -z "$DEV_TARGET" ]; then
|
||||
err "Could not resolve the device selection."
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# --- profiles.json --------------------------------------------------------
|
||||
fetch_profiles() {
|
||||
local ver="$1" target="$2"
|
||||
local cache="$CACHE_DIR/profiles-$ver-${target//\//_}.json"
|
||||
|
||||
[ "$ver" = "SNAPSHOT" ] && REFRESH=true
|
||||
|
||||
if ! fetch_cached "$(url_profiles "$ver" "$target")" "$cache"; then
|
||||
err "Cannot fetch profiles.json for $target at version $ver."
|
||||
err " $(url_profiles "$ver" "$target")"
|
||||
exit 1
|
||||
fi
|
||||
echo "$cache"
|
||||
}
|
||||
|
||||
# default_packages + device_packages, order preserved, deduplicated.
|
||||
profile_packages() {
|
||||
python3 - "$1" "$2" <<'PY'
|
||||
import json, sys
|
||||
|
||||
data = json.load(open(sys.argv[1], encoding="utf-8"))
|
||||
pid = sys.argv[2]
|
||||
prof = (data.get("profiles") or {}).get(pid)
|
||||
if prof is None:
|
||||
sys.stderr.write("profile '%s' is not in this target's profiles.json\n" % pid)
|
||||
sys.exit(2)
|
||||
|
||||
seen, out = set(), []
|
||||
for p in list(data.get("default_packages") or []) + list(prof.get("device_packages") or []):
|
||||
if p not in seen:
|
||||
seen.add(p)
|
||||
out.append(p)
|
||||
print("\n".join(out))
|
||||
PY
|
||||
}
|
||||
|
||||
# Image filenames straight from profiles.json -- no string building, which is
|
||||
# what made the old firmware URL wrong (the target still contained a '/').
|
||||
profile_images() {
|
||||
python3 - "$1" "$2" <<'PY'
|
||||
import json, sys
|
||||
|
||||
data = json.load(open(sys.argv[1], encoding="utf-8"))
|
||||
prof = (data.get("profiles") or {}).get(sys.argv[2]) or {}
|
||||
for img in prof.get("images") or []:
|
||||
name = img.get("name")
|
||||
if name:
|
||||
print(name)
|
||||
PY
|
||||
}
|
||||
Reference in New Issue
Block a user