Generate the built-in package list from the official index
The ##built-in section of each package list was copied by hand from the Firmware Selector for every new release. It is now generated from the target's profiles.json (default_packages + device_packages, plus the two packages the Selector adds on top), which reproduces the hand-written list byte for byte, order included. New lib-openwrt-upstream.sh holds the shared upstream access: URL construction, cached fetches, version detection, device search over .overview.json and profiles.json parsing. It needs python3, not jq. New gen-package-list.sh writes config_<ver>/packages_<dev>_<ver>.txt, replacing only the generated block so hand-written extras and the ##module section survive; --stdout and --apply skip the file entirely. download-config.sh: - Look the device up in the official index instead of a hardcoded table of three devices, so any supported device can be selected by name. - Build the firmware URLs from profiles.json images[]. The old code pasted the target into the filename with its '/' intact, so the URL was always wrong and the check always warned. - Stage the download in a temp file and validate it before touching .config, and keep a backup. A 404 used to truncate .config to nothing. - Fix version detection: `git describe --tags --abbrev=0` returns a tag whenever any tag is reachable, so the entire fallback chain below it was unreachable and a branch tip silently produced the wrong release URL. - Build only the selected device instead of all 35 in the target, and turn off the buildbot flags (ALL_KMODS, ALL_NONSHARED, SDK, IB, MAKE_TOOLCHAIN, COLLECT_KERNEL_DEBUG, AUTOREMOVE). Verified not to change the firmware: the set of packages built into the image is identical either way (193 packages), while the module packages to build drop from 1248 to 0. --keep-buildbot-flags and --all-profiles restore the old behaviour. - Assert the device symbol survived make defconfig, so a tree that does not match the release fails loudly instead of silently building the default. add-openwrt-packages.sh: - Rewrite existing CONFIG_PACKAGE_ lines in place. Repeated runs used to append duplicates and grow .config every time. - Support removal via a -pkg prefix or a ##remove section. - Fix the parser dropping a lone '#' or an unknown ##header into the package list, where it was reported as a missing package named '#'. - Read .config once into a map instead of grepping it per package, and compare package names exactly rather than as regexes. - After make defconfig, report any package whose final state differs from what was asked for, which is the only way to see a removal that a dependency pulled back in. Delete add-openwrt-packages_old.sh; nothing referenced it and it was a strict subset of the current script. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
@@ -1,96 +1,276 @@
|
||||
#!/bin/bash
|
||||
# Download OpenWRT config.buildinfo, generate .config and run defconfig
|
||||
# Fetch the official config.buildinfo for a device and install it as .config.
|
||||
#
|
||||
# The device is looked up in the official index, so any of the ~1900 supported
|
||||
# devices can be selected by name; nothing is hardcoded.
|
||||
#
|
||||
# Two things are changed in the downloaded config by default, because it is
|
||||
# the buildbot's configuration rather than a personal one:
|
||||
# - only the selected device is built, instead of every model in the target
|
||||
# - the buildbot flags (ALL_KMODS, ALL_NONSHARED, SDK, IB, MAKE_TOOLCHAIN,
|
||||
# COLLECT_KERNEL_DEBUG, AUTOREMOVE, ...) are turned off
|
||||
# Neither changes the contents of the firmware image; they only stop the build
|
||||
# from also producing every optional module, the SDK and the ImageBuilder.
|
||||
#
|
||||
# Usage: ./download-config.sh [options]
|
||||
#
|
||||
# --device <id> profile id, e.g. linksys_mx8500
|
||||
# --target <t> disambiguate an id present in several targets
|
||||
# --version <ver> override version detection (e.g. 25.12.5)
|
||||
# --snapshot shorthand for --version SNAPSHOT
|
||||
# --with-packages also refresh the device's package list file
|
||||
# --all-profiles keep every device in the target (upstream default)
|
||||
# --keep-buildbot-flags keep ALL_KMODS / SDK / IB / ... as upstream ships
|
||||
# --save-buildinfo keep a copy in config_<ver>/<id>_<ver>.buildinfo
|
||||
# --offline use a previously saved buildinfo, no network
|
||||
# --output <file> write here instead of ./.config (for testing)
|
||||
# --refresh ignore the cached JSON indexes
|
||||
# --skip-defconfig do not run make defconfig afterwards
|
||||
# -y, --yes do not ask for confirmation
|
||||
# -n, --dry-run show the plan, change nothing
|
||||
# -h, --help show this help
|
||||
#
|
||||
# Run it from the OpenWrt root: ./helper/download-config.sh
|
||||
#
|
||||
# Author: Zhe Yuan
|
||||
|
||||
set -e
|
||||
set -euo pipefail
|
||||
umask 022
|
||||
|
||||
# --- Check if inside OpenWRT source directory ---
|
||||
if [ ! -f "feeds.conf.default" ] || [ ! -d "package" ]; then
|
||||
echo "[ERROR] Must run inside OpenWRT root directory."
|
||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
|
||||
# shellcheck source=lib-openwrt-upstream.sh
|
||||
source "$SCRIPT_DIR/lib-openwrt-upstream.sh"
|
||||
|
||||
DEVICE_ID=""
|
||||
DEVICE_TARGET=""
|
||||
VERSION_OVERRIDE=""
|
||||
WITH_PACKAGES=false
|
||||
SINGLE_PROFILE=true
|
||||
STRIP_BUILDBOT=true
|
||||
SAVE_BUILDINFO=false
|
||||
OFFLINE=false
|
||||
OUTPUT=""
|
||||
RUN_DEFCONFIG=true
|
||||
|
||||
usage() { sed -n '2,37p' "$0" | sed 's/^# \{0,1\}//;s/^#$//'; }
|
||||
|
||||
need_value() {
|
||||
if [ "$2" -lt 2 ] || [ -z "${3:-}" ]; then
|
||||
err "$1 requires a value."
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
# --- Detect Git version ---
|
||||
VERSION=$(git describe --tags --abbrev=0 2>/dev/null || echo "")
|
||||
VERSION="${VERSION#v}"
|
||||
IS_LATEST=false
|
||||
|
||||
if [ -z "$VERSION" ]; then
|
||||
BRANCH=$(git rev-parse --abbrev-ref HEAD)
|
||||
if [[ "$BRANCH" =~ openwrt-([0-9]+\.[0-9]+) ]]; then
|
||||
PARTIAL_VERSION="${BASH_REMATCH[1]}"
|
||||
# Find latest patch tag
|
||||
LATEST_TAG=$(git tag -l "v${PARTIAL_VERSION}.*" | sort -V | tail -n1)
|
||||
if [ -z "$LATEST_TAG" ]; then
|
||||
echo "[ERROR] Cannot find latest tag for $PARTIAL_VERSION.*"
|
||||
exit 1
|
||||
fi
|
||||
VERSION="${LATEST_TAG#v}"
|
||||
IS_LATEST=true
|
||||
elif [[ "$BRANCH" == *SNAPSHOT* ]]; then
|
||||
VERSION="SNAPSHOT"
|
||||
else
|
||||
read -rp "Cannot detect version, input manually (e.g., 24.10.4 or SNAPSHOT): " VERSION
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "[INFO] OpenWRT version: $VERSION"
|
||||
|
||||
# --- Devices list ---
|
||||
# Format: Name Target FirmwareDevice
|
||||
DEVICES=(
|
||||
"Linksys_MX8500 qualcommax/ipq807x linksys_mx8500"
|
||||
"Cudy_TR3000_128MB mediatek/filogic cudy_tr3000-v1"
|
||||
"Cudy_TR3000_256MB mediatek/filogic cudy_tr3000-256mb-v1"
|
||||
)
|
||||
|
||||
echo "Available devices:"
|
||||
for i in "${!DEVICES[@]}"; do
|
||||
NAME=$(echo "${DEVICES[$i]}" | awk '{print $1}')
|
||||
echo " $i) $NAME"
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--device) need_value "$1" $# "${2:-}"; DEVICE_ID="$2"; shift ;;
|
||||
--target) need_value "$1" $# "${2:-}"; DEVICE_TARGET="$2"; shift ;;
|
||||
--version) need_value "$1" $# "${2:-}"; VERSION_OVERRIDE="$2"; shift ;;
|
||||
--snapshot) VERSION_OVERRIDE="SNAPSHOT" ;;
|
||||
--with-packages) WITH_PACKAGES=true ;;
|
||||
--all-profiles) SINGLE_PROFILE=false ;;
|
||||
--keep-buildbot-flags) STRIP_BUILDBOT=false ;;
|
||||
--save-buildinfo) SAVE_BUILDINFO=true ;;
|
||||
--offline) OFFLINE=true ;;
|
||||
--output) need_value "$1" $# "${2:-}"; OUTPUT="$2"; shift ;;
|
||||
--refresh) REFRESH=true ;;
|
||||
--skip-defconfig) RUN_DEFCONFIG=false ;;
|
||||
-y|--yes) ASSUME_YES=true ;;
|
||||
-n|--dry-run) DRY_RUN=true ;;
|
||||
-h|--help) usage; exit 0 ;;
|
||||
*) err "Unknown option: $1"; echo "Run '$0 --help' for usage." >&2; exit 1 ;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
read -rp "Select device number: " DEV_INDEX
|
||||
if [[ -z "${DEVICES[$DEV_INDEX]}" ]]; then
|
||||
echo "[ERROR] Invalid selection."
|
||||
[ -t 0 ] || ASSUME_YES=true
|
||||
|
||||
require_python3
|
||||
|
||||
if [ -n "$OUTPUT" ]; then
|
||||
DEST="$OUTPUT"
|
||||
RUN_DEFCONFIG=false
|
||||
else
|
||||
require_openwrt_root
|
||||
DEST=".config"
|
||||
fi
|
||||
DEST_DIR="$(dirname -- "$DEST")"
|
||||
|
||||
VER="$(detect_version)"
|
||||
|
||||
if [ "$OFFLINE" != true ] && ! version_exists "$VER"; then
|
||||
err "Version '$VER' is not published at $BASE_URL."
|
||||
err " $(url_overview "$VER")"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
select_device "$VER"
|
||||
|
||||
SAVED_BUILDINFO="$SCRIPT_DIR/config_$VER/${DEV_ID}_${VER}.buildinfo"
|
||||
|
||||
SYSUPGRADE_URL=""
|
||||
FACTORY_URL=""
|
||||
PKG_COUNT="?"
|
||||
if [ "$OFFLINE" != true ]; then
|
||||
PROFILES_JSON="$(fetch_profiles "$VER" "$DEV_TARGET")"
|
||||
PKG_COUNT="$(profile_packages "$PROFILES_JSON" "$DEV_ID" | grep -c . || true)"
|
||||
while IFS= read -r img; do
|
||||
case "$img" in
|
||||
*squashfs-sysupgrade*) SYSUPGRADE_URL="$(url_image "$VER" "$DEV_TARGET" "$img")" ;;
|
||||
*squashfs-factory*) FACTORY_URL="$(url_image "$VER" "$DEV_TARGET" "$img")" ;;
|
||||
esac
|
||||
done < <(profile_images "$PROFILES_JSON" "$DEV_ID")
|
||||
fi
|
||||
|
||||
CONFIG_URL="$(url_buildinfo "$VER" "$DEV_TARGET")"
|
||||
|
||||
echo
|
||||
echo "========================================"
|
||||
echo " Planned actions"
|
||||
echo "========================================"
|
||||
echo " Device : $DEV_ID ($DEV_LABEL)"
|
||||
echo " Target : $DEV_TARGET"
|
||||
echo " Version : $VER"
|
||||
if [ "$OFFLINE" = true ]; then
|
||||
echo " Source : $SAVED_BUILDINFO (offline)"
|
||||
else
|
||||
echo " Source : $CONFIG_URL"
|
||||
fi
|
||||
echo " Destination : $DEST$([ -f "$DEST" ] && echo ' (backed up first)')"
|
||||
echo " Device scope : $([ "$SINGLE_PROFILE" = true ] && echo 'this device only' || echo 'all devices in target')"
|
||||
echo " Buildbot flags: $([ "$STRIP_BUILDBOT" = true ] && echo 'stripped' || echo 'kept')"
|
||||
echo " Built-in pkgs : $PKG_COUNT (per profiles.json)"
|
||||
[ -n "$SYSUPGRADE_URL" ] && echo " Sysupgrade : $SYSUPGRADE_URL"
|
||||
echo " Post : $([ "$RUN_DEFCONFIG" = true ] && echo 'make defconfig' || echo 'skipped')"
|
||||
echo "========================================"
|
||||
|
||||
if [ "$DRY_RUN" != true ]; then
|
||||
echo
|
||||
if ! confirm "Proceed?" yes; then
|
||||
info "Aborted by user."
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
|
||||
# --- Download into a temp file next to the destination -------------------
|
||||
# Same directory keeps the final mv atomic; `curl -f` plus this staging is
|
||||
# what stops a 404 from truncating an existing .config.
|
||||
TMP="$(mktemp "$DEST_DIR/.config.dl.XXXXXX")"
|
||||
trap 'rm -f "$TMP" "$TMP.pp"' EXIT
|
||||
|
||||
if [ "$OFFLINE" = true ]; then
|
||||
if [ ! -f "$SAVED_BUILDINFO" ]; then
|
||||
err "No saved buildinfo at $SAVED_BUILDINFO"
|
||||
err "Run once without --offline (add --save-buildinfo) to create it."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# --- Extract device info ---
|
||||
read -r DEV_NAME DEV_TARGET DEV_FW <<<"${DEVICES[$DEV_INDEX]}"
|
||||
echo "[INFO] Selected device: $DEV_NAME"
|
||||
|
||||
# --- Construct URLs ---
|
||||
if [[ "$VERSION" == "SNAPSHOT" ]]; then
|
||||
CONFIG_URL="https://downloads.openwrt.org/snapshots/targets/$DEV_TARGET/config.buildinfo"
|
||||
FIRMWARE_URL="https://downloads.openwrt.org/snapshots/targets/$DEV_TARGET/openwrt-SNAPSHOT-$DEV_TARGET-$DEV_FW-squashfs-sysupgrade.bin"
|
||||
fi
|
||||
info "Using the saved buildinfo (offline)."
|
||||
cp "$SAVED_BUILDINFO" "$TMP"
|
||||
else
|
||||
CONFIG_URL="https://downloads.openwrt.org/releases/$VERSION/targets/$DEV_TARGET/config.buildinfo"
|
||||
FIRMWARE_URL="https://downloads.openwrt.org/releases/$VERSION/targets/$DEV_TARGET/openwrt-$VERSION-$DEV_TARGET-$DEV_FW-squashfs-sysupgrade.bin"
|
||||
info "Downloading $CONFIG_URL"
|
||||
if ! fetch_url "$CONFIG_URL" "$TMP"; then
|
||||
err "Download failed; $DEST was not touched."
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "[INFO] Config URL: $CONFIG_URL"
|
||||
echo "[INFO] Firmware URL: $FIRMWARE_URL"
|
||||
# --- Validate before letting it near .config ------------------------------
|
||||
if [ ! -s "$TMP" ] || [ "$(wc -l < "$TMP")" -lt 5 ]; then
|
||||
err "Downloaded config looks empty or truncated; $DEST was not touched."
|
||||
exit 1
|
||||
fi
|
||||
if ! grep -qx "CONFIG_TARGET_${DEV_TSYM}=y" "$TMP"; then
|
||||
err "Downloaded config is not for target $DEV_TARGET; $DEST was not touched."
|
||||
exit 1
|
||||
fi
|
||||
if ! grep -q "_DEVICE_${DEV_ID}=y" "$TMP"; then
|
||||
err "Downloaded config does not contain device $DEV_ID; $DEST was not touched."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# --- Download config.buildinfo ---
|
||||
echo "[INFO] Downloading config.buildinfo..."
|
||||
wget -q --show-progress -O ".config" "$CONFIG_URL"
|
||||
echo "[INFO] Saved as .config"
|
||||
# --- Post-process ---------------------------------------------------------
|
||||
# One awk pass: a chain of `sed -i` would rewrite the file repeatedly and the
|
||||
# symbol names contain '-' and '+', which are awkward in sed replacements.
|
||||
awk -v tsym="$DEV_TSYM" -v prof="$DEV_ID" \
|
||||
-v single="$SINGLE_PROFILE" -v strip="$STRIP_BUILDBOT" '
|
||||
# Matches both CONFIG_TARGET_DEVICE_<t>_DEVICE_<id>=y and
|
||||
# CONFIG_TARGET_DEVICE_PACKAGES_<t>_DEVICE_<id>="". It cannot match the
|
||||
# single-profile symbol appended below, which has no DEVICE_ prefix.
|
||||
single == "true" && /^CONFIG_TARGET_DEVICE_/ { next }
|
||||
single == "true" && /^CONFIG_TARGET_MULTI_PROFILE=y$/ { print "# CONFIG_TARGET_MULTI_PROFILE is not set"; next }
|
||||
single == "true" && /^CONFIG_TARGET_ALL_PROFILES=y$/ { print "# CONFIG_TARGET_ALL_PROFILES is not set"; next }
|
||||
single == "true" && /^CONFIG_TARGET_PER_DEVICE_ROOTFS=y$/ { print "# CONFIG_TARGET_PER_DEVICE_ROOTFS is not set"; next }
|
||||
strip == "true" && $0 ~ /^CONFIG_(ALL_KMODS|ALL_NONSHARED|BUILDBOT|SDK|SDK_LLVM_BPF|IB|MAKE_TOOLCHAIN|COLLECT_KERNEL_DEBUG|JSON_CYCLONEDX_SBOM|AUTOREMOVE)=y$/ {
|
||||
sym = $0; sub(/=y$/, "", sym); print "# " sym " is not set"; next
|
||||
}
|
||||
{ print }
|
||||
END {
|
||||
# CONFIG_TARGET_PROFILE is deliberately not written: Kconfig derives it.
|
||||
if (single == "true") print "CONFIG_TARGET_" tsym "_DEVICE_" prof "=y"
|
||||
}
|
||||
' "$TMP" > "$TMP.pp"
|
||||
|
||||
# --- Run make defconfig ---
|
||||
echo "[INFO] Running make defconfig..."
|
||||
make defconfig >/dev/null
|
||||
echo "[INFO] defconfig done."
|
||||
if [ "$DRY_RUN" = true ]; then
|
||||
echo
|
||||
info "Resulting config (first differences against the download):"
|
||||
diff -u "$TMP" "$TMP.pp" | head -40 || true
|
||||
echo
|
||||
info "Dry run complete; $DEST was not touched."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# --- Check firmware existence ---
|
||||
if curl --head --silent --fail "$FIRMWARE_URL" >/dev/null; then
|
||||
echo "[INFO] Firmware exists: $FIRMWARE_URL"
|
||||
if [ "$SAVE_BUILDINFO" = true ]; then
|
||||
mkdir -p "$(dirname "$SAVED_BUILDINFO")"
|
||||
cp "$TMP" "$SAVED_BUILDINFO"
|
||||
info "Saved the raw buildinfo to $SAVED_BUILDINFO"
|
||||
fi
|
||||
|
||||
if [ -f "$DEST" ]; then
|
||||
cp -a "$DEST" "$DEST.download.bak"
|
||||
info "Previous config kept at $DEST.download.bak"
|
||||
fi
|
||||
mv -f "$TMP.pp" "$DEST"
|
||||
chmod 644 "$DEST"
|
||||
info "Wrote $DEST"
|
||||
|
||||
if [ "$RUN_DEFCONFIG" = true ]; then
|
||||
info "Running make defconfig..."
|
||||
make defconfig >/dev/null
|
||||
|
||||
# A config whose symbols do not exist in this tree is silently dropped by
|
||||
# Kconfig, leaving a config that looks fine and builds the wrong thing.
|
||||
if [ "$SINGLE_PROFILE" = true ]; then
|
||||
if ! grep -qx "CONFIG_TARGET_${DEV_TSYM}_DEVICE_${DEV_ID}=y" .config ||
|
||||
! grep -qx "CONFIG_TARGET_PROFILE=\"DEVICE_${DEV_ID}\"" .config; then
|
||||
err "The device symbol did not survive make defconfig."
|
||||
err "Your tree ($(git describe --tags 2>/dev/null || echo unknown)) probably"
|
||||
err "does not match release $VER. Restore with: cp $DEST.download.bak $DEST"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
echo
|
||||
echo "========================================"
|
||||
echo "OK Configuration installed."
|
||||
echo " Device: $DEV_ID ($DEV_LABEL)"
|
||||
echo " Target: $DEV_TARGET"
|
||||
echo " Version: $VER"
|
||||
echo "========================================"
|
||||
[ -n "$SYSUPGRADE_URL" ] && echo " Official sysupgrade: $SYSUPGRADE_URL"
|
||||
[ -n "$FACTORY_URL" ] && echo " Official factory: $FACTORY_URL"
|
||||
echo " Firmware Selector: $(selector_url "$VER" "$DEV_TARGET" "$DEV_ID")"
|
||||
echo
|
||||
|
||||
if [ "$WITH_PACKAGES" = true ]; then
|
||||
GEN_ARGS=(--device "$DEV_ID" --target "$DEV_TARGET" --version "$VER")
|
||||
[ "$ASSUME_YES" = true ] && GEN_ARGS+=(-y)
|
||||
"$SCRIPT_DIR/gen-package-list.sh" "${GEN_ARGS[@]}"
|
||||
else
|
||||
echo "[WARN] Firmware not found or URL invalid."
|
||||
echo "Next steps:"
|
||||
echo " ./helper/gen-package-list.sh --device $DEV_ID --version $VER"
|
||||
echo " ./helper/add-openwrt-packages.sh helper/config_$VER/packages_${DEV_ID//-/_}_$VER.txt"
|
||||
echo " make menuconfig"
|
||||
echo " make -j\$(nproc) download world"
|
||||
echo
|
||||
fi
|
||||
|
||||
# --- Print Firmware Selector URL ---
|
||||
TARGET_ESC=$(echo $DEV_TARGET | sed 's|/|%2F|g')
|
||||
FS_URL="https://firmware-selector.openwrt.org/?version=$VERSION&target=$TARGET_ESC&id=$DEV_FW"
|
||||
echo "[INFO] Check Installed Packages at: $FS_URL"
|
||||
|
||||
Reference in New Issue
Block a user