Keep one extras list per device instead of per version
The package lists were stored as config_<version>/packages_<device>_<version>.txt, but the version dimension was pure duplication: the Linksys MX8500 extras were byte-identical across 24.10.4, 25.12.0, 25.12.2 and 25.12.4, and the Cudy lists only ever changed with the device, never with the release. Your packages now live in packages/<device-id>.txt, one file per device and independent of the OpenWrt version. The built-in part is fetched per release anyway, so upgrading needs no file changes at all. The combined list is a build artefact and moves to .generated/, which is gitignored, along with the buildinfo saved by --save-buildinfo --offline. This removes work rather than adding it: because the generated file is now entirely machine-owned, the sentinel comments, the in-section replacement state machine and the "line containing base-files and libc is the old generated one" migration heuristic are all gone. Hand-written content is isolated in the extras file, generated content in .generated/. Also fix version detection when a script is run from the helper directory. The helper checkout is its own git repository inside the OpenWrt worktree, so git commands there described the helper repo -- a checkout of OpenWrt 25.12.5 was reported as SNAPSHOT because the helper repo is on main. Version queries are now anchored to the OpenWrt tree. The config_* archives are deleted; git history keeps them. That includes config_24.10.4/apply-dahdi-patches.sh, which only applied to 24.10.4. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
@@ -1,33 +1,34 @@
|
||||
#!/bin/bash
|
||||
# Generate the built-in package list for a device from the official index.
|
||||
# Build a device's complete package list.
|
||||
#
|
||||
# The list is default_packages + device_packages from the target's
|
||||
# profiles.json, plus the packages the Firmware Selector adds on top
|
||||
# (luci, luci-app-attendedsysupgrade). This is exactly what the Selector
|
||||
# shows, so it replaces copying that list by hand for every new release.
|
||||
# The built-in part comes from the target's profiles.json (default_packages +
|
||||
# device_packages, plus the packages the Firmware Selector adds on top), so it
|
||||
# never has to be copied by hand for a new release. Your own packages live in
|
||||
# packages/<device-id>.txt and are version independent.
|
||||
#
|
||||
# By default the list is written into config_<ver>/packages_<id>_<ver>.txt,
|
||||
# replacing only the block between the generated-list sentinels inside the
|
||||
# ##built-in section. Everything else in the file -- your own extra packages,
|
||||
# the ##module section, blank lines -- is preserved verbatim.
|
||||
# packages/<device-id>.txt your packages, kept in git, edited by hand
|
||||
# .generated/ the combined list, not kept in git
|
||||
#
|
||||
# Usage: ./gen-package-list.sh [options]
|
||||
#
|
||||
# --device <id> profile id, e.g. linksys_mx8500
|
||||
# --target <t> disambiguate an id present in several targets
|
||||
# --version <ver> override version detection (e.g. 25.12.5)
|
||||
# --snapshot shorthand for --version SNAPSHOT
|
||||
# --extras "<pkgs>" override the Firmware Selector extras
|
||||
# --no-extras emit only default_packages + device_packages
|
||||
# --wrap <cols> wrap the generated list (default 0 = one line)
|
||||
# --file <path> write to this file instead of the default path
|
||||
# --stdout print the list, one package per line, and exit
|
||||
# --apply apply the list to .config instead of writing a file
|
||||
# --no-migrate do not replace an existing unmarked built-in line
|
||||
# --refresh ignore the cached JSON indexes
|
||||
# -y, --yes do not ask for confirmation
|
||||
# -n, --dry-run show what would change, write nothing
|
||||
# -h, --help show this help
|
||||
# --device <id> profile id, e.g. linksys_mx8500
|
||||
# --target <t> disambiguate an id present in several targets
|
||||
# --version <ver> override version detection (e.g. 25.12.5)
|
||||
# --snapshot shorthand for --version SNAPSHOT
|
||||
# --extras-file <f> use this instead of packages/<device-id>.txt
|
||||
# --no-extras-file built-in packages only
|
||||
# --init-extras create packages/<device-id>.txt if it is missing
|
||||
# --extras "<pkgs>" override the Firmware Selector additions
|
||||
# --no-extras omit the Firmware Selector additions
|
||||
# --wrap <cols> wrap the built-in list (default 0 = one line)
|
||||
# --out <path> write here instead of .generated/
|
||||
# --stdout print the built-in list only, one per line
|
||||
# --full-stdout print the complete list and exit
|
||||
# --apply apply the complete list to .config
|
||||
# --refresh ignore the cached JSON indexes
|
||||
# -y, --yes do not ask for confirmation
|
||||
# -n, --dry-run show what would happen, write nothing
|
||||
# -h, --help show this help
|
||||
#
|
||||
# Author: Zhe Yuan
|
||||
|
||||
@@ -35,6 +36,7 @@ set -euo pipefail
|
||||
umask 022
|
||||
|
||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
|
||||
LIB_SCRIPT_DIR="$SCRIPT_DIR"
|
||||
# shellcheck source=lib-openwrt-upstream.sh
|
||||
source "$SCRIPT_DIR/lib-openwrt-upstream.sh"
|
||||
|
||||
@@ -42,15 +44,14 @@ DEVICE_ID=""
|
||||
DEVICE_TARGET=""
|
||||
VERSION_OVERRIDE=""
|
||||
EXTRAS="$SELECTOR_EXTRAS"
|
||||
EXTRAS_FILE=""
|
||||
USE_EXTRAS_FILE=true
|
||||
INIT_EXTRAS=false
|
||||
WRAP=0
|
||||
OUT_FILE=""
|
||||
MODE="file" # file | stdout | apply
|
||||
MIGRATE=true
|
||||
MODE="file" # file | stdout | full-stdout | apply
|
||||
|
||||
BEGIN_MARK="# >>> generated built-in list - do not edit by hand"
|
||||
END_MARK="# <<< generated built-in list"
|
||||
|
||||
usage() { sed -n '2,35p' "$0" | sed 's/^# \{0,1\}//;s/^#$//'; }
|
||||
usage() { sed -n '2,34p' "$0" | sed 's/^# \{0,1\}//;s/^#$//'; }
|
||||
|
||||
need_value() {
|
||||
if [ "$2" -lt 2 ] || [ -z "${3:-}" ]; then
|
||||
@@ -65,13 +66,16 @@ while [ $# -gt 0 ]; do
|
||||
--target) need_value "$1" $# "${2:-}"; DEVICE_TARGET="$2"; shift ;;
|
||||
--version) need_value "$1" $# "${2:-}"; VERSION_OVERRIDE="$2"; shift ;;
|
||||
--snapshot) VERSION_OVERRIDE="SNAPSHOT" ;;
|
||||
--extras-file) need_value "$1" $# "${2:-}"; EXTRAS_FILE="$2"; shift ;;
|
||||
--no-extras-file) USE_EXTRAS_FILE=false ;;
|
||||
--init-extras) INIT_EXTRAS=true ;;
|
||||
--extras) need_value "$1" $# "${2:-}"; EXTRAS="$2"; shift ;;
|
||||
--no-extras) EXTRAS="" ;;
|
||||
--wrap) need_value "$1" $# "${2:-}"; WRAP="$2"; shift ;;
|
||||
--file) need_value "$1" $# "${2:-}"; OUT_FILE="$2"; shift ;;
|
||||
--out) need_value "$1" $# "${2:-}"; OUT_FILE="$2"; shift ;;
|
||||
--stdout) MODE="stdout" ;;
|
||||
--full-stdout) MODE="full-stdout" ;;
|
||||
--apply) MODE="apply" ;;
|
||||
--no-migrate) MIGRATE=false ;;
|
||||
--refresh) REFRESH=true ;;
|
||||
-y|--yes) ASSUME_YES=true ;;
|
||||
-n|--dry-run) DRY_RUN=true ;;
|
||||
@@ -94,53 +98,98 @@ VER="$(detect_version)"
|
||||
select_device "$VER"
|
||||
|
||||
PROFILES_JSON="$(fetch_profiles "$VER" "$DEV_TARGET")"
|
||||
|
||||
PKGS="$(profile_packages "$PROFILES_JSON" "$DEV_ID")"
|
||||
|
||||
# Extras go last and never duplicate something the target already provides,
|
||||
# which is what keeps the output identical to the hand-written lists.
|
||||
# Selector additions go last and never duplicate what the target provides.
|
||||
for extra in $EXTRAS; do
|
||||
if ! printf '%s\n' "$PKGS" | grep -qxF "$extra"; then
|
||||
PKGS="$PKGS"$'\n'"$extra"
|
||||
fi
|
||||
done
|
||||
|
||||
COUNT="$(printf '%s\n' "$PKGS" | grep -c . || true)"
|
||||
BUILTIN_COUNT="$(printf '%s\n' "$PKGS" | grep -c . || true)"
|
||||
|
||||
if [ "$MODE" = "stdout" ]; then
|
||||
printf '%s\n' "$PKGS"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# --- Render the package block -------------------------------------------
|
||||
render_block() {
|
||||
echo "$BEGIN_MARK"
|
||||
echo "# device: $DEV_ID target: $DEV_TARGET version: $VER"
|
||||
echo "# source: profiles.json default_packages + device_packages + selector extras"
|
||||
# --- Extras file ----------------------------------------------------------
|
||||
if [ "$USE_EXTRAS_FILE" = true ] && [ -z "$EXTRAS_FILE" ]; then
|
||||
EXTRAS_FILE="$SCRIPT_DIR/packages/$DEV_ID.txt"
|
||||
fi
|
||||
|
||||
init_extras_file() {
|
||||
mkdir -p "$(dirname "$EXTRAS_FILE")"
|
||||
cat > "$EXTRAS_FILE" <<EOF
|
||||
# Extra packages for $DEV_ID, on top of the built-in list that
|
||||
# gen-package-list.sh pulls from the official profiles.json.
|
||||
#
|
||||
# Whitespace separated, any number per line. '#' starts a comment.
|
||||
# '-pkg' removes a package; '##module' / '##remove' switch modes.
|
||||
|
||||
EOF
|
||||
info "Created $EXTRAS_FILE"
|
||||
}
|
||||
|
||||
EXTRAS_COUNT=0
|
||||
if [ "$USE_EXTRAS_FILE" = true ]; then
|
||||
if [ ! -f "$EXTRAS_FILE" ]; then
|
||||
if [ "$INIT_EXTRAS" = true ] && [ "$DRY_RUN" != true ]; then
|
||||
init_extras_file
|
||||
else
|
||||
warn "No extras file at $EXTRAS_FILE"
|
||||
warn "Only the built-in packages will be included (--init-extras creates one)."
|
||||
EXTRAS_FILE=""
|
||||
fi
|
||||
fi
|
||||
if [ -n "$EXTRAS_FILE" ] && [ -f "$EXTRAS_FILE" ]; then
|
||||
EXTRAS_COUNT="$(grep -v '^[[:space:]]*#' "$EXTRAS_FILE" | tr -s ' \t\n' '\n' |
|
||||
grep -c . || true)"
|
||||
fi
|
||||
fi
|
||||
|
||||
# --- Render the complete list ---------------------------------------------
|
||||
# The generated file is entirely machine-owned, so it is simply rewritten;
|
||||
# everything hand-written lives in the extras file instead.
|
||||
render_full() {
|
||||
echo "##built-in"
|
||||
echo "# Generated from profiles.json for $DEV_ID ($DEV_TARGET) at $VER."
|
||||
echo "# Do not edit: regenerate with gen-package-list.sh."
|
||||
if [ "$WRAP" -gt 0 ]; then
|
||||
printf '%s\n' "$PKGS" | tr '\n' ' ' | fold -s -w "$WRAP" | sed 's/[[:space:]]*$//'
|
||||
else
|
||||
printf '%s\n' "$PKGS" | tr '\n' ' ' | sed 's/[[:space:]]*$//'
|
||||
echo
|
||||
fi
|
||||
echo "$END_MARK"
|
||||
|
||||
if [ -n "$EXTRAS_FILE" ] && [ -f "$EXTRAS_FILE" ]; then
|
||||
echo
|
||||
echo "# --- extras from ${EXTRAS_FILE#"$SCRIPT_DIR"/} ---"
|
||||
# Appended verbatim: section headers are positional, so a ##module in
|
||||
# the extras file keeps working after the built-in block.
|
||||
cat "$EXTRAS_FILE"
|
||||
fi
|
||||
}
|
||||
|
||||
if [ "$MODE" = "full-stdout" ]; then
|
||||
render_full
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "$MODE" = "apply" ]; then
|
||||
info "Applying $COUNT built-in packages to .config via add-openwrt-packages.sh"
|
||||
info "Applying $BUILTIN_COUNT built-in + $EXTRAS_COUNT extra packages to .config"
|
||||
APPLY_ARGS=()
|
||||
[ "$ASSUME_YES" = true ] && APPLY_ARGS+=(-y)
|
||||
[ "$DRY_RUN" = true ] && APPLY_ARGS+=(-n)
|
||||
{ echo "##built-in"; printf '%s\n' "$PKGS"; } |
|
||||
"$SCRIPT_DIR/add-openwrt-packages.sh" "${APPLY_ARGS[@]}" -
|
||||
render_full | "$SCRIPT_DIR/add-openwrt-packages.sh" "${APPLY_ARGS[@]}" -
|
||||
exit $?
|
||||
fi
|
||||
|
||||
# --- File mode -----------------------------------------------------------
|
||||
# --- File mode ------------------------------------------------------------
|
||||
if [ -z "$OUT_FILE" ]; then
|
||||
OUT_FILE="$SCRIPT_DIR/config_$VER/packages_${DEV_ID//-/_}_$VER.txt"
|
||||
OUT_FILE="$SCRIPT_DIR/.generated/packages_${DEV_ID}_${VER}.txt"
|
||||
fi
|
||||
OUT_DIR="$(dirname -- "$OUT_FILE")"
|
||||
|
||||
echo
|
||||
echo "========================================"
|
||||
@@ -149,80 +198,15 @@ echo "========================================"
|
||||
echo " Device : $DEV_ID ($DEV_LABEL)"
|
||||
echo " Target : $DEV_TARGET"
|
||||
echo " Version : $VER"
|
||||
echo " Packages : $COUNT built-in ($(printf '%s\n' "$PKGS" | grep -c . || true) total)"
|
||||
echo " Extras : ${EXTRAS:-(none)}"
|
||||
echo " File : $OUT_FILE $([ -f "$OUT_FILE" ] && echo '(update)' || echo '(create)')"
|
||||
echo " Built-in : $BUILTIN_COUNT packages (from profiles.json)"
|
||||
echo " Extras : $EXTRAS_COUNT packages${EXTRAS_FILE:+ from ${EXTRAS_FILE#"$SCRIPT_DIR"/}}"
|
||||
echo " Output : $OUT_FILE"
|
||||
echo "========================================"
|
||||
|
||||
NEW_FILE="$(mktemp "${TMPDIR:-/tmp}/pkglist.XXXXXX")"
|
||||
trap 'rm -f "$NEW_FILE"' EXIT
|
||||
|
||||
if [ -f "$OUT_FILE" ]; then
|
||||
# Replace only the generated block; preserve everything the user wrote.
|
||||
render_block > "$NEW_FILE.block"
|
||||
awk -v blockfile="$NEW_FILE.block" -v migrate="$MIGRATE" '
|
||||
function emit_block( line) {
|
||||
while ((getline line < blockfile) > 0) print line
|
||||
close(blockfile)
|
||||
emitted = 1
|
||||
}
|
||||
BEGIN { inb = 0; skip = 0; emitted = 0; replaced = 0 }
|
||||
/^[ \t]*##/ {
|
||||
if (inb && !emitted) {
|
||||
# Section ended without a marked block: insert before leaving.
|
||||
emit_block()
|
||||
}
|
||||
inb = ($0 ~ /^[ \t]*##built-in/)
|
||||
skip = 0
|
||||
print
|
||||
next
|
||||
}
|
||||
inb && index($0, "# >>> generated built-in list") == 1 { skip = 1; emit_block(); next }
|
||||
inb && index($0, "# <<< generated built-in list") == 1 { skip = 0; next }
|
||||
skip { next }
|
||||
{
|
||||
# Migration: the pre-sentinel auto-generated line is the one carrying
|
||||
# both base-files and libc, which are in default_packages for every
|
||||
# target and never appear in a hand-written extras line.
|
||||
if (inb && !emitted && migrate == "true" && !replaced) {
|
||||
has_bf = 0; has_libc = 0
|
||||
for (i = 1; i <= NF; i++) {
|
||||
if ($i == "base-files") has_bf = 1
|
||||
if ($i == "libc") has_libc = 1
|
||||
}
|
||||
if (has_bf && has_libc) { emit_block(); replaced = 1; next }
|
||||
}
|
||||
print
|
||||
}
|
||||
END {
|
||||
if (inb && !emitted) emit_block()
|
||||
if (!emitted) {
|
||||
print "##built-in"
|
||||
print ""
|
||||
emit_block()
|
||||
}
|
||||
}
|
||||
' "$OUT_FILE" > "$NEW_FILE"
|
||||
rm -f "$NEW_FILE.block"
|
||||
else
|
||||
{
|
||||
echo "##built-in"
|
||||
echo
|
||||
render_block
|
||||
echo
|
||||
echo "##module"
|
||||
} > "$NEW_FILE"
|
||||
fi
|
||||
|
||||
if [ -f "$OUT_FILE" ] && cmp -s "$OUT_FILE" "$NEW_FILE"; then
|
||||
info "No change: $OUT_FILE is already up to date."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "$DRY_RUN" = true ]; then
|
||||
echo
|
||||
info "Changes that would be written:"
|
||||
diff -u "$OUT_FILE" "$NEW_FILE" 2>/dev/null || true
|
||||
info "Complete list that would be written:"
|
||||
render_full | sed 's/^/ /'
|
||||
echo
|
||||
info "Dry run complete; nothing was written."
|
||||
exit 0
|
||||
@@ -234,23 +218,19 @@ if ! confirm "Write $OUT_FILE?" yes; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
mkdir -p "$OUT_DIR"
|
||||
if [ -f "$OUT_FILE" ]; then
|
||||
cp -a "$OUT_FILE" "$OUT_FILE.bak"
|
||||
info "Previous version kept at $(basename "$OUT_FILE").bak"
|
||||
fi
|
||||
cp "$NEW_FILE" "$OUT_FILE"
|
||||
mkdir -p "$(dirname -- "$OUT_FILE")"
|
||||
render_full > "$OUT_FILE"
|
||||
chmod 644 "$OUT_FILE"
|
||||
|
||||
echo
|
||||
echo "========================================"
|
||||
echo "OK Package list written."
|
||||
echo " File: $OUT_FILE"
|
||||
echo " Packages: $COUNT built-in"
|
||||
echo " Packages: $BUILTIN_COUNT built-in + $EXTRAS_COUNT extra"
|
||||
echo "========================================"
|
||||
echo
|
||||
echo "Next steps:"
|
||||
echo " Edit $(basename "$OUT_FILE") to add your own packages below the generated block"
|
||||
[ -n "$EXTRAS_FILE" ] && echo " Edit ${EXTRAS_FILE#"$SCRIPT_DIR"/} to change your own packages"
|
||||
echo " cd $(dirname "$SCRIPT_DIR")"
|
||||
echo " ./helper/add-openwrt-packages.sh helper/${OUT_FILE#"$SCRIPT_DIR"/}"
|
||||
echo
|
||||
|
||||
Reference in New Issue
Block a user