openwrt-build-helper/gen-package-list.sh

#!/bin/bash
# Generate the built-in package list for a device from the official index.
#
# The list is default_packages + device_packages from the target's
# profiles.json, plus the packages the Firmware Selector adds on top
# (luci, luci-app-attendedsysupgrade). This is exactly what the Selector
# shows, so it replaces copying that list by hand for every new release.
#
# By default the list is written into config_<ver>/packages_<id>_<ver>.txt,
# replacing only the block between the generated-list sentinels inside the
# ##built-in section. Everything else in the file -- your own extra packages,
# the ##module section, blank lines -- is preserved verbatim.
#
# Usage: ./gen-package-list.sh [options]
#
#       --device <id>     profile id, e.g. linksys_mx8500
#       --target <t>      disambiguate an id present in several targets
#       --version <ver>   override version detection (e.g. 25.12.5)
#       --snapshot        shorthand for --version SNAPSHOT
#       --extras "<pkgs>" override the Firmware Selector extras
#       --no-extras       emit only default_packages + device_packages
#       --wrap <cols>     wrap the generated list (default 0 = one line)
#       --file <path>     write to this file instead of the default path
#       --stdout          print the list, one package per line, and exit
#       --apply           apply the list to .config instead of writing a file
#       --no-migrate      do not replace an existing unmarked built-in line
#       --refresh         ignore the cached JSON indexes
#   -y, --yes             do not ask for confirmation
#   -n, --dry-run         show what would change, write nothing
#   -h, --help            show this help
#
# Author: Zhe Yuan

set -euo pipefail
umask 022

SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
# shellcheck source=lib-openwrt-upstream.sh
source "$SCRIPT_DIR/lib-openwrt-upstream.sh"

DEVICE_ID=""
DEVICE_TARGET=""
VERSION_OVERRIDE=""
EXTRAS="$SELECTOR_EXTRAS"
WRAP=0
OUT_FILE=""
MODE="file"          # file | stdout | apply
MIGRATE=true

BEGIN_MARK="# >>> generated built-in list - do not edit by hand"
END_MARK="# <<< generated built-in list"

usage() { sed -n '2,35p' "$0" | sed 's/^# \{0,1\}//;s/^#$//'; }

need_value() {
  if [ "$2" -lt 2 ] || [ -z "${3:-}" ]; then
    err "$1 requires a value."
    exit 1
  fi
}

while [ $# -gt 0 ]; do
  case "$1" in
    --device)  need_value "$1" $# "${2:-}"; DEVICE_ID="$2"; shift ;;
    --target)  need_value "$1" $# "${2:-}"; DEVICE_TARGET="$2"; shift ;;
    --version) need_value "$1" $# "${2:-}"; VERSION_OVERRIDE="$2"; shift ;;
    --snapshot) VERSION_OVERRIDE="SNAPSHOT" ;;
    --extras)  need_value "$1" $# "${2:-}"; EXTRAS="$2"; shift ;;
    --no-extras) EXTRAS="" ;;
    --wrap)    need_value "$1" $# "${2:-}"; WRAP="$2"; shift ;;
    --file)    need_value "$1" $# "${2:-}"; OUT_FILE="$2"; shift ;;
    --stdout)  MODE="stdout" ;;
    --apply)   MODE="apply" ;;
    --no-migrate) MIGRATE=false ;;
    --refresh) REFRESH=true ;;
    -y|--yes)  ASSUME_YES=true ;;
    -n|--dry-run) DRY_RUN=true ;;
    -h|--help) usage; exit 0 ;;
    *) err "Unknown option: $1"; echo "Run '$0 --help' for usage." >&2; exit 1 ;;
  esac
  shift
done

[ -t 0 ] || ASSUME_YES=true

if ! [[ "$WRAP" =~ ^[0-9]+$ ]]; then
  err "--wrap needs a non-negative number."
  exit 1
fi

require_python3

VER="$(detect_version)"
select_device "$VER"

PROFILES_JSON="$(fetch_profiles "$VER" "$DEV_TARGET")"

PKGS="$(profile_packages "$PROFILES_JSON" "$DEV_ID")"

# Extras go last and never duplicate something the target already provides,
# which is what keeps the output identical to the hand-written lists.
for extra in $EXTRAS; do
  if ! printf '%s\n' "$PKGS" | grep -qxF "$extra"; then
    PKGS="$PKGS"$'\n'"$extra"
  fi
done

COUNT="$(printf '%s\n' "$PKGS" | grep -c . || true)"

if [ "$MODE" = "stdout" ]; then
  printf '%s\n' "$PKGS"
  exit 0
fi

# --- Render the package block -------------------------------------------
render_block() {
  echo "$BEGIN_MARK"
  echo "# device: $DEV_ID  target: $DEV_TARGET  version: $VER"
  echo "# source: profiles.json default_packages + device_packages + selector extras"
  if [ "$WRAP" -gt 0 ]; then
    printf '%s\n' "$PKGS" | tr '\n' ' ' | fold -s -w "$WRAP" | sed 's/[[:space:]]*$//'
  else
    printf '%s\n' "$PKGS" | tr '\n' ' ' | sed 's/[[:space:]]*$//'
    echo
  fi
  echo "$END_MARK"
}

if [ "$MODE" = "apply" ]; then
  info "Applying $COUNT built-in packages to .config via add-openwrt-packages.sh"
  APPLY_ARGS=()
  [ "$ASSUME_YES" = true ] && APPLY_ARGS+=(-y)
  [ "$DRY_RUN" = true ] && APPLY_ARGS+=(-n)
  { echo "##built-in"; printf '%s\n' "$PKGS"; } |
    "$SCRIPT_DIR/add-openwrt-packages.sh" "${APPLY_ARGS[@]}" -
  exit $?
fi

# --- File mode -----------------------------------------------------------
if [ -z "$OUT_FILE" ]; then
  OUT_FILE="$SCRIPT_DIR/config_$VER/packages_${DEV_ID//-/_}_$VER.txt"
fi
OUT_DIR="$(dirname -- "$OUT_FILE")"

echo
echo "========================================"
echo " Planned actions"
echo "========================================"
echo "   Device   : $DEV_ID  ($DEV_LABEL)"
echo "   Target   : $DEV_TARGET"
echo "   Version  : $VER"
echo "   Packages : $COUNT built-in ($(printf '%s\n' "$PKGS" | grep -c . || true) total)"
echo "   Extras   : ${EXTRAS:-(none)}"
echo "   File     : $OUT_FILE $([ -f "$OUT_FILE" ] && echo '(update)' || echo '(create)')"
echo "========================================"

NEW_FILE="$(mktemp "${TMPDIR:-/tmp}/pkglist.XXXXXX")"
trap 'rm -f "$NEW_FILE"' EXIT

if [ -f "$OUT_FILE" ]; then
  # Replace only the generated block; preserve everything the user wrote.
  render_block > "$NEW_FILE.block"
  awk -v blockfile="$NEW_FILE.block" -v migrate="$MIGRATE" '
    function emit_block(   line) {
      while ((getline line < blockfile) > 0) print line
      close(blockfile)
      emitted = 1
    }
    BEGIN { inb = 0; skip = 0; emitted = 0; replaced = 0 }
    /^[ \t]*##/ {
      if (inb && !emitted) {
        # Section ended without a marked block: insert before leaving.
        emit_block()
      }
      inb = ($0 ~ /^[ \t]*##built-in/)
      skip = 0
      print
      next
    }
    inb && index($0, "# >>> generated built-in list") == 1 { skip = 1; emit_block(); next }
    inb && index($0, "# <<< generated built-in list") == 1 { skip = 0; next }
    skip { next }
    {
      # Migration: the pre-sentinel auto-generated line is the one carrying
      # both base-files and libc, which are in default_packages for every
      # target and never appear in a hand-written extras line.
      if (inb && !emitted && migrate == "true" && !replaced) {
        has_bf = 0; has_libc = 0
        for (i = 1; i <= NF; i++) {
          if ($i == "base-files") has_bf = 1
          if ($i == "libc") has_libc = 1
        }
        if (has_bf && has_libc) { emit_block(); replaced = 1; next }
      }
      print
    }
    END {
      if (inb && !emitted) emit_block()
      if (!emitted) {
        print "##built-in"
        print ""
        emit_block()
      }
    }
  ' "$OUT_FILE" > "$NEW_FILE"
  rm -f "$NEW_FILE.block"
else
  {
    echo "##built-in"
    echo
    render_block
    echo
    echo "##module"
  } > "$NEW_FILE"
fi

if [ -f "$OUT_FILE" ] && cmp -s "$OUT_FILE" "$NEW_FILE"; then
  info "No change: $OUT_FILE is already up to date."
  exit 0
fi

if [ "$DRY_RUN" = true ]; then
  echo
  info "Changes that would be written:"
  diff -u "$OUT_FILE" "$NEW_FILE" 2>/dev/null || true
  echo
  info "Dry run complete; nothing was written."
  exit 0
fi

echo
if ! confirm "Write $OUT_FILE?" yes; then
  info "Aborted by user."
  exit 0
fi

mkdir -p "$OUT_DIR"
if [ -f "$OUT_FILE" ]; then
  cp -a "$OUT_FILE" "$OUT_FILE.bak"
  info "Previous version kept at $(basename "$OUT_FILE").bak"
fi
cp "$NEW_FILE" "$OUT_FILE"
chmod 644 "$OUT_FILE"

echo
echo "========================================"
echo "OK  Package list written."
echo "   File:     $OUT_FILE"
echo "   Packages: $COUNT built-in"
echo "========================================"
echo
echo "Next steps:"
echo "   Edit $(basename "$OUT_FILE") to add your own packages below the generated block"
echo "   cd $(dirname "$SCRIPT_DIR")"
echo "   ./helper/add-openwrt-packages.sh helper/${OUT_FILE#"$SCRIPT_DIR"/}"
echo