openwrt-build-helper/gen-package-list.sh

#!/bin/bash
# Build a device's complete package list.
#
# The built-in part comes from the target's profiles.json (default_packages +
# device_packages, plus the packages the Firmware Selector adds on top), so it
# never has to be copied by hand for a new release. Your own packages live in
# packages/<device-id>.txt and are version independent.
#
#   packages/<device-id>.txt    your packages, kept in git, edited by hand
#   .generated/                 the combined list, not kept in git
#
# Usage: ./gen-package-list.sh [options]
#
#       --device <id>       profile id, e.g. linksys_mx8500
#       --target <t>        disambiguate an id present in several targets
#       --version <ver>     override version detection (e.g. 25.12.5)
#       --snapshot          shorthand for --version SNAPSHOT
#       --extras-file <f>   use this instead of packages/<device-id>.txt
#       --no-extras-file    built-in packages only
#       --init-extras       create packages/<device-id>.txt if it is missing
#       --extras "<pkgs>"   override the Firmware Selector additions
#       --no-extras         omit the Firmware Selector additions
#       --wrap <cols>       wrap the built-in list (default 0 = one line)
#       --out <path>        write here instead of .generated/
#       --stdout            print the built-in list only, one per line
#       --full-stdout       print the complete list and exit
#       --apply             apply the complete list to .config
#       --refresh           ignore the cached JSON indexes
#   -y, --yes               do not ask for confirmation
#   -n, --dry-run           show what would happen, write nothing
#   -h, --help              show this help
#
# Author: Zhe Yuan

set -euo pipefail
umask 022

SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
LIB_SCRIPT_DIR="$SCRIPT_DIR"
# shellcheck source=lib-openwrt-upstream.sh
source "$SCRIPT_DIR/lib-openwrt-upstream.sh"

DEVICE_ID=""
DEVICE_TARGET=""
VERSION_OVERRIDE=""
EXTRAS="$SELECTOR_EXTRAS"
EXTRAS_FILE=""
USE_EXTRAS_FILE=true
INIT_EXTRAS=false
WRAP=0
OUT_FILE=""
MODE="file"          # file | stdout | full-stdout | apply

usage() { sed -n '2,34p' "$0" | sed 's/^# \{0,1\}//;s/^#$//'; }

need_value() {
  if [ "$2" -lt 2 ] || [ -z "${3:-}" ]; then
    err "$1 requires a value."
    exit 1
  fi
}

while [ $# -gt 0 ]; do
  case "$1" in
    --device)  need_value "$1" $# "${2:-}"; DEVICE_ID="$2"; shift ;;
    --target)  need_value "$1" $# "${2:-}"; DEVICE_TARGET="$2"; shift ;;
    --version) need_value "$1" $# "${2:-}"; VERSION_OVERRIDE="$2"; shift ;;
    --snapshot) VERSION_OVERRIDE="SNAPSHOT" ;;
    --extras-file) need_value "$1" $# "${2:-}"; EXTRAS_FILE="$2"; shift ;;
    --no-extras-file) USE_EXTRAS_FILE=false ;;
    --init-extras) INIT_EXTRAS=true ;;
    --extras)  need_value "$1" $# "${2:-}"; EXTRAS="$2"; shift ;;
    --no-extras) EXTRAS="" ;;
    --wrap)    need_value "$1" $# "${2:-}"; WRAP="$2"; shift ;;
    --out)     need_value "$1" $# "${2:-}"; OUT_FILE="$2"; shift ;;
    --stdout)  MODE="stdout" ;;
    --full-stdout) MODE="full-stdout" ;;
    --apply)   MODE="apply" ;;
    --refresh) REFRESH=true ;;
    -y|--yes)  ASSUME_YES=true ;;
    -n|--dry-run) DRY_RUN=true ;;
    -h|--help) usage; exit 0 ;;
    *) err "Unknown option: $1"; echo "Run '$0 --help' for usage." >&2; exit 1 ;;
  esac
  shift
done

[ -t 0 ] || ASSUME_YES=true

if ! [[ "$WRAP" =~ ^[0-9]+$ ]]; then
  err "--wrap needs a non-negative number."
  exit 1
fi

require_python3

VER="$(detect_version)"
select_device "$VER"

PROFILES_JSON="$(fetch_profiles "$VER" "$DEV_TARGET")"
PKGS="$(profile_packages "$PROFILES_JSON" "$DEV_ID")"

# Selector additions go last and never duplicate what the target provides.
for extra in $EXTRAS; do
  if ! printf '%s\n' "$PKGS" | grep -qxF "$extra"; then
    PKGS="$PKGS"$'\n'"$extra"
  fi
done

BUILTIN_COUNT="$(printf '%s\n' "$PKGS" | grep -c . || true)"

if [ "$MODE" = "stdout" ]; then
  printf '%s\n' "$PKGS"
  exit 0
fi

# --- Extras file ----------------------------------------------------------
if [ "$USE_EXTRAS_FILE" = true ] && [ -z "$EXTRAS_FILE" ]; then
  EXTRAS_FILE="$SCRIPT_DIR/packages/$DEV_ID.txt"
fi

init_extras_file() {
  mkdir -p "$(dirname "$EXTRAS_FILE")"
  cat > "$EXTRAS_FILE" <<EOF
# Extra packages for $DEV_ID, on top of the built-in list that
# gen-package-list.sh pulls from the official profiles.json.
#
# Whitespace separated, any number per line. '#' starts a comment.
# '-pkg' removes a package; '##module' / '##remove' switch modes.

EOF
  info "Created $EXTRAS_FILE"
}

EXTRAS_COUNT=0
if [ "$USE_EXTRAS_FILE" = true ]; then
  if [ ! -f "$EXTRAS_FILE" ]; then
    if [ "$INIT_EXTRAS" = true ] && [ "$DRY_RUN" != true ]; then
      init_extras_file
    else
      warn "No extras file at $EXTRAS_FILE"
      warn "Only the built-in packages will be included (--init-extras creates one)."
      EXTRAS_FILE=""
    fi
  fi
  if [ -n "$EXTRAS_FILE" ] && [ -f "$EXTRAS_FILE" ]; then
    EXTRAS_COUNT="$(grep -v '^[[:space:]]*#' "$EXTRAS_FILE" | tr -s ' \t\n' '\n' |
      grep -c . || true)"
  fi
fi

# --- Render the complete list ---------------------------------------------
# The generated file is entirely machine-owned, so it is simply rewritten;
# everything hand-written lives in the extras file instead.
render_full() {
  echo "##built-in"
  echo "# Generated from profiles.json for $DEV_ID ($DEV_TARGET) at $VER."
  echo "# Do not edit: regenerate with gen-package-list.sh."
  if [ "$WRAP" -gt 0 ]; then
    printf '%s\n' "$PKGS" | tr '\n' ' ' | fold -s -w "$WRAP" | sed 's/[[:space:]]*$//'
  else
    printf '%s\n' "$PKGS" | tr '\n' ' ' | sed 's/[[:space:]]*$//'
    echo
  fi

  if [ -n "$EXTRAS_FILE" ] && [ -f "$EXTRAS_FILE" ]; then
    echo
    echo "# --- extras from ${EXTRAS_FILE#"$SCRIPT_DIR"/} ---"
    # Appended verbatim: section headers are positional, so a ##module in
    # the extras file keeps working after the built-in block.
    cat "$EXTRAS_FILE"
  fi
}

if [ "$MODE" = "full-stdout" ]; then
  render_full
  exit 0
fi

if [ "$MODE" = "apply" ]; then
  info "Applying $BUILTIN_COUNT built-in + $EXTRAS_COUNT extra packages to .config"
  APPLY_ARGS=()
  [ "$ASSUME_YES" = true ] && APPLY_ARGS+=(-y)
  [ "$DRY_RUN" = true ] && APPLY_ARGS+=(-n)
  render_full | "$SCRIPT_DIR/add-openwrt-packages.sh" "${APPLY_ARGS[@]}" -
  exit $?
fi

# --- File mode ------------------------------------------------------------
if [ -z "$OUT_FILE" ]; then
  OUT_FILE="$SCRIPT_DIR/.generated/packages_${DEV_ID}_${VER}.txt"
fi

echo
echo "========================================"
echo " Planned actions"
echo "========================================"
echo "   Device   : $DEV_ID  ($DEV_LABEL)"
echo "   Target   : $DEV_TARGET"
echo "   Version  : $VER"
echo "   Built-in : $BUILTIN_COUNT packages (from profiles.json)"
echo "   Extras   : $EXTRAS_COUNT packages${EXTRAS_FILE:+ from ${EXTRAS_FILE#"$SCRIPT_DIR"/}}"
echo "   Output   : $OUT_FILE"
echo "========================================"

if [ "$DRY_RUN" = true ]; then
  echo
  info "Complete list that would be written:"
  render_full | sed 's/^/   /'
  echo
  info "Dry run complete; nothing was written."
  exit 0
fi

echo
if ! confirm "Write $OUT_FILE?" yes; then
  info "Aborted by user."
  exit 0
fi

mkdir -p "$(dirname -- "$OUT_FILE")"
render_full > "$OUT_FILE"
chmod 644 "$OUT_FILE"

echo
echo "========================================"
echo "OK  Package list written."
echo "   File:     $OUT_FILE"
echo "   Packages: $BUILTIN_COUNT built-in + $EXTRAS_COUNT extra"
echo "========================================"
echo
echo "Next steps:"
[ -n "$EXTRAS_FILE" ] && echo "   Edit ${EXTRAS_FILE#"$SCRIPT_DIR"/} to change your own packages"
echo "   cd $(dirname "$SCRIPT_DIR")"
echo "   ./helper/add-openwrt-packages.sh helper/${OUT_FILE#"$SCRIPT_DIR"/}"
echo