openwrt-build-helper/download-config.sh

#!/bin/bash
# Fetch the official config.buildinfo for a device and install it as .config.
#
# The device is looked up in the official index, so any of the ~1900 supported
# devices can be selected by name; nothing is hardcoded.
#
# Two things are changed in the downloaded config by default, because it is
# the buildbot's configuration rather than a personal one:
#   - only the selected device is built, instead of every model in the target
#   - the buildbot flags (ALL_KMODS, ALL_NONSHARED, SDK, IB, MAKE_TOOLCHAIN,
#     COLLECT_KERNEL_DEBUG, AUTOREMOVE, ...) are turned off
# Neither changes the contents of the firmware image; they only stop the build
# from also producing every optional module, the SDK and the ImageBuilder.
#
# Usage: ./download-config.sh [options]
#
#       --device <id>          profile id, e.g. linksys_mx8500
#       --target <t>           disambiguate an id present in several targets
#       --version <ver>        override version detection (e.g. 25.12.5)
#       --snapshot             shorthand for --version SNAPSHOT
#       --with-packages        also refresh the device's package list file
#       --all-profiles         keep every device in the target (upstream default)
#       --keep-buildbot-flags  keep ALL_KMODS / SDK / IB / ... as upstream ships
#       --save-buildinfo       keep a copy in config_<ver>/<id>_<ver>.buildinfo
#       --offline              use a previously saved buildinfo, no network
#       --output <file>        write here instead of ./.config (for testing)
#       --refresh              ignore the cached JSON indexes
#       --skip-defconfig       do not run make defconfig afterwards
#   -y, --yes                  do not ask for confirmation
#   -n, --dry-run              show the plan, change nothing
#   -h, --help                 show this help
#
# Run it from the OpenWrt root:  ./helper/download-config.sh
#
# Author: Zhe Yuan

set -euo pipefail
umask 022

SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
# shellcheck source=lib-openwrt-upstream.sh
source "$SCRIPT_DIR/lib-openwrt-upstream.sh"

DEVICE_ID=""
DEVICE_TARGET=""
VERSION_OVERRIDE=""
WITH_PACKAGES=false
SINGLE_PROFILE=true
STRIP_BUILDBOT=true
SAVE_BUILDINFO=false
OFFLINE=false
OUTPUT=""
RUN_DEFCONFIG=true

usage() { sed -n '2,37p' "$0" | sed 's/^# \{0,1\}//;s/^#$//'; }

need_value() {
  if [ "$2" -lt 2 ] || [ -z "${3:-}" ]; then
    err "$1 requires a value."
    exit 1
  fi
}

while [ $# -gt 0 ]; do
  case "$1" in
    --device)  need_value "$1" $# "${2:-}"; DEVICE_ID="$2"; shift ;;
    --target)  need_value "$1" $# "${2:-}"; DEVICE_TARGET="$2"; shift ;;
    --version) need_value "$1" $# "${2:-}"; VERSION_OVERRIDE="$2"; shift ;;
    --snapshot) VERSION_OVERRIDE="SNAPSHOT" ;;
    --with-packages) WITH_PACKAGES=true ;;
    --all-profiles) SINGLE_PROFILE=false ;;
    --keep-buildbot-flags) STRIP_BUILDBOT=false ;;
    --save-buildinfo) SAVE_BUILDINFO=true ;;
    --offline) OFFLINE=true ;;
    --output)  need_value "$1" $# "${2:-}"; OUTPUT="$2"; shift ;;
    --refresh) REFRESH=true ;;
    --skip-defconfig) RUN_DEFCONFIG=false ;;
    -y|--yes) ASSUME_YES=true ;;
    -n|--dry-run) DRY_RUN=true ;;
    -h|--help) usage; exit 0 ;;
    *) err "Unknown option: $1"; echo "Run '$0 --help' for usage." >&2; exit 1 ;;
  esac
  shift
done

[ -t 0 ] || ASSUME_YES=true

require_python3

if [ -n "$OUTPUT" ]; then
  DEST="$OUTPUT"
  RUN_DEFCONFIG=false
else
  require_openwrt_root
  DEST=".config"
fi
DEST_DIR="$(dirname -- "$DEST")"

VER="$(detect_version)"

if [ "$OFFLINE" != true ] && ! version_exists "$VER"; then
  err "Version '$VER' is not published at $BASE_URL."
  err "  $(url_overview "$VER")"
  exit 1
fi

select_device "$VER"

SAVED_BUILDINFO="$SCRIPT_DIR/config_$VER/${DEV_ID}_${VER}.buildinfo"

SYSUPGRADE_URL=""
FACTORY_URL=""
PKG_COUNT="?"
if [ "$OFFLINE" != true ]; then
  PROFILES_JSON="$(fetch_profiles "$VER" "$DEV_TARGET")"
  PKG_COUNT="$(profile_packages "$PROFILES_JSON" "$DEV_ID" | grep -c . || true)"
  while IFS= read -r img; do
    case "$img" in
      *squashfs-sysupgrade*) SYSUPGRADE_URL="$(url_image "$VER" "$DEV_TARGET" "$img")" ;;
      *squashfs-factory*)    FACTORY_URL="$(url_image "$VER" "$DEV_TARGET" "$img")" ;;
    esac
  done < <(profile_images "$PROFILES_JSON" "$DEV_ID")
fi

CONFIG_URL="$(url_buildinfo "$VER" "$DEV_TARGET")"

echo
echo "========================================"
echo " Planned actions"
echo "========================================"
echo "   Device        : $DEV_ID  ($DEV_LABEL)"
echo "   Target        : $DEV_TARGET"
echo "   Version       : $VER"
if [ "$OFFLINE" = true ]; then
  echo "   Source        : $SAVED_BUILDINFO (offline)"
else
  echo "   Source        : $CONFIG_URL"
fi
echo "   Destination   : $DEST$([ -f "$DEST" ] && echo '  (backed up first)')"
echo "   Device scope  : $([ "$SINGLE_PROFILE" = true ] && echo 'this device only' || echo 'all devices in target')"
echo "   Buildbot flags: $([ "$STRIP_BUILDBOT" = true ] && echo 'stripped' || echo 'kept')"
echo "   Built-in pkgs : $PKG_COUNT (per profiles.json)"
[ -n "$SYSUPGRADE_URL" ] && echo "   Sysupgrade    : $SYSUPGRADE_URL"
echo "   Post          : $([ "$RUN_DEFCONFIG" = true ] && echo 'make defconfig' || echo 'skipped')"
echo "========================================"

if [ "$DRY_RUN" != true ]; then
  echo
  if ! confirm "Proceed?" yes; then
    info "Aborted by user."
    exit 0
  fi
fi

# --- Download into a temp file next to the destination -------------------
# Same directory keeps the final mv atomic; `curl -f` plus this staging is
# what stops a 404 from truncating an existing .config.
TMP="$(mktemp "$DEST_DIR/.config.dl.XXXXXX")"
trap 'rm -f "$TMP" "$TMP.pp"' EXIT

if [ "$OFFLINE" = true ]; then
  if [ ! -f "$SAVED_BUILDINFO" ]; then
    err "No saved buildinfo at $SAVED_BUILDINFO"
    err "Run once without --offline (add --save-buildinfo) to create it."
    exit 1
  fi
  info "Using the saved buildinfo (offline)."
  cp "$SAVED_BUILDINFO" "$TMP"
else
  info "Downloading $CONFIG_URL"
  if ! fetch_url "$CONFIG_URL" "$TMP"; then
    err "Download failed; $DEST was not touched."
    exit 1
  fi
fi

# --- Validate before letting it near .config ------------------------------
if [ ! -s "$TMP" ] || [ "$(wc -l < "$TMP")" -lt 5 ]; then
  err "Downloaded config looks empty or truncated; $DEST was not touched."
  exit 1
fi
if ! grep -qx "CONFIG_TARGET_${DEV_TSYM}=y" "$TMP"; then
  err "Downloaded config is not for target $DEV_TARGET; $DEST was not touched."
  exit 1
fi
if ! grep -q "_DEVICE_${DEV_ID}=y" "$TMP"; then
  err "Downloaded config does not contain device $DEV_ID; $DEST was not touched."
  exit 1
fi

# --- Post-process ---------------------------------------------------------
# One awk pass: a chain of `sed -i` would rewrite the file repeatedly and the
# symbol names contain '-' and '+', which are awkward in sed replacements.
awk -v tsym="$DEV_TSYM" -v prof="$DEV_ID" \
    -v single="$SINGLE_PROFILE" -v strip="$STRIP_BUILDBOT" '
  # Matches both CONFIG_TARGET_DEVICE_<t>_DEVICE_<id>=y and
  # CONFIG_TARGET_DEVICE_PACKAGES_<t>_DEVICE_<id>="". It cannot match the
  # single-profile symbol appended below, which has no DEVICE_ prefix.
  single == "true" && /^CONFIG_TARGET_DEVICE_/ { next }
  single == "true" && /^CONFIG_TARGET_MULTI_PROFILE=y$/     { print "# CONFIG_TARGET_MULTI_PROFILE is not set"; next }
  single == "true" && /^CONFIG_TARGET_ALL_PROFILES=y$/      { print "# CONFIG_TARGET_ALL_PROFILES is not set"; next }
  single == "true" && /^CONFIG_TARGET_PER_DEVICE_ROOTFS=y$/ { print "# CONFIG_TARGET_PER_DEVICE_ROOTFS is not set"; next }
  strip == "true" && $0 ~ /^CONFIG_(ALL_KMODS|ALL_NONSHARED|BUILDBOT|SDK|SDK_LLVM_BPF|IB|MAKE_TOOLCHAIN|COLLECT_KERNEL_DEBUG|JSON_CYCLONEDX_SBOM|AUTOREMOVE)=y$/ {
    sym = $0; sub(/=y$/, "", sym); print "# " sym " is not set"; next
  }
  { print }
  END {
    # CONFIG_TARGET_PROFILE is deliberately not written: Kconfig derives it.
    if (single == "true") print "CONFIG_TARGET_" tsym "_DEVICE_" prof "=y"
  }
' "$TMP" > "$TMP.pp"

if [ "$DRY_RUN" = true ]; then
  echo
  info "Resulting config (first differences against the download):"
  diff -u "$TMP" "$TMP.pp" | head -40 || true
  echo
  info "Dry run complete; $DEST was not touched."
  exit 0
fi

if [ "$SAVE_BUILDINFO" = true ]; then
  mkdir -p "$(dirname "$SAVED_BUILDINFO")"
  cp "$TMP" "$SAVED_BUILDINFO"
  info "Saved the raw buildinfo to $SAVED_BUILDINFO"
fi

if [ -f "$DEST" ]; then
  cp -a "$DEST" "$DEST.download.bak"
  info "Previous config kept at $DEST.download.bak"
fi
mv -f "$TMP.pp" "$DEST"
chmod 644 "$DEST"
info "Wrote $DEST"

if [ "$RUN_DEFCONFIG" = true ]; then
  info "Running make defconfig..."
  make defconfig >/dev/null

  # A config whose symbols do not exist in this tree is silently dropped by
  # Kconfig, leaving a config that looks fine and builds the wrong thing.
  if [ "$SINGLE_PROFILE" = true ]; then
    if ! grep -qx "CONFIG_TARGET_${DEV_TSYM}_DEVICE_${DEV_ID}=y" .config ||
      ! grep -qx "CONFIG_TARGET_PROFILE=\"DEVICE_${DEV_ID}\"" .config; then
      err "The device symbol did not survive make defconfig."
      err "Your tree ($(git describe --tags 2>/dev/null || echo unknown)) probably"
      err "does not match release $VER. Restore with: cp $DEST.download.bak $DEST"
      exit 1
    fi
  fi
fi

echo
echo "========================================"
echo "OK  Configuration installed."
echo "   Device:  $DEV_ID ($DEV_LABEL)"
echo "   Target:  $DEV_TARGET"
echo "   Version: $VER"
echo "========================================"
[ -n "$SYSUPGRADE_URL" ] && echo "   Official sysupgrade: $SYSUPGRADE_URL"
[ -n "$FACTORY_URL" ] && echo "   Official factory:    $FACTORY_URL"
echo "   Firmware Selector:   $(selector_url "$VER" "$DEV_TARGET" "$DEV_ID")"
echo

if [ "$WITH_PACKAGES" = true ]; then
  GEN_ARGS=(--device "$DEV_ID" --target "$DEV_TARGET" --version "$VER")
  [ "$ASSUME_YES" = true ] && GEN_ARGS+=(-y)
  "$SCRIPT_DIR/gen-package-list.sh" "${GEN_ARGS[@]}"
else
  echo "Next steps:"
  echo "   ./helper/gen-package-list.sh --device $DEV_ID --version $VER"
  echo "   ./helper/add-openwrt-packages.sh helper/config_$VER/packages_${DEV_ID//-/_}_$VER.txt"
  echo "   make menuconfig"
  echo "   make -j\$(nproc) download world"
  echo
fi