openwrt-build-helper/lib-openwrt-upstream.sh

#!/bin/bash
# Shared helpers for talking to downloads.openwrt.org.
#
# Sourced by download-config.sh and gen-package-list.sh. Not executable on its
# own. Callers are expected to set `set -euo pipefail`; this file does not.
#
# Author: Zhe Yuan

[ -n "${_OWRT_LIB_LOADED:-}" ] && return 0
_OWRT_LIB_LOADED=1

BASE_URL="${BASE_URL:-https://downloads.openwrt.org}"
CACHE_DIR="${CACHE_DIR:-${XDG_CACHE_HOME:-$HOME/.cache}/openwrt-build-helper}"
CACHE_TTL="${CACHE_TTL:-86400}"

# What the Firmware Selector adds on top of the target's own package set.
# This is a UI convention, not an API, so keep it overridable.
SELECTOR_EXTRAS="${SELECTOR_EXTRAS:-luci luci-app-attendedsysupgrade}"

# Devices offered when an interactive search is left empty. Only a convenience
# shortcut -- the target still comes from .overview.json, never hardcoded.
FAVOURITE_DEVICES="${FAVOURITE_DEVICES:-linksys_mx8500 cudy_tr3000-v1 cudy_tr3000-256mb-v1}"

REFRESH="${REFRESH:-false}"
DRY_RUN="${DRY_RUN:-false}"
ASSUME_YES="${ASSUME_YES:-false}"

# --- Output helpers -------------------------------------------------------
# All logging goes to stderr so that functions returning a value on stdout
# (detect_version, profile_packages, ...) can be captured safely.
info() { echo "[INFO] $*" >&2; }
warn() { echo "[WARN] $*" >&2; }
err()  { echo "[ERROR] $*" >&2; }

# Wrap every mutating command so --dry-run prints instead of running.
run() {
  if [ "$DRY_RUN" = true ]; then
    printf '[DRY-RUN]'
    printf ' %q' "$@"
    echo
    return 0
  fi
  "$@"
}

confirm() {
  # confirm "<question>" <default: yes|no>
  local question="$1" default="$2" reply=""

  [ "$ASSUME_YES" = true ] && return 0

  if [ "$default" = "yes" ]; then
    read -r -p "$question [Y/n]: " reply || reply=""
    case "${reply:-}" in
      [nN] | [nN][oO]) return 1 ;;
      *) return 0 ;;
    esac
  else
    read -r -p "$question [y/N]: " reply || reply=""
    case "${reply:-}" in
      [yY] | [yY][eE][sS]) return 0 ;;
      *) return 1 ;;
    esac
  fi
}

require_python3() {
  if ! command -v python3 >/dev/null 2>&1; then
    err "python3 is required to parse the upstream JSON indexes."
    exit 1
  fi
}

# The helper checkout is itself a git repository living inside the OpenWrt
# worktree, so plain git commands run from here describe the HELPER repo.
# Every version query must be anchored to the OpenWrt tree instead.
find_openwrt_root() {
  local d
  for d in "$PWD" "$(dirname -- "${LIB_SCRIPT_DIR:-$PWD}")"; do
    [ -n "$d" ] || continue
    if [ -f "$d/feeds.conf.default" ] && [ -f "$d/Makefile" ] && [ -d "$d/scripts" ]; then
      ( cd -- "$d" && pwd -P )
      return 0
    fi
  done
  return 1
}

require_openwrt_root() {
  if [ ! -f feeds.conf.default ] || [ ! -f Makefile ] || [ ! -d scripts ]; then
    err "Must run inside the OpenWrt root directory."
    err "From the OpenWrt root, run it as: ./helper/$(basename "$0")"
    exit 1
  fi
}

# --- URL construction -----------------------------------------------------
# Every upstream URL is built from this one function.
release_path() {
  local ver="$1"
  if [ "$ver" = "SNAPSHOT" ]; then
    echo "snapshots"
  else
    echo "releases/$ver"
  fi
}

url_overview()  { echo "$BASE_URL/$(release_path "$1")/.overview.json"; }
url_profiles()  { echo "$BASE_URL/$(release_path "$1")/targets/$2/profiles.json"; }
url_buildinfo() { echo "$BASE_URL/$(release_path "$1")/targets/$2/config.buildinfo"; }
url_image()     { echo "$BASE_URL/$(release_path "$1")/targets/$2/$3"; }

selector_url() {
  local ver="$1" target="$2" id="$3"
  echo "https://firmware-selector.openwrt.org/?version=$ver&target=${target//\//%2F}&id=$id"
}

# --- Fetching -------------------------------------------------------------
# `curl -f` is the key flag: a 404 exits non-zero and never writes a body, so
# a failed download can never truncate the destination.
fetch_url() {
  local url="$1" dest="$2"

  if ! curl -fsSL --retry 3 --retry-delay 2 --max-time 60 -o "$dest.part" "$url"; then
    rm -f "$dest.part"
    return 1
  fi
  mv -f "$dest.part" "$dest"
}

fetch_cached() {
  local url="$1" cache="$2" age=0

  mkdir -p "$(dirname "$cache")"

  if [ "$REFRESH" != true ] && [ -s "$cache" ]; then
    age=$(( $(date +%s) - $(stat -c %Y "$cache" 2>/dev/null || echo 0) ))
    if [ "$age" -lt "$CACHE_TTL" ]; then
      return 0
    fi
  fi

  if fetch_url "$url" "$cache"; then
    return 0
  fi

  # A stale cache beats no data when the network is down.
  if [ -s "$cache" ]; then
    warn "Could not refresh $url; using the cached copy."
    return 0
  fi
  return 1
}

version_exists() {
  curl -fsI --max-time 30 "$(url_overview "$1")" >/dev/null 2>&1
}

# --- Version detection ----------------------------------------------------
# Replaces `git describe --tags --abbrev=0`, which returns a tag whenever ANY
# tag is reachable -- so the old script's whole fallback chain was dead code
# and a branch tip past a tag silently produced the wrong release URL.
detect_version() {
  local ver="" branch="" tag="" drift=0

  if [ -n "${VERSION_OVERRIDE:-}" ]; then
    ver="${VERSION_OVERRIDE#v}"
    info "Version from the command line: $ver"
    echo "$ver"
    return 0
  fi

  local root=""
  if ! root="$(find_openwrt_root)"; then
    err "Cannot locate the OpenWrt source tree; pass --version <ver> explicitly."
    exit 1
  fi
  if ! git -C "$root" rev-parse --is-inside-work-tree >/dev/null 2>&1; then
    err "'$root' is not a Git repository and no --version was given."
    exit 1
  fi

  if tag="$(git -C "$root" describe --tags --exact-match HEAD 2>/dev/null)"; then
    ver="${tag#v}"
    info "HEAD is exactly at tag $tag; using version $ver."
    echo "$ver"
    return 0
  fi

  if branch="$(git -C "$root" symbolic-ref --quiet --short HEAD 2>/dev/null)"; then
    case "$branch" in
      main | master)
        info "On branch '$branch'; using SNAPSHOT."
        echo "SNAPSHOT"
        return 0
        ;;
      openwrt-* | [0-9]*)
        local series="${branch#openwrt-}"
        # --merged HEAD keeps tags from unrelated branches out of the picture.
        tag="$(git -C "$root" tag --merged HEAD -l "v${series}.*" 2>/dev/null |
          grep -v -- '-rc' | sort -V | tail -n 1)"
        if [ -n "$tag" ]; then
          drift="$(git -C "$root" rev-list --count "$tag..HEAD" 2>/dev/null || echo 0)"
          if [ "$drift" -gt 0 ]; then
            warn "HEAD is $drift commit(s) past $tag."
            warn "The downloaded config describes the release, not your tree."
          fi
          ver="${tag#v}"
          info "Branch '$branch' -> newest merged tag $tag; using version $ver."
          echo "$ver"
          return 0
        fi
        ;;
    esac
  fi

  if tag="$(git -C "$root" describe --tags --long 2>/dev/null)"; then
    tag="${tag%-*-g*}"
    drift="$(git -C "$root" rev-list --count "$tag..HEAD" 2>/dev/null || echo 0)"
    warn "HEAD is not at a tag; nearest is $tag ($drift commit(s) behind HEAD)."
    if [ "$ASSUME_YES" != true ] &&
      ! confirm "Use version ${tag#v}?" no; then
      err "Aborted; pass --version explicitly."
      exit 1
    fi
    ver="${tag#v}"
    info "Using version $ver."
    echo "$ver"
    return 0
  fi

  if [ "$ASSUME_YES" = true ] || [ ! -t 0 ]; then
    err "Cannot determine the OpenWrt version; pass --version <ver> or --snapshot."
    exit 1
  fi

  read -r -p "Enter version (e.g. 25.12.5 or SNAPSHOT): " ver || ver=""
  if [ -z "$ver" ]; then
    err "No version given."
    exit 1
  fi
  echo "${ver#v}"
}

# --- Device index ---------------------------------------------------------
fetch_overview() {
  local ver="$1"
  # Separate statement on purpose: `local a=$1 b=$a` expands every word before
  # assigning, so $a would be unbound under `set -u`.
  local cache="$CACHE_DIR/overview-$ver.json"

  # Snapshots move daily, so never serve them from a day-old cache.
  [ "$ver" = "SNAPSHOT" ] && REFRESH=true

  if ! fetch_cached "$(url_overview "$ver")" "$cache"; then
    err "Cannot fetch the device index for version '$ver'."
    err "  $(url_overview "$ver")"
    exit 1
  fi
  echo "$cache"
}

# Emits TSV: id <tab> target <tab> label
_overview_py() {
  python3 - "$@" <<'PY'
import json, sys, difflib

path, mode, arg = sys.argv[1], sys.argv[2], sys.argv[3]
data = json.load(open(path, encoding="utf-8"))
profiles = data.get("profiles") or []


def label(p):
    parts = []
    for t in p.get("titles") or []:
        if not isinstance(t, dict):
            continue
        # Upstream entries come in three shapes: {title}, {vendor,model},
        # and {vendor,model,variant}.
        if t.get("title"):
            parts.append(t["title"])
        else:
            parts.append(" ".join(
                str(x) for x in (t.get("vendor"), t.get("model"), t.get("variant")) if x
            ))
    parts = [p for p in parts if p]
    return " / ".join(parts) or p.get("id", "")


def emit(rows):
    for p in rows:
        print("%s\t%s\t%s" % (p.get("id", ""), p.get("target", ""), label(p)))


if mode == "id":
    rows = [p for p in profiles if p.get("id") == arg]
    if not rows:
        names = [p.get("id", "") for p in profiles]
        close = difflib.get_close_matches(arg, names, n=5, cutoff=0.5)
        sys.stderr.write("no device with id '%s'\n" % arg)
        if close:
            sys.stderr.write("closest matches: %s\n" % ", ".join(close))
        sys.exit(2)
    emit(rows)
elif mode == "search":
    terms = [t.lower() for t in arg.split() if t]
    rows = []
    for p in profiles:
        hay = (p.get("id", "") + " " + label(p)).lower()
        if all(t in hay for t in terms):
            rows.append(p)
    rows.sort(key=lambda p: p.get("id", ""))
    emit(rows)
elif mode == "list":
    wanted = arg.split()
    by_id = {p.get("id"): p for p in profiles}
    emit([by_id[i] for i in wanted if i in by_id])
PY
}

# Sets DEV_ID, DEV_TARGET, DEV_TSYM, DEV_LABEL
select_device() {
  local ver="$1" json="" rows="" n=0 choice="" tries=0 query=""

  json="$(fetch_overview "$ver")"

  if [ -n "${DEVICE_ID:-}" ]; then
    if ! rows="$(_overview_py "$json" id "$DEVICE_ID" 2>&1)"; then
      err "$rows"
      exit 1
    fi
    if [ -n "${DEVICE_TARGET:-}" ]; then
      rows="$(printf '%s\n' "$rows" | awk -F'\t' -v t="$DEVICE_TARGET" '$2 == t')"
    fi
    n="$(printf '%s\n' "$rows" | grep -c . || true)"
    if [ "$n" -eq 0 ]; then
      err "Device '$DEVICE_ID' exists but not for target '${DEVICE_TARGET:-}'."
      exit 1
    fi
    if [ "$n" -gt 1 ]; then
      err "Device id '$DEVICE_ID' is present in more than one target:"
      printf '%s\n' "$rows" | awk -F'\t' '{print "   " $2 "  (" $3 ")"}' >&2
      err "Disambiguate with --target <target>."
      exit 1
    fi
  else
    if [ "$ASSUME_YES" = true ] || [ ! -t 0 ]; then
      err "No device selected. Pass --device <id> for non-interactive use."
      exit 1
    fi

    while [ "$tries" -lt 3 ]; do
      tries=$((tries + 1))
      read -r -p "Search device (vendor, model or id; empty for favourites): " query || query=""

      if [ -z "${query:-}" ]; then
        rows="$(_overview_py "$json" list "$FAVOURITE_DEVICES")"
      else
        rows="$(_overview_py "$json" search "$query")"
      fi

      n="$(printf '%s\n' "$rows" | grep -c . || true)"
      if [ "$n" -eq 0 ]; then
        warn "No device matched '$query'."
        continue
      fi
      if [ "$n" -gt 25 ]; then
        warn "$n devices matched; please narrow the search."
        continue
      fi

      echo
      printf '%s\n' "$rows" | awk -F'\t' '{printf "  %2d) %-32s %-22s %s\n", NR, $1, $2, $3}'
      echo
      read -r -p "Select device number [1-$n]: " choice || choice=""

      # 1-based plus an explicit numeric test: an unset/non-numeric index in
      # an array subscript is an arithmetic context that silently yields 0.
      if [[ "${choice:-}" =~ ^[0-9]+$ ]] && [ "$choice" -ge 1 ] && [ "$choice" -le "$n" ]; then
        rows="$(printf '%s\n' "$rows" | sed -n "${choice}p")"
        break
      fi
      warn "Invalid selection."
      rows=""
    done

    if [ -z "$rows" ]; then
      err "No device selected."
      exit 1
    fi
  fi

  IFS=$'\t' read -r DEV_ID DEV_TARGET DEV_LABEL <<<"$rows"
  DEV_TSYM="${DEV_TARGET//\//_}"

  if [ -z "$DEV_ID" ] || [ -z "$DEV_TARGET" ]; then
    err "Could not resolve the device selection."
    exit 1
  fi
}

# --- profiles.json --------------------------------------------------------
fetch_profiles() {
  local ver="$1" target="$2"
  local cache="$CACHE_DIR/profiles-$ver-${target//\//_}.json"

  [ "$ver" = "SNAPSHOT" ] && REFRESH=true

  if ! fetch_cached "$(url_profiles "$ver" "$target")" "$cache"; then
    err "Cannot fetch profiles.json for $target at version $ver."
    err "  $(url_profiles "$ver" "$target")"
    exit 1
  fi
  echo "$cache"
}

# default_packages + device_packages, order preserved, deduplicated.
profile_packages() {
  python3 - "$1" "$2" <<'PY'
import json, sys

data = json.load(open(sys.argv[1], encoding="utf-8"))
pid = sys.argv[2]
prof = (data.get("profiles") or {}).get(pid)
if prof is None:
    sys.stderr.write("profile '%s' is not in this target's profiles.json\n" % pid)
    sys.exit(2)

seen, out = set(), []
for p in list(data.get("default_packages") or []) + list(prof.get("device_packages") or []):
    if p not in seen:
        seen.add(p)
        out.append(p)
print("\n".join(out))
PY
}

# Image filenames straight from profiles.json -- no string building, which is
# what made the old firmware URL wrong (the target still contained a '/').
profile_images() {
  python3 - "$1" "$2" <<'PY'
import json, sys

data = json.load(open(sys.argv[1], encoding="utf-8"))
prof = (data.get("profiles") or {}).get(sys.argv[2]) or {}
for img in prof.get("images") or []:
    name = img.get("name")
    if name:
        print(name)
PY
}