293 lines
10 KiB
Bash
293 lines
10 KiB
Bash
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
OUTDIR="${OUTDIR:-offline-seed}"
|
|
M2OUT="$OUTDIR/m2repo"
|
|
LOGDIR="$OUTDIR/logs"
|
|
|
|
ARTIFACT_URLS="$OUTDIR/artifact-urls.txt"
|
|
FAILED_URLS="$OUTDIR/failed-urls.txt"
|
|
REAL_FAILED_URLS="$OUTDIR/real-failed-urls.txt"
|
|
IGNORED_FAILED_URLS="$OUTDIR/ignored-failed-urls.txt"
|
|
MISSING_URLS="$OUTDIR/missing-from-nexus.txt"
|
|
REAL_MISSING_URLS="$OUTDIR/real-missing-from-nexus.txt"
|
|
RETRY_URLS="$OUTDIR/retry-source-urls.txt"
|
|
REPO_CANDIDATES="$OUTDIR/repository-candidates.txt"
|
|
REPO_HINTS="$OUTDIR/repository-hints.txt"
|
|
|
|
mkdir -p "$M2OUT" "$LOGDIR"
|
|
: > "$FAILED_URLS"
|
|
: > "$REAL_FAILED_URLS"
|
|
: > "$IGNORED_FAILED_URLS"
|
|
: > "$MISSING_URLS"
|
|
: > "$REAL_MISSING_URLS"
|
|
: > "$RETRY_URLS"
|
|
: > "$REPO_CANDIDATES"
|
|
: > "$REPO_HINTS"
|
|
|
|
GRADLE_CMD=(./gradlew --no-daemon)
|
|
|
|
SPRING_BOOT_PLUGIN_VERSION="${SPRING_BOOT_PLUGIN_VERSION:-3.5.7}"
|
|
DEPENDENCY_MGMT_PLUGIN_VERSION="${DEPENDENCY_MGMT_PLUGIN_VERSION:-1.1.7}"
|
|
SPOTLESS_PLUGIN_VERSION="${SPOTLESS_PLUGIN_VERSION:-6.25.0}"
|
|
QUERYDSL_VERSION="${QUERYDSL_VERSION:-5.0.0}"
|
|
|
|
MAVEN_CENTRAL_BASE="${MAVEN_CENTRAL_BASE:-https://repo1.maven.org/maven2}"
|
|
GRADLE_PLUGIN_PORTAL_MAVEN_BASE="${GRADLE_PLUGIN_PORTAL_MAVEN_BASE:-https://plugins.gradle.org/m2}"
|
|
OSGEO_RELEASE_BASE="${OSGEO_RELEASE_BASE:-https://repo.osgeo.org/repository/release}"
|
|
|
|
echo "[1/8] Running Gradle to trigger resolves..."
|
|
"${GRADLE_CMD[@]}" \
|
|
clean compileJava testClasses spotlessCheck build \
|
|
-x test \
|
|
--refresh-dependencies \
|
|
--debug \
|
|
2>&1 | tee "$LOGDIR/gradle-debug.log" || true
|
|
|
|
echo "[2/8] Extracting resolved artifact URLs from Gradle debug log..."
|
|
grep -Eo 'https?://[^ ]+\.(pom|jar|module|aar|sha1|md5)(\?[^ ]+)?' "$LOGDIR/gradle-debug.log" \
|
|
| sed -E 's/\?.*$//' \
|
|
| sort -u \
|
|
> "$ARTIFACT_URLS"
|
|
|
|
echo " - URLs from log: $(wc -l < "$ARTIFACT_URLS")"
|
|
|
|
echo "[3/8] Appending known required artifacts..."
|
|
cat >> "$ARTIFACT_URLS" <<EOF
|
|
# Spring Boot Gradle plugin marker + implementation
|
|
${MAVEN_CENTRAL_BASE}/org/springframework/boot/org.springframework.boot.gradle.plugin/${SPRING_BOOT_PLUGIN_VERSION}/org.springframework.boot.gradle.plugin-${SPRING_BOOT_PLUGIN_VERSION}.pom
|
|
${MAVEN_CENTRAL_BASE}/org/springframework/boot/spring-boot-gradle-plugin/${SPRING_BOOT_PLUGIN_VERSION}/spring-boot-gradle-plugin-${SPRING_BOOT_PLUGIN_VERSION}.pom
|
|
${MAVEN_CENTRAL_BASE}/org/springframework/boot/spring-boot-gradle-plugin/${SPRING_BOOT_PLUGIN_VERSION}/spring-boot-gradle-plugin-${SPRING_BOOT_PLUGIN_VERSION}.module
|
|
${MAVEN_CENTRAL_BASE}/org/springframework/boot/spring-boot-gradle-plugin/${SPRING_BOOT_PLUGIN_VERSION}/spring-boot-gradle-plugin-${SPRING_BOOT_PLUGIN_VERSION}.jar
|
|
|
|
# Dependency management Gradle plugin marker + implementation
|
|
${GRADLE_PLUGIN_PORTAL_MAVEN_BASE}/io/spring/dependency-management/io.spring.dependency-management.gradle.plugin/${DEPENDENCY_MGMT_PLUGIN_VERSION}/io.spring.dependency-management.gradle.plugin-${DEPENDENCY_MGMT_PLUGIN_VERSION}.pom
|
|
${GRADLE_PLUGIN_PORTAL_MAVEN_BASE}/io/spring/gradle/dependency-management-plugin/${DEPENDENCY_MGMT_PLUGIN_VERSION}/dependency-management-plugin-${DEPENDENCY_MGMT_PLUGIN_VERSION}.pom
|
|
${GRADLE_PLUGIN_PORTAL_MAVEN_BASE}/io/spring/gradle/dependency-management-plugin/${DEPENDENCY_MGMT_PLUGIN_VERSION}/dependency-management-plugin-${DEPENDENCY_MGMT_PLUGIN_VERSION}.module
|
|
${GRADLE_PLUGIN_PORTAL_MAVEN_BASE}/io/spring/gradle/dependency-management-plugin/${DEPENDENCY_MGMT_PLUGIN_VERSION}/dependency-management-plugin-${DEPENDENCY_MGMT_PLUGIN_VERSION}.jar
|
|
|
|
# Spotless Gradle plugin marker + implementation
|
|
${GRADLE_PLUGIN_PORTAL_MAVEN_BASE}/com/diffplug/spotless/com.diffplug.spotless.gradle.plugin/${SPOTLESS_PLUGIN_VERSION}/com.diffplug.spotless.gradle.plugin-${SPOTLESS_PLUGIN_VERSION}.pom
|
|
${GRADLE_PLUGIN_PORTAL_MAVEN_BASE}/com/diffplug/spotless/spotless-plugin-gradle/${SPOTLESS_PLUGIN_VERSION}/spotless-plugin-gradle-${SPOTLESS_PLUGIN_VERSION}.pom
|
|
${GRADLE_PLUGIN_PORTAL_MAVEN_BASE}/com/diffplug/spotless/spotless-plugin-gradle/${SPOTLESS_PLUGIN_VERSION}/spotless-plugin-gradle-${SPOTLESS_PLUGIN_VERSION}.module
|
|
${GRADLE_PLUGIN_PORTAL_MAVEN_BASE}/com/diffplug/spotless/spotless-plugin-gradle/${SPOTLESS_PLUGIN_VERSION}/spotless-plugin-gradle-${SPOTLESS_PLUGIN_VERSION}.jar
|
|
|
|
# Querydsl jakarta classifier artifacts
|
|
${MAVEN_CENTRAL_BASE}/com/querydsl/querydsl-apt/${QUERYDSL_VERSION}/querydsl-apt-${QUERYDSL_VERSION}.pom
|
|
${MAVEN_CENTRAL_BASE}/com/querydsl/querydsl-apt/${QUERYDSL_VERSION}/querydsl-apt-${QUERYDSL_VERSION}.jar
|
|
${MAVEN_CENTRAL_BASE}/com/querydsl/querydsl-apt/${QUERYDSL_VERSION}/querydsl-apt-${QUERYDSL_VERSION}-jakarta.jar
|
|
${MAVEN_CENTRAL_BASE}/com/querydsl/querydsl-jpa/${QUERYDSL_VERSION}/querydsl-jpa-${QUERYDSL_VERSION}.pom
|
|
${MAVEN_CENTRAL_BASE}/com/querydsl/querydsl-jpa/${QUERYDSL_VERSION}/querydsl-jpa-${QUERYDSL_VERSION}.jar
|
|
${MAVEN_CENTRAL_BASE}/com/querydsl/querydsl-jpa/${QUERYDSL_VERSION}/querydsl-jpa-${QUERYDSL_VERSION}-jakarta.jar
|
|
${MAVEN_CENTRAL_BASE}/com/querydsl/querydsl-jpa-codegen/${QUERYDSL_VERSION}/querydsl-jpa-codegen-${QUERYDSL_VERSION}.pom
|
|
${MAVEN_CENTRAL_BASE}/com/querydsl/querydsl-jpa-codegen/${QUERYDSL_VERSION}/querydsl-jpa-codegen-${QUERYDSL_VERSION}.jar
|
|
${MAVEN_CENTRAL_BASE}/com/querydsl/querydsl-jpa-codegen/${QUERYDSL_VERSION}/querydsl-jpa-codegen-${QUERYDSL_VERSION}-jakarta.jar
|
|
EOF
|
|
|
|
grep -vE '^\s*#|^\s*$' "$ARTIFACT_URLS" | sort -u > "$ARTIFACT_URLS.tmp"
|
|
mv "$ARTIFACT_URLS.tmp" "$ARTIFACT_URLS"
|
|
echo " - URLs after manual append: $(wc -l < "$ARTIFACT_URLS")"
|
|
|
|
to_relative_path() {
|
|
local url="$1"
|
|
|
|
if [[ "$url" == *"/maven2/"* ]]; then
|
|
printf '%s\n' "${url#*/maven2/}"
|
|
return 0
|
|
fi
|
|
|
|
if [[ "$url" == *"/m2/"* ]]; then
|
|
printf '%s\n' "${url#*/m2/}"
|
|
return 0
|
|
fi
|
|
|
|
if [[ "$url" =~ ^https?://[^/]+/repository/[^/]+/(.+)$ ]]; then
|
|
printf '%s\n' "${BASH_REMATCH[1]}"
|
|
return 0
|
|
fi
|
|
|
|
return 1
|
|
}
|
|
|
|
is_ignorable_url() {
|
|
local url="$1"
|
|
|
|
case "$url" in
|
|
*/org.springframework.boot.gradle.plugin-*.jar)
|
|
return 0
|
|
;;
|
|
*/org.springframework.boot.gradle.plugin-*.module)
|
|
return 0
|
|
;;
|
|
*/io.spring.dependency-management.gradle.plugin-*.jar)
|
|
return 0
|
|
;;
|
|
*/io.spring.dependency-management.gradle.plugin-*.module)
|
|
return 0
|
|
;;
|
|
*/com.diffplug.spotless.gradle.plugin-*.jar)
|
|
return 0
|
|
;;
|
|
*/com.diffplug.spotless.gradle.plugin-*.module)
|
|
return 0
|
|
;;
|
|
*/com/querydsl/querydsl-apt/*/querydsl-apt-*.module)
|
|
return 0
|
|
;;
|
|
*/com/querydsl/querydsl-jpa/*/querydsl-jpa-*.module)
|
|
return 0
|
|
;;
|
|
*)
|
|
return 1
|
|
;;
|
|
esac
|
|
}
|
|
|
|
map_upstream_url() {
|
|
local url="$1"
|
|
local rel
|
|
|
|
if ! rel="$(to_relative_path "$url")"; then
|
|
return 1
|
|
fi
|
|
|
|
case "$rel" in
|
|
org/geotools/*|it/geosolutions/*|javax/media/*)
|
|
printf '%s/%s\n' "$OSGEO_RELEASE_BASE" "$rel"
|
|
;;
|
|
io/spring/dependency-management/*|io/spring/gradle/dependency-management-plugin/*)
|
|
printf '%s/%s\n' "$GRADLE_PLUGIN_PORTAL_MAVEN_BASE" "$rel"
|
|
;;
|
|
com/diffplug/spotless/*)
|
|
printf '%s/%s\n' "$GRADLE_PLUGIN_PORTAL_MAVEN_BASE" "$rel"
|
|
;;
|
|
*)
|
|
printf '%s/%s\n' "$MAVEN_CENTRAL_BASE" "$rel"
|
|
;;
|
|
esac
|
|
}
|
|
|
|
echo "[4/8] Collecting artifacts that were missing in Nexus during Gradle resolve..."
|
|
grep 'Resource missing\.' "$LOGDIR/gradle-debug.log" \
|
|
| sed -nE 's#.*\[HTTP (HEAD|GET): (https?://[^]]+)\]#\2#p' \
|
|
| sort -u \
|
|
> "$MISSING_URLS" || true
|
|
|
|
while IFS= read -r url; do
|
|
[[ -z "$url" ]] && continue
|
|
if is_ignorable_url "$url"; then
|
|
printf '%s\n' "$url" >> "$IGNORED_FAILED_URLS"
|
|
continue
|
|
fi
|
|
printf '%s\n' "$url" >> "$REAL_MISSING_URLS"
|
|
mapped="$(map_upstream_url "$url" || true)"
|
|
if [[ -n "${mapped:-}" ]] && ! is_ignorable_url "$mapped"; then
|
|
printf '%s\n' "$mapped" >> "$RETRY_URLS"
|
|
printf '%s\n' "${mapped%/*}" >> "$REPO_HINTS"
|
|
fi
|
|
done < "$MISSING_URLS"
|
|
|
|
sort -u "$REAL_MISSING_URLS" -o "$REAL_MISSING_URLS"
|
|
sort -u "$IGNORED_FAILED_URLS" -o "$IGNORED_FAILED_URLS"
|
|
sort -u "$RETRY_URLS" -o "$RETRY_URLS"
|
|
sort -u "$REPO_HINTS" -o "$REPO_HINTS"
|
|
|
|
cat "$RETRY_URLS" >> "$ARTIFACT_URLS"
|
|
sort -u "$ARTIFACT_URLS" -o "$ARTIFACT_URLS"
|
|
|
|
echo " - Missing URLs in Nexus log: $(wc -l < "$MISSING_URLS")"
|
|
echo " - Real missing URLs: $(wc -l < "$REAL_MISSING_URLS")"
|
|
echo " - Ignored missing URLs: $(wc -l < "$IGNORED_FAILED_URLS")"
|
|
|
|
download_one() {
|
|
local url="$1"
|
|
local rel
|
|
|
|
if is_ignorable_url "$url"; then
|
|
printf '%s\n' "$url" >> "$IGNORED_FAILED_URLS"
|
|
return 0
|
|
fi
|
|
|
|
if ! rel="$(to_relative_path "$url")"; then
|
|
echo " - [SKIP] unknown URL form: $url"
|
|
return 0
|
|
fi
|
|
|
|
local dest="$M2OUT/$rel"
|
|
mkdir -p "$(dirname "$dest")"
|
|
|
|
if [[ -f "$dest" ]]; then
|
|
return 0
|
|
fi
|
|
|
|
echo " - [GET] $rel"
|
|
if ! curl -fSL "$url" -o "$dest"; then
|
|
echo "$url" >> "$FAILED_URLS"
|
|
rm -f "$dest" 2>/dev/null || true
|
|
if is_ignorable_url "$url"; then
|
|
printf '%s\n' "$url" >> "$IGNORED_FAILED_URLS"
|
|
else
|
|
printf '%s\n' "$url" >> "$REAL_FAILED_URLS"
|
|
echo " !! failed: $url"
|
|
fi
|
|
fi
|
|
}
|
|
|
|
export -f to_relative_path
|
|
export -f is_ignorable_url
|
|
export -f download_one
|
|
export M2OUT FAILED_URLS REAL_FAILED_URLS IGNORED_FAILED_URLS
|
|
|
|
echo "[5/8] Downloading artifacts into m2 layout..."
|
|
while IFS= read -r url; do
|
|
[[ -z "$url" ]] && continue
|
|
download_one "$url"
|
|
done < "$ARTIFACT_URLS"
|
|
|
|
sort -u "$FAILED_URLS" -o "$FAILED_URLS"
|
|
sort -u "$REAL_FAILED_URLS" -o "$REAL_FAILED_URLS"
|
|
sort -u "$IGNORED_FAILED_URLS" -o "$IGNORED_FAILED_URLS"
|
|
|
|
echo "[6/8] Extracting repository candidates declared inside downloaded POMs..."
|
|
while IFS= read -r -d '' pom; do
|
|
awk -v pom="$pom" '
|
|
/<repository>/ { in_repo=1; id=""; url=""; next }
|
|
/<\/repository>/ {
|
|
if (in_repo && url != "" && url !~ /^file:/) {
|
|
print url "\t" pom "\t" id
|
|
}
|
|
in_repo=0
|
|
id=""
|
|
url=""
|
|
next
|
|
}
|
|
in_repo {
|
|
if (match($0, /<id>([^<]+)<\/id>/, m)) {
|
|
id=m[1]
|
|
}
|
|
if (match($0, /<url>([^<]+)<\/url>/, m)) {
|
|
url=m[1]
|
|
}
|
|
}
|
|
' "$pom"
|
|
done < <(find "$M2OUT" -type f -name '*.pom' -print0) \
|
|
| sort -u \
|
|
> "$REPO_CANDIDATES"
|
|
|
|
echo " - POM repository candidates: $(wc -l < "$REPO_CANDIDATES")"
|
|
|
|
echo "[7/8] Packing seed bundle..."
|
|
tar -czf "$OUTDIR/m2repo-seed.tgz" -C "$OUTDIR" m2repo
|
|
ls -lh "$OUTDIR/m2repo-seed.tgz"
|
|
|
|
echo "[8/8] Summary"
|
|
echo " seed bundle : $OUTDIR/m2repo-seed.tgz"
|
|
echo " artifact URLs : $ARTIFACT_URLS"
|
|
echo " raw missing : $MISSING_URLS"
|
|
echo " real missing : $REAL_MISSING_URLS"
|
|
echo " raw failed : $FAILED_URLS"
|
|
echo " real failed : $REAL_FAILED_URLS"
|
|
echo " ignored failed : $IGNORED_FAILED_URLS"
|
|
echo " retry source URLs : $RETRY_URLS"
|
|
echo " repository hints : $REPO_HINTS"
|
|
echo " POM repo candidates : $REPO_CANDIDATES" |