test3
This commit is contained in:
@@ -1,7 +1,12 @@
|
|||||||
{
|
{
|
||||||
"permissions": {
|
"permissions": {
|
||||||
"allow": [
|
"allow": [
|
||||||
"WebSearch"
|
"WebSearch",
|
||||||
|
"Bash(cp /Users/d-pn-0131/dev/kamco-cd-cron/shp-exporter/gradlew /Users/d-pn-0131/dev/kamco-cd-cron/shp-exporter_v2/gradlew)",
|
||||||
|
"Bash(cp /Users/d-pn-0131/dev/kamco-cd-cron/shp-exporter/gradlew.bat /Users/d-pn-0131/dev/kamco-cd-cron/shp-exporter_v2/gradlew.bat)",
|
||||||
|
"Bash(cp -r /Users/d-pn-0131/dev/kamco-cd-cron/shp-exporter/gradle /Users/d-pn-0131/dev/kamco-cd-cron/shp-exporter_v2/gradle)",
|
||||||
|
"Read(//Users/d-pn-0131/dev/kamco-cd-cron/shp-exporter_v2/**)",
|
||||||
|
"Bash(./gradlew clean *)"
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
129
shp-exporter_v2/README.md
Normal file
129
shp-exporter_v2/README.md
Normal file
@@ -0,0 +1,129 @@
|
|||||||
|
# shp-exporter-v2
|
||||||
|
|
||||||
|
`inference_results_testing` 테이블에서 공간 데이터를 읽어 **map_id 단위로 Shapefile(.shp)을 생성**하는 Spring Batch 애플리케이션입니다.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 요구사항
|
||||||
|
|
||||||
|
| 항목 | 버전 |
|
||||||
|
|------|------|
|
||||||
|
| Java | 21 |
|
||||||
|
| Gradle | Wrapper 사용 (별도 설치 불필요) |
|
||||||
|
| DB | PostgreSQL + PostGIS (`inference_results_testing` 테이블) |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 설정
|
||||||
|
|
||||||
|
`src/main/resources/application-prod.yml` 에서 아래 항목을 환경에 맞게 수정합니다.
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
spring:
|
||||||
|
datasource:
|
||||||
|
url: jdbc:postgresql://<HOST>:<PORT>/<DB>
|
||||||
|
username: <USERNAME>
|
||||||
|
password: <PASSWORD>
|
||||||
|
|
||||||
|
exporter:
|
||||||
|
inference-id: 'D5E46F60FC40B1A8BE0CD1F3547AA6' # 추출 대상 inference ID
|
||||||
|
batch-ids: # 추출 대상 batch_id 목록
|
||||||
|
- 252
|
||||||
|
- 253
|
||||||
|
- 257
|
||||||
|
output-base-dir: '/data/model_output/export/' # Shapefile 저장 경로 (디렉토리)
|
||||||
|
crs: 'EPSG:5186' # 출력 좌표계
|
||||||
|
chunk-size: 1000 # 청크 단위 (기본값 유지 권장)
|
||||||
|
fetch-size: 1000 # DB cursor fetch 크기
|
||||||
|
skip-limit: 100 # 오류 허용 건수 (초과 시 Job 실패)
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 빌드
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./gradlew bootJar
|
||||||
|
```
|
||||||
|
|
||||||
|
빌드 결과물: `build/libs/shp-exporter-v2.jar`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 실행
|
||||||
|
|
||||||
|
### 방법 1 — Gradle로 직접 실행 (개발/테스트)
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./gradlew bootRun
|
||||||
|
```
|
||||||
|
|
||||||
|
### 방법 2 — JAR 실행 (운영)
|
||||||
|
|
||||||
|
```bash
|
||||||
|
java \
|
||||||
|
-Xmx128g -Xms8g \
|
||||||
|
-XX:+UseG1GC \
|
||||||
|
-XX:MaxGCPauseMillis=200 \
|
||||||
|
-XX:G1HeapRegionSize=16m \
|
||||||
|
-XX:+ParallelRefProcEnabled \
|
||||||
|
-jar build/libs/shp-exporter-v2.jar
|
||||||
|
```
|
||||||
|
|
||||||
|
> 대용량 데이터 처리를 위해 힙 메모리를 충분히 확보하는 것을 권장합니다.
|
||||||
|
|
||||||
|
### 방법 3 — 설정을 커맨드라인으로 오버라이드
|
||||||
|
|
||||||
|
```bash
|
||||||
|
java -jar build/libs/shp-exporter-v2.jar \
|
||||||
|
--exporter.batch-ids=252,253,257 \
|
||||||
|
--exporter.output-base-dir=/data/model_output/export/ \
|
||||||
|
--exporter.inference-id=D5E46F60FC40B1A8BE0CD1F3547AA6
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 출력 구조
|
||||||
|
|
||||||
|
```
|
||||||
|
{output-base-dir}/
|
||||||
|
{map_id}/
|
||||||
|
{map_id}.shp
|
||||||
|
{map_id}.dbf
|
||||||
|
{map_id}.shx
|
||||||
|
{map_id}.prj
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Job 처리 흐름
|
||||||
|
|
||||||
|
```
|
||||||
|
Step 1 (geomTypeStep)
|
||||||
|
└─ DB에서 geometry 타입 확인
|
||||||
|
|
||||||
|
Step 2 (generateMapIdFilesStep)
|
||||||
|
└─ inference_results_testing 조회 (batch_id 필터, ORDER BY map_id, uid)
|
||||||
|
└─ map_id가 바뀔 때마다 Shapefile 파일 분리 저장
|
||||||
|
```
|
||||||
|
|
||||||
|
**데이터 필터 조건 (고정)**
|
||||||
|
- `batch_id = ANY(batch-ids 목록)`
|
||||||
|
- geometry 타입: `ST_Polygon` 또는 `ST_MultiPolygon`
|
||||||
|
- SRID: `5186`
|
||||||
|
- 유효한 geometry (`ST_IsValid = true`)
|
||||||
|
- 좌표 범위: X `125000~530000`, Y `-600000~988000` (EPSG:5186 한반도 범위)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 로그 확인
|
||||||
|
|
||||||
|
정상 실행 시 아래와 같은 로그가 출력됩니다.
|
||||||
|
|
||||||
|
```
|
||||||
|
=== shp-exporter-v2 시작 ===
|
||||||
|
inference-id : D5E46F60FC40B1A8BE0CD1F3547AA6
|
||||||
|
batch-ids : [252, 253, 257]
|
||||||
|
output : /data/model_output/export/
|
||||||
|
Job 완료: ExitStatus [COMPLETED]
|
||||||
|
```
|
||||||
67
shp-exporter_v2/build.gradle
Normal file
67
shp-exporter_v2/build.gradle
Normal file
@@ -0,0 +1,67 @@
|
|||||||
|
plugins {
|
||||||
|
id 'java'
|
||||||
|
id 'org.springframework.boot' version '3.5.7'
|
||||||
|
id 'io.spring.dependency-management' version '1.1.7'
|
||||||
|
}
|
||||||
|
|
||||||
|
group = 'com.kamco'
|
||||||
|
version = '2.0.0'
|
||||||
|
|
||||||
|
java {
|
||||||
|
toolchain {
|
||||||
|
languageVersion = JavaLanguageVersion.of(21)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
repositories {
|
||||||
|
mavenCentral()
|
||||||
|
maven { url 'https://repo.osgeo.org/repository/release/' }
|
||||||
|
maven { url 'https://repo.osgeo.org/repository/geotools-releases/' }
|
||||||
|
}
|
||||||
|
|
||||||
|
ext {
|
||||||
|
geoToolsVersion = '30.0'
|
||||||
|
}
|
||||||
|
|
||||||
|
configurations.all {
|
||||||
|
exclude group: 'javax.media', module: 'jai_core'
|
||||||
|
}
|
||||||
|
|
||||||
|
bootJar {
|
||||||
|
archiveFileName = "shp-exporter-v2.jar"
|
||||||
|
}
|
||||||
|
|
||||||
|
jar {
|
||||||
|
enabled = false
|
||||||
|
}
|
||||||
|
|
||||||
|
dependencies {
|
||||||
|
implementation 'org.springframework.boot:spring-boot-starter'
|
||||||
|
implementation 'org.springframework.boot:spring-boot-starter-jdbc'
|
||||||
|
implementation 'org.springframework.boot:spring-boot-starter-batch'
|
||||||
|
implementation 'org.springframework.boot:spring-boot-starter-validation'
|
||||||
|
|
||||||
|
implementation 'org.postgresql:postgresql'
|
||||||
|
implementation 'com.zaxxer:HikariCP'
|
||||||
|
implementation 'net.postgis:postgis-jdbc:2.5.1'
|
||||||
|
|
||||||
|
implementation 'org.locationtech.jts:jts-core:1.19.0'
|
||||||
|
|
||||||
|
implementation "org.geotools:gt-shapefile:${geoToolsVersion}"
|
||||||
|
implementation "org.geotools:gt-referencing:${geoToolsVersion}"
|
||||||
|
implementation "org.geotools:gt-epsg-hsql:${geoToolsVersion}"
|
||||||
|
|
||||||
|
testImplementation 'org.springframework.boot:spring-boot-starter-test'
|
||||||
|
testRuntimeOnly 'org.junit.platform:junit-platform-launcher'
|
||||||
|
}
|
||||||
|
|
||||||
|
tasks.named('test') {
|
||||||
|
useJUnitPlatform()
|
||||||
|
}
|
||||||
|
|
||||||
|
bootRun {
|
||||||
|
jvmArgs = ['-Xmx128g', '-Xms8g', '-XX:+UseG1GC',
|
||||||
|
'-XX:MaxGCPauseMillis=200',
|
||||||
|
'-XX:G1HeapRegionSize=16m',
|
||||||
|
'-XX:+ParallelRefProcEnabled']
|
||||||
|
}
|
||||||
BIN
shp-exporter_v2/gradle/wrapper/gradle-wrapper.jar
vendored
Executable file
BIN
shp-exporter_v2/gradle/wrapper/gradle-wrapper.jar
vendored
Executable file
Binary file not shown.
8
shp-exporter_v2/gradle/wrapper/gradle-wrapper.properties
vendored
Executable file
8
shp-exporter_v2/gradle/wrapper/gradle-wrapper.properties
vendored
Executable file
@@ -0,0 +1,8 @@
|
|||||||
|
distributionBase=GRADLE_USER_HOME
|
||||||
|
distributionPath=wrapper/dists
|
||||||
|
distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.3-bin.zip
|
||||||
|
#distributionUrl=http\://172.16.4.56:18100/repository/gradle-distributions/gradle-8.14.3-bin.zip
|
||||||
|
networkTimeout=10000
|
||||||
|
validateDistributionUrl=true
|
||||||
|
zipStoreBase=GRADLE_USER_HOME
|
||||||
|
zipStorePath=wrapper/dists
|
||||||
251
shp-exporter_v2/gradlew
vendored
Executable file
251
shp-exporter_v2/gradlew
vendored
Executable file
@@ -0,0 +1,251 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
|
||||||
|
#
|
||||||
|
# Copyright © 2015-2021 the original authors.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# https://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
#
|
||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
#
|
||||||
|
|
||||||
|
##############################################################################
|
||||||
|
#
|
||||||
|
# Gradle start up script for POSIX generated by Gradle.
|
||||||
|
#
|
||||||
|
# Important for running:
|
||||||
|
#
|
||||||
|
# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is
|
||||||
|
# noncompliant, but you have some other compliant shell such as ksh or
|
||||||
|
# bash, then to run this script, type that shell name before the whole
|
||||||
|
# command line, like:
|
||||||
|
#
|
||||||
|
# ksh Gradle
|
||||||
|
#
|
||||||
|
# Busybox and similar reduced shells will NOT work, because this script
|
||||||
|
# requires all of these POSIX shell features:
|
||||||
|
# * functions;
|
||||||
|
# * expansions «$var», «${var}», «${var:-default}», «${var+SET}»,
|
||||||
|
# «${var#prefix}», «${var%suffix}», and «$( cmd )»;
|
||||||
|
# * compound commands having a testable exit status, especially «case»;
|
||||||
|
# * various built-in commands including «command», «set», and «ulimit».
|
||||||
|
#
|
||||||
|
# Important for patching:
|
||||||
|
#
|
||||||
|
# (2) This script targets any POSIX shell, so it avoids extensions provided
|
||||||
|
# by Bash, Ksh, etc; in particular arrays are avoided.
|
||||||
|
#
|
||||||
|
# The "traditional" practice of packing multiple parameters into a
|
||||||
|
# space-separated string is a well documented source of bugs and security
|
||||||
|
# problems, so this is (mostly) avoided, by progressively accumulating
|
||||||
|
# options in "$@", and eventually passing that to Java.
|
||||||
|
#
|
||||||
|
# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS,
|
||||||
|
# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly;
|
||||||
|
# see the in-line comments for details.
|
||||||
|
#
|
||||||
|
# There are tweaks for specific operating systems such as AIX, CygWin,
|
||||||
|
# Darwin, MinGW, and NonStop.
|
||||||
|
#
|
||||||
|
# (3) This script is generated from the Groovy template
|
||||||
|
# https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
|
||||||
|
# within the Gradle project.
|
||||||
|
#
|
||||||
|
# You can find Gradle at https://github.com/gradle/gradle/.
|
||||||
|
#
|
||||||
|
##############################################################################
|
||||||
|
|
||||||
|
# Attempt to set APP_HOME
|
||||||
|
|
||||||
|
# Resolve links: $0 may be a link
|
||||||
|
app_path=$0
|
||||||
|
|
||||||
|
# Need this for daisy-chained symlinks.
|
||||||
|
while
|
||||||
|
APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path
|
||||||
|
[ -h "$app_path" ]
|
||||||
|
do
|
||||||
|
ls=$( ls -ld "$app_path" )
|
||||||
|
link=${ls#*' -> '}
|
||||||
|
case $link in #(
|
||||||
|
/*) app_path=$link ;; #(
|
||||||
|
*) app_path=$APP_HOME$link ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
# This is normally unused
|
||||||
|
# shellcheck disable=SC2034
|
||||||
|
APP_BASE_NAME=${0##*/}
|
||||||
|
# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036)
|
||||||
|
APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s\n' "$PWD" ) || exit
|
||||||
|
|
||||||
|
# Use the maximum available, or set MAX_FD != -1 to use that value.
|
||||||
|
MAX_FD=maximum
|
||||||
|
|
||||||
|
warn () {
|
||||||
|
echo "$*"
|
||||||
|
} >&2
|
||||||
|
|
||||||
|
die () {
|
||||||
|
echo
|
||||||
|
echo "$*"
|
||||||
|
echo
|
||||||
|
exit 1
|
||||||
|
} >&2
|
||||||
|
|
||||||
|
# OS specific support (must be 'true' or 'false').
|
||||||
|
cygwin=false
|
||||||
|
msys=false
|
||||||
|
darwin=false
|
||||||
|
nonstop=false
|
||||||
|
case "$( uname )" in #(
|
||||||
|
CYGWIN* ) cygwin=true ;; #(
|
||||||
|
Darwin* ) darwin=true ;; #(
|
||||||
|
MSYS* | MINGW* ) msys=true ;; #(
|
||||||
|
NONSTOP* ) nonstop=true ;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
CLASSPATH="\\\"\\\""
|
||||||
|
|
||||||
|
|
||||||
|
# Determine the Java command to use to start the JVM.
|
||||||
|
if [ -n "$JAVA_HOME" ] ; then
|
||||||
|
if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
|
||||||
|
# IBM's JDK on AIX uses strange locations for the executables
|
||||||
|
JAVACMD=$JAVA_HOME/jre/sh/java
|
||||||
|
else
|
||||||
|
JAVACMD=$JAVA_HOME/bin/java
|
||||||
|
fi
|
||||||
|
if [ ! -x "$JAVACMD" ] ; then
|
||||||
|
die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
|
||||||
|
|
||||||
|
Please set the JAVA_HOME variable in your environment to match the
|
||||||
|
location of your Java installation."
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
JAVACMD=java
|
||||||
|
if ! command -v java >/dev/null 2>&1
|
||||||
|
then
|
||||||
|
die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
|
||||||
|
|
||||||
|
Please set the JAVA_HOME variable in your environment to match the
|
||||||
|
location of your Java installation."
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Increase the maximum file descriptors if we can.
|
||||||
|
if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
|
||||||
|
case $MAX_FD in #(
|
||||||
|
max*)
|
||||||
|
# In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked.
|
||||||
|
# shellcheck disable=SC2039,SC3045
|
||||||
|
MAX_FD=$( ulimit -H -n ) ||
|
||||||
|
warn "Could not query maximum file descriptor limit"
|
||||||
|
esac
|
||||||
|
case $MAX_FD in #(
|
||||||
|
'' | soft) :;; #(
|
||||||
|
*)
|
||||||
|
# In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked.
|
||||||
|
# shellcheck disable=SC2039,SC3045
|
||||||
|
ulimit -n "$MAX_FD" ||
|
||||||
|
warn "Could not set maximum file descriptor limit to $MAX_FD"
|
||||||
|
esac
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Collect all arguments for the java command, stacking in reverse order:
|
||||||
|
# * args from the command line
|
||||||
|
# * the main class name
|
||||||
|
# * -classpath
|
||||||
|
# * -D...appname settings
|
||||||
|
# * --module-path (only if needed)
|
||||||
|
# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables.
|
||||||
|
|
||||||
|
# For Cygwin or MSYS, switch paths to Windows format before running java
|
||||||
|
if "$cygwin" || "$msys" ; then
|
||||||
|
APP_HOME=$( cygpath --path --mixed "$APP_HOME" )
|
||||||
|
CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" )
|
||||||
|
|
||||||
|
JAVACMD=$( cygpath --unix "$JAVACMD" )
|
||||||
|
|
||||||
|
# Now convert the arguments - kludge to limit ourselves to /bin/sh
|
||||||
|
for arg do
|
||||||
|
if
|
||||||
|
case $arg in #(
|
||||||
|
-*) false ;; # don't mess with options #(
|
||||||
|
/?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath
|
||||||
|
[ -e "$t" ] ;; #(
|
||||||
|
*) false ;;
|
||||||
|
esac
|
||||||
|
then
|
||||||
|
arg=$( cygpath --path --ignore --mixed "$arg" )
|
||||||
|
fi
|
||||||
|
# Roll the args list around exactly as many times as the number of
|
||||||
|
# args, so each arg winds up back in the position where it started, but
|
||||||
|
# possibly modified.
|
||||||
|
#
|
||||||
|
# NB: a `for` loop captures its iteration list before it begins, so
|
||||||
|
# changing the positional parameters here affects neither the number of
|
||||||
|
# iterations, nor the values presented in `arg`.
|
||||||
|
shift # remove old arg
|
||||||
|
set -- "$@" "$arg" # push replacement arg
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
|
||||||
|
|
||||||
|
# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
|
||||||
|
DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
|
||||||
|
|
||||||
|
# Collect all arguments for the java command:
|
||||||
|
# * DEFAULT_JVM_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments,
|
||||||
|
# and any embedded shellness will be escaped.
|
||||||
|
# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be
|
||||||
|
# treated as '${Hostname}' itself on the command line.
|
||||||
|
|
||||||
|
set -- \
|
||||||
|
"-Dorg.gradle.appname=$APP_BASE_NAME" \
|
||||||
|
-classpath "$CLASSPATH" \
|
||||||
|
-jar "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" \
|
||||||
|
"$@"
|
||||||
|
|
||||||
|
# Stop when "xargs" is not available.
|
||||||
|
if ! command -v xargs >/dev/null 2>&1
|
||||||
|
then
|
||||||
|
die "xargs is not available"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Use "xargs" to parse quoted args.
|
||||||
|
#
|
||||||
|
# With -n1 it outputs one arg per line, with the quotes and backslashes removed.
|
||||||
|
#
|
||||||
|
# In Bash we could simply go:
|
||||||
|
#
|
||||||
|
# readarray ARGS < <( xargs -n1 <<<"$var" ) &&
|
||||||
|
# set -- "${ARGS[@]}" "$@"
|
||||||
|
#
|
||||||
|
# but POSIX shell has neither arrays nor command substitution, so instead we
|
||||||
|
# post-process each arg (as a line of input to sed) to backslash-escape any
|
||||||
|
# character that might be a shell metacharacter, then use eval to reverse
|
||||||
|
# that process (while maintaining the separation between arguments), and wrap
|
||||||
|
# the whole thing up as a single "set" statement.
|
||||||
|
#
|
||||||
|
# This will of course break if any of these variables contains a newline or
|
||||||
|
# an unmatched quote.
|
||||||
|
#
|
||||||
|
|
||||||
|
eval "set -- $(
|
||||||
|
printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" |
|
||||||
|
xargs -n1 |
|
||||||
|
sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' |
|
||||||
|
tr '\n' ' '
|
||||||
|
)" '"$@"'
|
||||||
|
|
||||||
|
exec "$JAVACMD" "$@"
|
||||||
94
shp-exporter_v2/gradlew.bat
vendored
Executable file
94
shp-exporter_v2/gradlew.bat
vendored
Executable file
@@ -0,0 +1,94 @@
|
|||||||
|
@rem
|
||||||
|
@rem Copyright 2015 the original author or authors.
|
||||||
|
@rem
|
||||||
|
@rem Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
@rem you may not use this file except in compliance with the License.
|
||||||
|
@rem You may obtain a copy of the License at
|
||||||
|
@rem
|
||||||
|
@rem https://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
@rem
|
||||||
|
@rem Unless required by applicable law or agreed to in writing, software
|
||||||
|
@rem distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
@rem See the License for the specific language governing permissions and
|
||||||
|
@rem limitations under the License.
|
||||||
|
@rem
|
||||||
|
@rem SPDX-License-Identifier: Apache-2.0
|
||||||
|
@rem
|
||||||
|
|
||||||
|
@if "%DEBUG%"=="" @echo off
|
||||||
|
@rem ##########################################################################
|
||||||
|
@rem
|
||||||
|
@rem Gradle startup script for Windows
|
||||||
|
@rem
|
||||||
|
@rem ##########################################################################
|
||||||
|
|
||||||
|
@rem Set local scope for the variables with windows NT shell
|
||||||
|
if "%OS%"=="Windows_NT" setlocal
|
||||||
|
|
||||||
|
set DIRNAME=%~dp0
|
||||||
|
if "%DIRNAME%"=="" set DIRNAME=.
|
||||||
|
@rem This is normally unused
|
||||||
|
set APP_BASE_NAME=%~n0
|
||||||
|
set APP_HOME=%DIRNAME%
|
||||||
|
|
||||||
|
@rem Resolve any "." and ".." in APP_HOME to make it shorter.
|
||||||
|
for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi
|
||||||
|
|
||||||
|
@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
|
||||||
|
set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m"
|
||||||
|
|
||||||
|
@rem Find java.exe
|
||||||
|
if defined JAVA_HOME goto findJavaFromJavaHome
|
||||||
|
|
||||||
|
set JAVA_EXE=java.exe
|
||||||
|
%JAVA_EXE% -version >NUL 2>&1
|
||||||
|
if %ERRORLEVEL% equ 0 goto execute
|
||||||
|
|
||||||
|
echo. 1>&2
|
||||||
|
echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2
|
||||||
|
echo. 1>&2
|
||||||
|
echo Please set the JAVA_HOME variable in your environment to match the 1>&2
|
||||||
|
echo location of your Java installation. 1>&2
|
||||||
|
|
||||||
|
goto fail
|
||||||
|
|
||||||
|
:findJavaFromJavaHome
|
||||||
|
set JAVA_HOME=%JAVA_HOME:"=%
|
||||||
|
set JAVA_EXE=%JAVA_HOME%/bin/java.exe
|
||||||
|
|
||||||
|
if exist "%JAVA_EXE%" goto execute
|
||||||
|
|
||||||
|
echo. 1>&2
|
||||||
|
echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2
|
||||||
|
echo. 1>&2
|
||||||
|
echo Please set the JAVA_HOME variable in your environment to match the 1>&2
|
||||||
|
echo location of your Java installation. 1>&2
|
||||||
|
|
||||||
|
goto fail
|
||||||
|
|
||||||
|
:execute
|
||||||
|
@rem Setup the command line
|
||||||
|
|
||||||
|
set CLASSPATH=
|
||||||
|
|
||||||
|
|
||||||
|
@rem Execute Gradle
|
||||||
|
"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" -jar "%APP_HOME%\gradle\wrapper\gradle-wrapper.jar" %*
|
||||||
|
|
||||||
|
:end
|
||||||
|
@rem End local scope for the variables with windows NT shell
|
||||||
|
if %ERRORLEVEL% equ 0 goto mainEnd
|
||||||
|
|
||||||
|
:fail
|
||||||
|
rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
|
||||||
|
rem the _cmd.exe /c_ return code!
|
||||||
|
set EXIT_CODE=%ERRORLEVEL%
|
||||||
|
if %EXIT_CODE% equ 0 set EXIT_CODE=1
|
||||||
|
if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE%
|
||||||
|
exit /b %EXIT_CODE%
|
||||||
|
|
||||||
|
:mainEnd
|
||||||
|
if "%OS%"=="Windows_NT" endlocal
|
||||||
|
|
||||||
|
:omega
|
||||||
1
shp-exporter_v2/settings.gradle
Normal file
1
shp-exporter_v2/settings.gradle
Normal file
@@ -0,0 +1 @@
|
|||||||
|
rootProject.name = 'shp-exporter-v2'
|
||||||
@@ -0,0 +1,50 @@
|
|||||||
|
package com.kamco.shpexporter;
|
||||||
|
|
||||||
|
import com.kamco.shpexporter.config.ExporterProperties;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
import org.springframework.batch.core.Job;
|
||||||
|
import org.springframework.batch.core.JobParameters;
|
||||||
|
import org.springframework.batch.core.JobParametersBuilder;
|
||||||
|
import org.springframework.batch.core.launch.JobLauncher;
|
||||||
|
import org.springframework.boot.CommandLineRunner;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
|
@Component
|
||||||
|
public class ExporterRunner implements CommandLineRunner {
|
||||||
|
|
||||||
|
private static final Logger log = LoggerFactory.getLogger(ExporterRunner.class);
|
||||||
|
|
||||||
|
private final JobLauncher jobLauncher;
|
||||||
|
private final Job shpExporterJob;
|
||||||
|
private final ExporterProperties properties;
|
||||||
|
|
||||||
|
public ExporterRunner(JobLauncher jobLauncher, Job shpExporterJob,
|
||||||
|
ExporterProperties properties) {
|
||||||
|
this.jobLauncher = jobLauncher;
|
||||||
|
this.shpExporterJob = shpExporterJob;
|
||||||
|
this.properties = properties;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void run(String... args) throws Exception {
|
||||||
|
if (properties.getBatchIds() == null || properties.getBatchIds().isEmpty()) {
|
||||||
|
log.error("exporter.batch-ids 가 설정되지 않았습니다.");
|
||||||
|
System.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
log.info("=== shp-exporter-v2 시작 ===");
|
||||||
|
log.info("inference-id : {}", properties.getInferenceId());
|
||||||
|
log.info("batch-ids : {}", properties.getBatchIds());
|
||||||
|
log.info("output : {}", properties.getOutputBaseDir());
|
||||||
|
|
||||||
|
JobParameters params = new JobParametersBuilder()
|
||||||
|
.addString("inferenceId", properties.getInferenceId())
|
||||||
|
.addString("batchIds", properties.getBatchIds().toString())
|
||||||
|
.addLong("timestamp", System.currentTimeMillis())
|
||||||
|
.toJobParameters();
|
||||||
|
|
||||||
|
var result = jobLauncher.run(shpExporterJob, params);
|
||||||
|
log.info("Job 완료: {}", result.getExitStatus());
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
package com.kamco.shpexporter;
|
||||||
|
|
||||||
|
import org.springframework.boot.SpringApplication;
|
||||||
|
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||||
|
import org.springframework.boot.context.properties.EnableConfigurationProperties;
|
||||||
|
|
||||||
|
@SpringBootApplication
|
||||||
|
@EnableConfigurationProperties
|
||||||
|
public class ShpExporterApplication {
|
||||||
|
|
||||||
|
public static void main(String[] args) {
|
||||||
|
System.exit(SpringApplication.exit(SpringApplication.run(ShpExporterApplication.class, args)));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,124 @@
|
|||||||
|
package com.kamco.shpexporter.batch;
|
||||||
|
|
||||||
|
import com.kamco.shpexporter.batch.reader.GeometryConvertingRowMapper;
|
||||||
|
import com.kamco.shpexporter.batch.tasklet.GeomTypeTasklet;
|
||||||
|
import com.kamco.shpexporter.batch.writer.MapIdSwitchingWriter;
|
||||||
|
import com.kamco.shpexporter.config.ExporterProperties;
|
||||||
|
import com.kamco.shpexporter.model.InferenceResult;
|
||||||
|
import java.util.List;
|
||||||
|
import javax.sql.DataSource;
|
||||||
|
import org.geotools.api.referencing.FactoryException;
|
||||||
|
import org.geotools.api.referencing.crs.CoordinateReferenceSystem;
|
||||||
|
import org.geotools.referencing.CRS;
|
||||||
|
import org.springframework.batch.core.Job;
|
||||||
|
import org.springframework.batch.core.Step;
|
||||||
|
import org.springframework.batch.core.job.builder.JobBuilder;
|
||||||
|
import org.springframework.batch.core.repository.JobRepository;
|
||||||
|
import org.springframework.batch.core.step.builder.StepBuilder;
|
||||||
|
import org.springframework.batch.item.database.JdbcCursorItemReader;
|
||||||
|
import org.springframework.batch.item.database.builder.JdbcCursorItemReaderBuilder;
|
||||||
|
import org.springframework.context.annotation.Bean;
|
||||||
|
import org.springframework.context.annotation.Configuration;
|
||||||
|
import org.springframework.jdbc.core.PreparedStatementSetter;
|
||||||
|
import org.springframework.transaction.PlatformTransactionManager;
|
||||||
|
|
||||||
|
@Configuration
|
||||||
|
public class BatchJobConfig {
|
||||||
|
|
||||||
|
private final ExporterProperties properties;
|
||||||
|
|
||||||
|
public BatchJobConfig(ExporterProperties properties) {
|
||||||
|
this.properties = properties;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── CRS Bean ──────────────────────────────────────────────────
|
||||||
|
|
||||||
|
@Bean
|
||||||
|
public CoordinateReferenceSystem coordinateReferenceSystem() throws FactoryException {
|
||||||
|
return CRS.decode(properties.getCrs());
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── Job ───────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
@Bean
|
||||||
|
public Job shpExporterJob(
|
||||||
|
JobRepository jobRepository,
|
||||||
|
Step geomTypeStep,
|
||||||
|
Step generateMapIdFilesStep) {
|
||||||
|
|
||||||
|
return new JobBuilder("shpExporterJob", jobRepository)
|
||||||
|
.start(geomTypeStep)
|
||||||
|
.next(generateMapIdFilesStep)
|
||||||
|
.build();
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── Step 1: geometry 타입 확인 ────────────────────────────────
|
||||||
|
|
||||||
|
@Bean
|
||||||
|
public Step geomTypeStep(
|
||||||
|
JobRepository jobRepository,
|
||||||
|
PlatformTransactionManager transactionManager,
|
||||||
|
GeomTypeTasklet geomTypeTasklet) {
|
||||||
|
|
||||||
|
return new StepBuilder("geomTypeStep", jobRepository)
|
||||||
|
.tasklet(geomTypeTasklet, transactionManager)
|
||||||
|
.build();
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── Step 2: map_id별 shapefile 생성 ──────────────────────────
|
||||||
|
|
||||||
|
@Bean
|
||||||
|
public Step generateMapIdFilesStep(
|
||||||
|
JobRepository jobRepository,
|
||||||
|
PlatformTransactionManager transactionManager,
|
||||||
|
JdbcCursorItemReader<InferenceResult> inferenceResultReader,
|
||||||
|
MapIdSwitchingWriter mapIdSwitchingWriter) {
|
||||||
|
|
||||||
|
return new StepBuilder("generateMapIdFilesStep", jobRepository)
|
||||||
|
.<InferenceResult, InferenceResult>chunk(properties.getChunkSize(), transactionManager)
|
||||||
|
.reader(inferenceResultReader)
|
||||||
|
.writer(mapIdSwitchingWriter)
|
||||||
|
.faultTolerant()
|
||||||
|
.skipLimit(properties.getSkipLimit())
|
||||||
|
.skip(Exception.class)
|
||||||
|
.listener(mapIdSwitchingWriter) // @BeforeStep / @AfterStep 등록
|
||||||
|
.build();
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── Reader: ORDER BY map_id, uid (전체 스캔) ──────────────────
|
||||||
|
|
||||||
|
@Bean
|
||||||
|
public JdbcCursorItemReader<InferenceResult> inferenceResultReader(
|
||||||
|
DataSource dataSource,
|
||||||
|
GeometryConvertingRowMapper rowMapper) {
|
||||||
|
|
||||||
|
List<Long> batchIds = properties.getBatchIds();
|
||||||
|
|
||||||
|
String sql =
|
||||||
|
"SELECT uid, map_id, probability, before_year, after_year, "
|
||||||
|
+ " before_c, before_p, after_c, after_p, "
|
||||||
|
+ " ST_AsText(geometry) as geometry_wkt "
|
||||||
|
+ "FROM inference_results_testing "
|
||||||
|
+ "WHERE batch_id = ANY(?) "
|
||||||
|
+ " AND ST_GeometryType(geometry) IN ('ST_Polygon', 'ST_MultiPolygon') "
|
||||||
|
+ " AND ST_SRID(geometry) = 5186 "
|
||||||
|
+ " AND ST_IsValid(geometry) = true "
|
||||||
|
+ " AND ST_XMin(geometry) >= 125000 AND ST_XMax(geometry) <= 530000 "
|
||||||
|
+ " AND ST_YMin(geometry) >= -600000 AND ST_YMax(geometry) <= 988000 "
|
||||||
|
+ "ORDER BY map_id, uid";
|
||||||
|
|
||||||
|
PreparedStatementSetter pss = ps -> {
|
||||||
|
var arr = ps.getConnection().createArrayOf("bigint", batchIds.toArray());
|
||||||
|
ps.setArray(1, arr);
|
||||||
|
};
|
||||||
|
|
||||||
|
return new JdbcCursorItemReaderBuilder<InferenceResult>()
|
||||||
|
.name("inferenceResultReader")
|
||||||
|
.dataSource(dataSource)
|
||||||
|
.sql(sql)
|
||||||
|
.preparedStatementSetter(pss)
|
||||||
|
.rowMapper(rowMapper)
|
||||||
|
.fetchSize(properties.getFetchSize())
|
||||||
|
.build();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,47 @@
|
|||||||
|
package com.kamco.shpexporter.batch.reader;
|
||||||
|
|
||||||
|
import com.kamco.shpexporter.model.InferenceResult;
|
||||||
|
import com.kamco.shpexporter.service.GeometryConverter;
|
||||||
|
import java.sql.ResultSet;
|
||||||
|
import java.sql.SQLException;
|
||||||
|
import org.springframework.jdbc.core.RowMapper;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
|
@Component
|
||||||
|
public class GeometryConvertingRowMapper implements RowMapper<InferenceResult> {
|
||||||
|
|
||||||
|
private final GeometryConverter geometryConverter;
|
||||||
|
|
||||||
|
public GeometryConvertingRowMapper(GeometryConverter geometryConverter) {
|
||||||
|
this.geometryConverter = geometryConverter;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public InferenceResult mapRow(ResultSet rs, int rowNum) throws SQLException {
|
||||||
|
InferenceResult r = new InferenceResult();
|
||||||
|
r.setUid(rs.getString("uid"));
|
||||||
|
r.setMapId(rs.getString("map_id"));
|
||||||
|
r.setProbability(getDoubleOrNull(rs, "probability"));
|
||||||
|
r.setBeforeYear(getLongOrNull(rs, "before_year"));
|
||||||
|
r.setAfterYear(getLongOrNull(rs, "after_year"));
|
||||||
|
r.setBeforeC(rs.getString("before_c"));
|
||||||
|
r.setBeforeP(getDoubleOrNull(rs, "before_p"));
|
||||||
|
r.setAfterC(rs.getString("after_c"));
|
||||||
|
r.setAfterP(getDoubleOrNull(rs, "after_p"));
|
||||||
|
|
||||||
|
String wkt = rs.getString("geometry_wkt");
|
||||||
|
if (wkt != null) r.setGeometry(geometryConverter.convertWKTToJTS(wkt));
|
||||||
|
|
||||||
|
return r;
|
||||||
|
}
|
||||||
|
|
||||||
|
private Long getLongOrNull(ResultSet rs, String col) throws SQLException {
|
||||||
|
long v = rs.getLong(col);
|
||||||
|
return rs.wasNull() ? null : v;
|
||||||
|
}
|
||||||
|
|
||||||
|
private Double getDoubleOrNull(ResultSet rs, String col) throws SQLException {
|
||||||
|
double v = rs.getDouble(col);
|
||||||
|
return rs.wasNull() ? null : v;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,65 @@
|
|||||||
|
package com.kamco.shpexporter.batch.tasklet;
|
||||||
|
|
||||||
|
import com.kamco.shpexporter.config.ExporterProperties;
|
||||||
|
import java.sql.Array;
|
||||||
|
import java.util.List;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
import org.springframework.batch.core.StepContribution;
|
||||||
|
import org.springframework.batch.core.scope.context.ChunkContext;
|
||||||
|
import org.springframework.batch.core.step.tasklet.Tasklet;
|
||||||
|
import org.springframework.batch.repeat.RepeatStatus;
|
||||||
|
import org.springframework.jdbc.core.JdbcTemplate;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Step 1: geometry 타입 사전 확인 후 JobExecutionContext에 저장.
|
||||||
|
* MapIdSwitchingWriter가 이 값을 읽어 SimpleFeatureType을 구성합니다.
|
||||||
|
*/
|
||||||
|
@Component
|
||||||
|
public class GeomTypeTasklet implements Tasklet {
|
||||||
|
|
||||||
|
private static final Logger log = LoggerFactory.getLogger(GeomTypeTasklet.class);
|
||||||
|
|
||||||
|
private final JdbcTemplate jdbcTemplate;
|
||||||
|
private final ExporterProperties properties;
|
||||||
|
|
||||||
|
public GeomTypeTasklet(JdbcTemplate jdbcTemplate, ExporterProperties properties) {
|
||||||
|
this.jdbcTemplate = jdbcTemplate;
|
||||||
|
this.properties = properties;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public RepeatStatus execute(StepContribution contribution, ChunkContext chunkContext)
|
||||||
|
throws Exception {
|
||||||
|
|
||||||
|
List<Long> batchIds = properties.getBatchIds();
|
||||||
|
log.info("Validating geometry types for batch_ids: {}", batchIds);
|
||||||
|
|
||||||
|
List<String> geomTypes = jdbcTemplate.query(
|
||||||
|
con -> {
|
||||||
|
var ps = con.prepareStatement(
|
||||||
|
"SELECT DISTINCT ST_GeometryType(geometry) "
|
||||||
|
+ "FROM inference_results_testing "
|
||||||
|
+ "WHERE batch_id = ANY(?) AND geometry IS NOT NULL");
|
||||||
|
Array arr = con.createArrayOf("bigint", batchIds.toArray());
|
||||||
|
ps.setArray(1, arr);
|
||||||
|
return ps;
|
||||||
|
},
|
||||||
|
(rs, rowNum) -> rs.getString(1));
|
||||||
|
|
||||||
|
log.info("Detected geometry types: {}", geomTypes);
|
||||||
|
|
||||||
|
// MultiPolygon은 자동으로 Polygon으로 변환하므로 항상 Polygon으로 처리
|
||||||
|
String resolved = "Polygon";
|
||||||
|
log.info("Using geometry type: {}", resolved);
|
||||||
|
|
||||||
|
chunkContext.getStepContext()
|
||||||
|
.getStepExecution()
|
||||||
|
.getJobExecution()
|
||||||
|
.getExecutionContext()
|
||||||
|
.putString("geometryType", resolved);
|
||||||
|
|
||||||
|
return RepeatStatus.FINISHED;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,244 @@
|
|||||||
|
package com.kamco.shpexporter.batch.writer;
|
||||||
|
|
||||||
|
import com.kamco.shpexporter.config.ExporterProperties;
|
||||||
|
import com.kamco.shpexporter.model.InferenceResult;
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.Serializable;
|
||||||
|
import java.nio.file.Files;
|
||||||
|
import java.nio.file.Path;
|
||||||
|
import java.nio.file.Paths;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import org.geotools.api.data.SimpleFeatureStore;
|
||||||
|
import org.geotools.api.data.Transaction;
|
||||||
|
import org.geotools.api.feature.simple.SimpleFeature;
|
||||||
|
import org.geotools.api.feature.simple.SimpleFeatureType;
|
||||||
|
import org.geotools.api.referencing.crs.CoordinateReferenceSystem;
|
||||||
|
import org.geotools.data.DefaultTransaction;
|
||||||
|
import org.geotools.data.collection.ListFeatureCollection;
|
||||||
|
import org.geotools.data.shapefile.ShapefileDataStore;
|
||||||
|
import org.geotools.data.shapefile.ShapefileDataStoreFactory;
|
||||||
|
import org.geotools.feature.simple.SimpleFeatureBuilder;
|
||||||
|
import org.geotools.feature.simple.SimpleFeatureTypeBuilder;
|
||||||
|
import org.locationtech.jts.geom.Polygon;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
import org.springframework.batch.core.ExitStatus;
|
||||||
|
import org.springframework.batch.core.StepExecution;
|
||||||
|
import org.springframework.batch.core.annotation.AfterStep;
|
||||||
|
import org.springframework.batch.core.annotation.BeforeStep;
|
||||||
|
import org.springframework.batch.item.Chunk;
|
||||||
|
import org.springframework.batch.item.ExecutionContext;
|
||||||
|
import org.springframework.batch.item.ItemStreamException;
|
||||||
|
import org.springframework.batch.item.ItemStreamWriter;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* map_id별 Shapefile을 생성하는 Writer.
|
||||||
|
*
|
||||||
|
* <p>데이터를 map_id, uid 순서로 읽어 map_id가 바뀔 때마다 파일을 전환합니다.
|
||||||
|
* 파티셔닝 없이 단일 JDBC 커서로 전체 데이터를 스트리밍하며, 메모리에는
|
||||||
|
* chunk 1개(기본 1000건)만 유지합니다.
|
||||||
|
*
|
||||||
|
* <p>출력 경로: {output-base-dir}/{inference-id}/{map_id}/{map_id}.shp
|
||||||
|
*/
|
||||||
|
@Component
|
||||||
|
public class MapIdSwitchingWriter implements ItemStreamWriter<InferenceResult> {
|
||||||
|
|
||||||
|
private static final Logger log = LoggerFactory.getLogger(MapIdSwitchingWriter.class);
|
||||||
|
private static final int LOG_INTERVAL = 500; // 500 map_id마다 진행 로그
|
||||||
|
|
||||||
|
private final ExporterProperties properties;
|
||||||
|
private final CoordinateReferenceSystem crs;
|
||||||
|
private final ShapefileDataStoreFactory dsFactory = new ShapefileDataStoreFactory();
|
||||||
|
|
||||||
|
private SimpleFeatureType featureType;
|
||||||
|
private SimpleFeatureBuilder featureBuilder;
|
||||||
|
|
||||||
|
// 현재 열려 있는 파일 관련 상태
|
||||||
|
private String currentMapId = null;
|
||||||
|
private ShapefileDataStore currentDataStore;
|
||||||
|
private Transaction currentTransaction;
|
||||||
|
private SimpleFeatureStore currentFeatureStore;
|
||||||
|
|
||||||
|
// 통계
|
||||||
|
private int totalFiles = 0;
|
||||||
|
private long totalRecords = 0;
|
||||||
|
private long startTimeMs;
|
||||||
|
|
||||||
|
public MapIdSwitchingWriter(ExporterProperties properties, CoordinateReferenceSystem crs) {
|
||||||
|
this.properties = properties;
|
||||||
|
this.crs = crs;
|
||||||
|
}
|
||||||
|
|
||||||
|
@BeforeStep
|
||||||
|
public void beforeStep(StepExecution stepExecution) {
|
||||||
|
String geomTypeStr = stepExecution.getJobExecution()
|
||||||
|
.getExecutionContext()
|
||||||
|
.getString("geometryType", "Polygon");
|
||||||
|
|
||||||
|
Class<?> geomClass = resolveGeometryClass(geomTypeStr);
|
||||||
|
this.featureType = buildFeatureType(geomClass);
|
||||||
|
this.featureBuilder = new SimpleFeatureBuilder(featureType);
|
||||||
|
this.startTimeMs = System.currentTimeMillis();
|
||||||
|
|
||||||
|
log.info("MapIdSwitchingWriter initialized. geometryType={}, outputBase={}/{}",
|
||||||
|
geomTypeStr, properties.getOutputBaseDir(), properties.getInferenceId());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void open(ExecutionContext executionContext) throws ItemStreamException {
|
||||||
|
// beforeStep에서 초기화 완료. 별도 작업 없음.
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void write(Chunk<? extends InferenceResult> chunk) throws Exception {
|
||||||
|
List<SimpleFeature> buffer = new ArrayList<>();
|
||||||
|
|
||||||
|
for (InferenceResult result : chunk) {
|
||||||
|
if (result.getGeometry() == null) continue;
|
||||||
|
|
||||||
|
String mapId = result.getMapId();
|
||||||
|
if (mapId == null) continue;
|
||||||
|
|
||||||
|
// map_id가 바뀌면 버퍼를 flush하고 파일을 전환합니다
|
||||||
|
if (!mapId.equals(currentMapId)) {
|
||||||
|
if (!buffer.isEmpty()) {
|
||||||
|
currentFeatureStore.addFeatures(new ListFeatureCollection(featureType, buffer));
|
||||||
|
buffer.clear();
|
||||||
|
}
|
||||||
|
if (currentMapId != null) {
|
||||||
|
commitAndClose(currentMapId);
|
||||||
|
}
|
||||||
|
openForMapId(mapId);
|
||||||
|
currentMapId = mapId;
|
||||||
|
totalFiles++;
|
||||||
|
|
||||||
|
if (totalFiles % LOG_INTERVAL == 0) {
|
||||||
|
long elapsed = Math.max((System.currentTimeMillis() - startTimeMs) / 1000, 1);
|
||||||
|
log.info("진행: map_id {}개 완료 | 총 {}건 | 경과 {}s | 속도 {}건/s",
|
||||||
|
totalFiles, totalRecords, elapsed, totalRecords / elapsed);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
buffer.add(buildFeature(result));
|
||||||
|
totalRecords++;
|
||||||
|
}
|
||||||
|
|
||||||
|
// chunk 끝 - 마지막 map_id 구간 flush
|
||||||
|
if (!buffer.isEmpty()) {
|
||||||
|
currentFeatureStore.addFeatures(new ListFeatureCollection(featureType, buffer));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@AfterStep
|
||||||
|
public ExitStatus afterStep(StepExecution stepExecution) {
|
||||||
|
if (currentMapId != null) {
|
||||||
|
commitAndClose(currentMapId);
|
||||||
|
currentMapId = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
long elapsed = Math.max((System.currentTimeMillis() - startTimeMs) / 1000, 1);
|
||||||
|
log.info("=== 완료: map_id {}개, 총 {}건, 소요 {}s, 평균 {}건/s ===",
|
||||||
|
totalFiles, totalRecords, elapsed, totalRecords / elapsed);
|
||||||
|
|
||||||
|
return ExitStatus.COMPLETED;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void close() throws ItemStreamException {
|
||||||
|
// afterStep에서 처리. 혹시 남은 경우 안전하게 정리.
|
||||||
|
if (currentMapId != null) {
|
||||||
|
commitAndClose(currentMapId);
|
||||||
|
currentMapId = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void update(ExecutionContext executionContext) throws ItemStreamException {
|
||||||
|
executionContext.putLong("totalRecords", totalRecords);
|
||||||
|
executionContext.putInt("totalFiles", totalFiles);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── private helpers ───────────────────────────────────────────
|
||||||
|
|
||||||
|
private void openForMapId(String mapId) throws IOException {
|
||||||
|
Path dir = Paths.get(properties.getOutputBaseDir(), properties.getInferenceId(), mapId);
|
||||||
|
Files.createDirectories(dir);
|
||||||
|
File shpFile = dir.resolve(mapId + ".shp").toFile();
|
||||||
|
|
||||||
|
Map<String, Serializable> params = new HashMap<>();
|
||||||
|
params.put("url", shpFile.toURI().toURL());
|
||||||
|
params.put("create spatial index", Boolean.FALSE);
|
||||||
|
|
||||||
|
currentDataStore = (ShapefileDataStore) dsFactory.createNewDataStore(params);
|
||||||
|
currentDataStore.createSchema(featureType);
|
||||||
|
|
||||||
|
currentTransaction = new DefaultTransaction("create-" + mapId);
|
||||||
|
String typeName = currentDataStore.getTypeNames()[0];
|
||||||
|
currentFeatureStore = (SimpleFeatureStore) currentDataStore.getFeatureSource(typeName);
|
||||||
|
currentFeatureStore.setTransaction(currentTransaction);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void commitAndClose(String mapId) {
|
||||||
|
try {
|
||||||
|
currentTransaction.commit();
|
||||||
|
} catch (IOException e) {
|
||||||
|
log.error("[{}] commit 실패, rollback 시도", mapId, e);
|
||||||
|
try {
|
||||||
|
currentTransaction.rollback();
|
||||||
|
} catch (IOException ignored) {}
|
||||||
|
} finally {
|
||||||
|
try { currentTransaction.close(); } catch (IOException ignored) {}
|
||||||
|
currentDataStore.dispose();
|
||||||
|
currentTransaction = null;
|
||||||
|
currentDataStore = null;
|
||||||
|
currentFeatureStore = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private SimpleFeature buildFeature(InferenceResult r) {
|
||||||
|
featureBuilder.add(r.getGeometry());
|
||||||
|
featureBuilder.add(r.getUid());
|
||||||
|
featureBuilder.add(r.getMapId());
|
||||||
|
featureBuilder.add(r.getProbability() != null ? String.valueOf(r.getProbability()) : "0.0");
|
||||||
|
featureBuilder.add(r.getBeforeYear() != null ? r.getBeforeYear() : 0L);
|
||||||
|
featureBuilder.add(r.getAfterYear() != null ? r.getAfterYear() : 0L);
|
||||||
|
featureBuilder.add(r.getBeforeC());
|
||||||
|
featureBuilder.add(r.getBeforeP() != null ? String.valueOf(r.getBeforeP()) : "0.0");
|
||||||
|
featureBuilder.add(r.getAfterC());
|
||||||
|
featureBuilder.add(r.getAfterP() != null ? String.valueOf(r.getAfterP()) : "0.0");
|
||||||
|
return featureBuilder.buildFeature(null);
|
||||||
|
}
|
||||||
|
|
||||||
|
private SimpleFeatureType buildFeatureType(Class<?> geomClass) {
|
||||||
|
SimpleFeatureTypeBuilder builder = new SimpleFeatureTypeBuilder();
|
||||||
|
builder.setName("inference_results");
|
||||||
|
builder.setCRS(crs);
|
||||||
|
builder.add("the_geom", geomClass);
|
||||||
|
builder.setDefaultGeometry("the_geom");
|
||||||
|
builder.add("uid", String.class);
|
||||||
|
builder.add("map_id", String.class);
|
||||||
|
builder.add("chn_dtct_p", String.class);
|
||||||
|
builder.add("cprs_yr", Long.class);
|
||||||
|
builder.add("crtr_yr", Long.class);
|
||||||
|
builder.add("bf_cls_cd", String.class);
|
||||||
|
builder.add("bf_cls_pro", String.class);
|
||||||
|
builder.add("af_cls_cd", String.class);
|
||||||
|
builder.add("af_cls_pro", String.class);
|
||||||
|
return builder.buildFeatureType();
|
||||||
|
}
|
||||||
|
|
||||||
|
private Class<?> resolveGeometryClass(String typeStr) {
|
||||||
|
try {
|
||||||
|
String name = typeStr.replace("ST_", "");
|
||||||
|
return Class.forName("org.locationtech.jts.geom." + name);
|
||||||
|
} catch (ClassNotFoundException e) {
|
||||||
|
log.warn("알 수 없는 geometry type '{}', Polygon 사용", typeStr);
|
||||||
|
return Polygon.class;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,39 @@
|
|||||||
|
package com.kamco.shpexporter.config;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
|
@Component
|
||||||
|
@ConfigurationProperties(prefix = "exporter")
|
||||||
|
public class ExporterProperties {
|
||||||
|
|
||||||
|
private String inferenceId;
|
||||||
|
private List<Long> batchIds;
|
||||||
|
private String outputBaseDir;
|
||||||
|
private String crs = "EPSG:5186";
|
||||||
|
private int chunkSize = 1000;
|
||||||
|
private int fetchSize = 1000;
|
||||||
|
private int skipLimit = 100;
|
||||||
|
|
||||||
|
public String getInferenceId() { return inferenceId; }
|
||||||
|
public void setInferenceId(String inferenceId) { this.inferenceId = inferenceId; }
|
||||||
|
|
||||||
|
public List<Long> getBatchIds() { return batchIds; }
|
||||||
|
public void setBatchIds(List<Long> batchIds) { this.batchIds = batchIds; }
|
||||||
|
|
||||||
|
public String getOutputBaseDir() { return outputBaseDir; }
|
||||||
|
public void setOutputBaseDir(String outputBaseDir) { this.outputBaseDir = outputBaseDir; }
|
||||||
|
|
||||||
|
public String getCrs() { return crs; }
|
||||||
|
public void setCrs(String crs) { this.crs = crs; }
|
||||||
|
|
||||||
|
public int getChunkSize() { return chunkSize; }
|
||||||
|
public void setChunkSize(int chunkSize) { this.chunkSize = chunkSize; }
|
||||||
|
|
||||||
|
public int getFetchSize() { return fetchSize; }
|
||||||
|
public void setFetchSize(int fetchSize) { this.fetchSize = fetchSize; }
|
||||||
|
|
||||||
|
public int getSkipLimit() { return skipLimit; }
|
||||||
|
public void setSkipLimit(int skipLimit) { this.skipLimit = skipLimit; }
|
||||||
|
}
|
||||||
@@ -0,0 +1,47 @@
|
|||||||
|
package com.kamco.shpexporter.model;
|
||||||
|
|
||||||
|
import org.locationtech.jts.geom.Geometry;
|
||||||
|
|
||||||
|
public class InferenceResult {
|
||||||
|
|
||||||
|
private String uid;
|
||||||
|
private String mapId;
|
||||||
|
private Double probability;
|
||||||
|
private Long beforeYear;
|
||||||
|
private Long afterYear;
|
||||||
|
private String beforeC;
|
||||||
|
private Double beforeP;
|
||||||
|
private String afterC;
|
||||||
|
private Double afterP;
|
||||||
|
private Geometry geometry;
|
||||||
|
|
||||||
|
public String getUid() { return uid; }
|
||||||
|
public void setUid(String uid) { this.uid = uid; }
|
||||||
|
|
||||||
|
public String getMapId() { return mapId; }
|
||||||
|
public void setMapId(String mapId) { this.mapId = mapId; }
|
||||||
|
|
||||||
|
public Double getProbability() { return probability; }
|
||||||
|
public void setProbability(Double probability) { this.probability = probability; }
|
||||||
|
|
||||||
|
public Long getBeforeYear() { return beforeYear; }
|
||||||
|
public void setBeforeYear(Long beforeYear) { this.beforeYear = beforeYear; }
|
||||||
|
|
||||||
|
public Long getAfterYear() { return afterYear; }
|
||||||
|
public void setAfterYear(Long afterYear) { this.afterYear = afterYear; }
|
||||||
|
|
||||||
|
public String getBeforeC() { return beforeC; }
|
||||||
|
public void setBeforeC(String beforeC) { this.beforeC = beforeC; }
|
||||||
|
|
||||||
|
public Double getBeforeP() { return beforeP; }
|
||||||
|
public void setBeforeP(Double beforeP) { this.beforeP = beforeP; }
|
||||||
|
|
||||||
|
public String getAfterC() { return afterC; }
|
||||||
|
public void setAfterC(String afterC) { this.afterC = afterC; }
|
||||||
|
|
||||||
|
public Double getAfterP() { return afterP; }
|
||||||
|
public void setAfterP(Double afterP) { this.afterP = afterP; }
|
||||||
|
|
||||||
|
public Geometry getGeometry() { return geometry; }
|
||||||
|
public void setGeometry(Geometry geometry) { this.geometry = geometry; }
|
||||||
|
}
|
||||||
@@ -0,0 +1,37 @@
|
|||||||
|
package com.kamco.shpexporter.service;
|
||||||
|
|
||||||
|
import org.locationtech.jts.geom.Geometry;
|
||||||
|
import org.locationtech.jts.geom.MultiPolygon;
|
||||||
|
import org.locationtech.jts.geom.Polygon;
|
||||||
|
import org.locationtech.jts.io.ParseException;
|
||||||
|
import org.locationtech.jts.io.WKTReader;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
|
@Component
|
||||||
|
public class GeometryConverter {
|
||||||
|
|
||||||
|
private static final Logger log = LoggerFactory.getLogger(GeometryConverter.class);
|
||||||
|
|
||||||
|
private final WKTReader wktReader = new WKTReader();
|
||||||
|
|
||||||
|
public Geometry convertWKTToJTS(String wkt) {
|
||||||
|
if (wkt == null || wkt.isBlank()) return null;
|
||||||
|
|
||||||
|
try {
|
||||||
|
Geometry geom = wktReader.read(wkt);
|
||||||
|
|
||||||
|
// MultiPolygon → 첫 번째 Polygon으로 자동 변환
|
||||||
|
if (geom instanceof MultiPolygon mp) {
|
||||||
|
if (mp.getNumGeometries() == 0) return null;
|
||||||
|
geom = (Polygon) mp.getGeometryN(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
return geom;
|
||||||
|
} catch (ParseException e) {
|
||||||
|
log.warn("WKT parse failed: {}", e.getMessage());
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
30
shp-exporter_v2/src/main/resources/application-prod.yml
Normal file
30
shp-exporter_v2/src/main/resources/application-prod.yml
Normal file
@@ -0,0 +1,30 @@
|
|||||||
|
spring:
|
||||||
|
datasource:
|
||||||
|
url: jdbc:postgresql://172.16.4.56:15432/kamco_cds
|
||||||
|
username: kamco_cds
|
||||||
|
password: kamco_cds_Q!W@E#R$
|
||||||
|
driver-class-name: org.postgresql.Driver
|
||||||
|
hikari:
|
||||||
|
maximum-pool-size: 5 # cursor 1개 + 여유분. 단일 스텝이므로 많이 필요 없음
|
||||||
|
connection-timeout: 30000
|
||||||
|
idle-timeout: 600000
|
||||||
|
max-lifetime: 1800000
|
||||||
|
|
||||||
|
exporter:
|
||||||
|
inference-id: 'D5E46F60FC40B1A8BE0CD1F3547AA6'
|
||||||
|
batch-ids:
|
||||||
|
- 252
|
||||||
|
- 253
|
||||||
|
- 257
|
||||||
|
output-base-dir: '/data/model_output/export/'
|
||||||
|
crs: 'EPSG:5186'
|
||||||
|
chunk-size: 1000
|
||||||
|
fetch-size: 1000
|
||||||
|
skip-limit: 100
|
||||||
|
|
||||||
|
logging:
|
||||||
|
level:
|
||||||
|
com.kamco.shpexporter: INFO
|
||||||
|
org.springframework: WARN
|
||||||
|
pattern:
|
||||||
|
console: '%d{yyyy-MM-dd HH:mm:ss} - %msg%n'
|
||||||
13
shp-exporter_v2/src/main/resources/application.yml
Normal file
13
shp-exporter_v2/src/main/resources/application.yml
Normal file
@@ -0,0 +1,13 @@
|
|||||||
|
spring:
|
||||||
|
application:
|
||||||
|
name: shp-exporter-v2
|
||||||
|
profiles:
|
||||||
|
active: prod
|
||||||
|
main:
|
||||||
|
web-application-type: none
|
||||||
|
batch:
|
||||||
|
job:
|
||||||
|
enabled: false
|
||||||
|
jdbc:
|
||||||
|
initialize-schema: always
|
||||||
|
table-prefix: BATCH_
|
||||||
Reference in New Issue
Block a user