This commit is contained in:
dean
2026-04-15 12:36:58 +09:00
parent b23c3e2689
commit e358d9def5
19 changed files with 1266 additions and 1 deletions

129
shp-exporter_v2/README.md Normal file
View File

@@ -0,0 +1,129 @@
# shp-exporter-v2
`inference_results_testing` 테이블에서 공간 데이터를 읽어 **map_id 단위로 Shapefile(.shp)을 생성**하는 Spring Batch 애플리케이션입니다.
---
## 요구사항
| 항목 | 버전 |
|------|------|
| Java | 21 |
| Gradle | Wrapper 사용 (별도 설치 불필요) |
| DB | PostgreSQL + PostGIS (`inference_results_testing` 테이블) |
---
## 설정
`src/main/resources/application-prod.yml` 에서 아래 항목을 환경에 맞게 수정합니다.
```yaml
spring:
datasource:
url: jdbc:postgresql://<HOST>:<PORT>/<DB>
username: <USERNAME>
password: <PASSWORD>
exporter:
inference-id: 'D5E46F60FC40B1A8BE0CD1F3547AA6' # 추출 대상 inference ID
batch-ids: # 추출 대상 batch_id 목록
- 252
- 253
- 257
output-base-dir: '/data/model_output/export/' # Shapefile 저장 경로 (디렉토리)
crs: 'EPSG:5186' # 출력 좌표계
chunk-size: 1000 # 청크 단위 (기본값 유지 권장)
fetch-size: 1000 # DB cursor fetch 크기
skip-limit: 100 # 오류 허용 건수 (초과 시 Job 실패)
```
---
## 빌드
```bash
./gradlew bootJar
```
빌드 결과물: `build/libs/shp-exporter-v2.jar`
---
## 실행
### 방법 1 — Gradle로 직접 실행 (개발/테스트)
```bash
./gradlew bootRun
```
### 방법 2 — JAR 실행 (운영)
```bash
java \
-Xmx128g -Xms8g \
-XX:+UseG1GC \
-XX:MaxGCPauseMillis=200 \
-XX:G1HeapRegionSize=16m \
-XX:+ParallelRefProcEnabled \
-jar build/libs/shp-exporter-v2.jar
```
> 대용량 데이터 처리를 위해 힙 메모리를 충분히 확보하는 것을 권장합니다.
### 방법 3 — 설정을 커맨드라인으로 오버라이드
```bash
java -jar build/libs/shp-exporter-v2.jar \
--exporter.batch-ids=252,253,257 \
--exporter.output-base-dir=/data/model_output/export/ \
--exporter.inference-id=D5E46F60FC40B1A8BE0CD1F3547AA6
```
---
## 출력 구조
```
{output-base-dir}/
{map_id}/
{map_id}.shp
{map_id}.dbf
{map_id}.shx
{map_id}.prj
```
---
## Job 처리 흐름
```
Step 1 (geomTypeStep)
└─ DB에서 geometry 타입 확인
Step 2 (generateMapIdFilesStep)
└─ inference_results_testing 조회 (batch_id 필터, ORDER BY map_id, uid)
└─ map_id가 바뀔 때마다 Shapefile 파일 분리 저장
```
**데이터 필터 조건 (고정)**
- `batch_id = ANY(batch-ids 목록)`
- geometry 타입: `ST_Polygon` 또는 `ST_MultiPolygon`
- SRID: `5186`
- 유효한 geometry (`ST_IsValid = true`)
- 좌표 범위: X `125000~530000`, Y `-600000~988000` (EPSG:5186 한반도 범위)
---
## 로그 확인
정상 실행 시 아래와 같은 로그가 출력됩니다.
```
=== shp-exporter-v2 시작 ===
inference-id : D5E46F60FC40B1A8BE0CD1F3547AA6
batch-ids : [252, 253, 257]
output : /data/model_output/export/
Job 완료: ExitStatus [COMPLETED]
```

View File

@@ -0,0 +1,67 @@
plugins {
id 'java'
id 'org.springframework.boot' version '3.5.7'
id 'io.spring.dependency-management' version '1.1.7'
}
group = 'com.kamco'
version = '2.0.0'
java {
toolchain {
languageVersion = JavaLanguageVersion.of(21)
}
}
repositories {
mavenCentral()
maven { url 'https://repo.osgeo.org/repository/release/' }
maven { url 'https://repo.osgeo.org/repository/geotools-releases/' }
}
ext {
geoToolsVersion = '30.0'
}
configurations.all {
exclude group: 'javax.media', module: 'jai_core'
}
bootJar {
archiveFileName = "shp-exporter-v2.jar"
}
jar {
enabled = false
}
dependencies {
implementation 'org.springframework.boot:spring-boot-starter'
implementation 'org.springframework.boot:spring-boot-starter-jdbc'
implementation 'org.springframework.boot:spring-boot-starter-batch'
implementation 'org.springframework.boot:spring-boot-starter-validation'
implementation 'org.postgresql:postgresql'
implementation 'com.zaxxer:HikariCP'
implementation 'net.postgis:postgis-jdbc:2.5.1'
implementation 'org.locationtech.jts:jts-core:1.19.0'
implementation "org.geotools:gt-shapefile:${geoToolsVersion}"
implementation "org.geotools:gt-referencing:${geoToolsVersion}"
implementation "org.geotools:gt-epsg-hsql:${geoToolsVersion}"
testImplementation 'org.springframework.boot:spring-boot-starter-test'
testRuntimeOnly 'org.junit.platform:junit-platform-launcher'
}
tasks.named('test') {
useJUnitPlatform()
}
bootRun {
jvmArgs = ['-Xmx128g', '-Xms8g', '-XX:+UseG1GC',
'-XX:MaxGCPauseMillis=200',
'-XX:G1HeapRegionSize=16m',
'-XX:+ParallelRefProcEnabled']
}

Binary file not shown.

View File

@@ -0,0 +1,8 @@
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.3-bin.zip
#distributionUrl=http\://172.16.4.56:18100/repository/gradle-distributions/gradle-8.14.3-bin.zip
networkTimeout=10000
validateDistributionUrl=true
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists

251
shp-exporter_v2/gradlew vendored Executable file
View File

@@ -0,0 +1,251 @@
#!/bin/sh
#
# Copyright © 2015-2021 the original authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# SPDX-License-Identifier: Apache-2.0
#
##############################################################################
#
# Gradle start up script for POSIX generated by Gradle.
#
# Important for running:
#
# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is
# noncompliant, but you have some other compliant shell such as ksh or
# bash, then to run this script, type that shell name before the whole
# command line, like:
#
# ksh Gradle
#
# Busybox and similar reduced shells will NOT work, because this script
# requires all of these POSIX shell features:
# * functions;
# * expansions «$var», «${var}», «${var:-default}», «${var+SET}»,
# «${var#prefix}», «${var%suffix}», and «$( cmd )»;
# * compound commands having a testable exit status, especially «case»;
# * various built-in commands including «command», «set», and «ulimit».
#
# Important for patching:
#
# (2) This script targets any POSIX shell, so it avoids extensions provided
# by Bash, Ksh, etc; in particular arrays are avoided.
#
# The "traditional" practice of packing multiple parameters into a
# space-separated string is a well documented source of bugs and security
# problems, so this is (mostly) avoided, by progressively accumulating
# options in "$@", and eventually passing that to Java.
#
# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS,
# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly;
# see the in-line comments for details.
#
# There are tweaks for specific operating systems such as AIX, CygWin,
# Darwin, MinGW, and NonStop.
#
# (3) This script is generated from the Groovy template
# https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
# within the Gradle project.
#
# You can find Gradle at https://github.com/gradle/gradle/.
#
##############################################################################
# Attempt to set APP_HOME
# Resolve links: $0 may be a link
app_path=$0
# Need this for daisy-chained symlinks.
while
APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path
[ -h "$app_path" ]
do
ls=$( ls -ld "$app_path" )
link=${ls#*' -> '}
case $link in #(
/*) app_path=$link ;; #(
*) app_path=$APP_HOME$link ;;
esac
done
# This is normally unused
# shellcheck disable=SC2034
APP_BASE_NAME=${0##*/}
# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036)
APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s\n' "$PWD" ) || exit
# Use the maximum available, or set MAX_FD != -1 to use that value.
MAX_FD=maximum
warn () {
echo "$*"
} >&2
die () {
echo
echo "$*"
echo
exit 1
} >&2
# OS specific support (must be 'true' or 'false').
cygwin=false
msys=false
darwin=false
nonstop=false
case "$( uname )" in #(
CYGWIN* ) cygwin=true ;; #(
Darwin* ) darwin=true ;; #(
MSYS* | MINGW* ) msys=true ;; #(
NONSTOP* ) nonstop=true ;;
esac
CLASSPATH="\\\"\\\""
# Determine the Java command to use to start the JVM.
if [ -n "$JAVA_HOME" ] ; then
if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
# IBM's JDK on AIX uses strange locations for the executables
JAVACMD=$JAVA_HOME/jre/sh/java
else
JAVACMD=$JAVA_HOME/bin/java
fi
if [ ! -x "$JAVACMD" ] ; then
die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
Please set the JAVA_HOME variable in your environment to match the
location of your Java installation."
fi
else
JAVACMD=java
if ! command -v java >/dev/null 2>&1
then
die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
Please set the JAVA_HOME variable in your environment to match the
location of your Java installation."
fi
fi
# Increase the maximum file descriptors if we can.
if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
case $MAX_FD in #(
max*)
# In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked.
# shellcheck disable=SC2039,SC3045
MAX_FD=$( ulimit -H -n ) ||
warn "Could not query maximum file descriptor limit"
esac
case $MAX_FD in #(
'' | soft) :;; #(
*)
# In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked.
# shellcheck disable=SC2039,SC3045
ulimit -n "$MAX_FD" ||
warn "Could not set maximum file descriptor limit to $MAX_FD"
esac
fi
# Collect all arguments for the java command, stacking in reverse order:
# * args from the command line
# * the main class name
# * -classpath
# * -D...appname settings
# * --module-path (only if needed)
# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables.
# For Cygwin or MSYS, switch paths to Windows format before running java
if "$cygwin" || "$msys" ; then
APP_HOME=$( cygpath --path --mixed "$APP_HOME" )
CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" )
JAVACMD=$( cygpath --unix "$JAVACMD" )
# Now convert the arguments - kludge to limit ourselves to /bin/sh
for arg do
if
case $arg in #(
-*) false ;; # don't mess with options #(
/?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath
[ -e "$t" ] ;; #(
*) false ;;
esac
then
arg=$( cygpath --path --ignore --mixed "$arg" )
fi
# Roll the args list around exactly as many times as the number of
# args, so each arg winds up back in the position where it started, but
# possibly modified.
#
# NB: a `for` loop captures its iteration list before it begins, so
# changing the positional parameters here affects neither the number of
# iterations, nor the values presented in `arg`.
shift # remove old arg
set -- "$@" "$arg" # push replacement arg
done
fi
# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
# Collect all arguments for the java command:
# * DEFAULT_JVM_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments,
# and any embedded shellness will be escaped.
# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be
# treated as '${Hostname}' itself on the command line.
set -- \
"-Dorg.gradle.appname=$APP_BASE_NAME" \
-classpath "$CLASSPATH" \
-jar "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" \
"$@"
# Stop when "xargs" is not available.
if ! command -v xargs >/dev/null 2>&1
then
die "xargs is not available"
fi
# Use "xargs" to parse quoted args.
#
# With -n1 it outputs one arg per line, with the quotes and backslashes removed.
#
# In Bash we could simply go:
#
# readarray ARGS < <( xargs -n1 <<<"$var" ) &&
# set -- "${ARGS[@]}" "$@"
#
# but POSIX shell has neither arrays nor command substitution, so instead we
# post-process each arg (as a line of input to sed) to backslash-escape any
# character that might be a shell metacharacter, then use eval to reverse
# that process (while maintaining the separation between arguments), and wrap
# the whole thing up as a single "set" statement.
#
# This will of course break if any of these variables contains a newline or
# an unmatched quote.
#
eval "set -- $(
printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" |
xargs -n1 |
sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' |
tr '\n' ' '
)" '"$@"'
exec "$JAVACMD" "$@"

94
shp-exporter_v2/gradlew.bat vendored Executable file
View File

@@ -0,0 +1,94 @@
@rem
@rem Copyright 2015 the original author or authors.
@rem
@rem Licensed under the Apache License, Version 2.0 (the "License");
@rem you may not use this file except in compliance with the License.
@rem You may obtain a copy of the License at
@rem
@rem https://www.apache.org/licenses/LICENSE-2.0
@rem
@rem Unless required by applicable law or agreed to in writing, software
@rem distributed under the License is distributed on an "AS IS" BASIS,
@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@rem See the License for the specific language governing permissions and
@rem limitations under the License.
@rem
@rem SPDX-License-Identifier: Apache-2.0
@rem
@if "%DEBUG%"=="" @echo off
@rem ##########################################################################
@rem
@rem Gradle startup script for Windows
@rem
@rem ##########################################################################
@rem Set local scope for the variables with windows NT shell
if "%OS%"=="Windows_NT" setlocal
set DIRNAME=%~dp0
if "%DIRNAME%"=="" set DIRNAME=.
@rem This is normally unused
set APP_BASE_NAME=%~n0
set APP_HOME=%DIRNAME%
@rem Resolve any "." and ".." in APP_HOME to make it shorter.
for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi
@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m"
@rem Find java.exe
if defined JAVA_HOME goto findJavaFromJavaHome
set JAVA_EXE=java.exe
%JAVA_EXE% -version >NUL 2>&1
if %ERRORLEVEL% equ 0 goto execute
echo. 1>&2
echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2
echo. 1>&2
echo Please set the JAVA_HOME variable in your environment to match the 1>&2
echo location of your Java installation. 1>&2
goto fail
:findJavaFromJavaHome
set JAVA_HOME=%JAVA_HOME:"=%
set JAVA_EXE=%JAVA_HOME%/bin/java.exe
if exist "%JAVA_EXE%" goto execute
echo. 1>&2
echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2
echo. 1>&2
echo Please set the JAVA_HOME variable in your environment to match the 1>&2
echo location of your Java installation. 1>&2
goto fail
:execute
@rem Setup the command line
set CLASSPATH=
@rem Execute Gradle
"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" -jar "%APP_HOME%\gradle\wrapper\gradle-wrapper.jar" %*
:end
@rem End local scope for the variables with windows NT shell
if %ERRORLEVEL% equ 0 goto mainEnd
:fail
rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
rem the _cmd.exe /c_ return code!
set EXIT_CODE=%ERRORLEVEL%
if %EXIT_CODE% equ 0 set EXIT_CODE=1
if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE%
exit /b %EXIT_CODE%
:mainEnd
if "%OS%"=="Windows_NT" endlocal
:omega

View File

@@ -0,0 +1 @@
rootProject.name = 'shp-exporter-v2'

View File

@@ -0,0 +1,50 @@
package com.kamco.shpexporter;
import com.kamco.shpexporter.config.ExporterProperties;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.JobParameters;
import org.springframework.batch.core.JobParametersBuilder;
import org.springframework.batch.core.launch.JobLauncher;
import org.springframework.boot.CommandLineRunner;
import org.springframework.stereotype.Component;
@Component
public class ExporterRunner implements CommandLineRunner {
private static final Logger log = LoggerFactory.getLogger(ExporterRunner.class);
private final JobLauncher jobLauncher;
private final Job shpExporterJob;
private final ExporterProperties properties;
public ExporterRunner(JobLauncher jobLauncher, Job shpExporterJob,
ExporterProperties properties) {
this.jobLauncher = jobLauncher;
this.shpExporterJob = shpExporterJob;
this.properties = properties;
}
@Override
public void run(String... args) throws Exception {
if (properties.getBatchIds() == null || properties.getBatchIds().isEmpty()) {
log.error("exporter.batch-ids 가 설정되지 않았습니다.");
System.exit(1);
}
log.info("=== shp-exporter-v2 시작 ===");
log.info("inference-id : {}", properties.getInferenceId());
log.info("batch-ids : {}", properties.getBatchIds());
log.info("output : {}", properties.getOutputBaseDir());
JobParameters params = new JobParametersBuilder()
.addString("inferenceId", properties.getInferenceId())
.addString("batchIds", properties.getBatchIds().toString())
.addLong("timestamp", System.currentTimeMillis())
.toJobParameters();
var result = jobLauncher.run(shpExporterJob, params);
log.info("Job 완료: {}", result.getExitStatus());
}
}

View File

@@ -0,0 +1,14 @@
package com.kamco.shpexporter;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
@SpringBootApplication
@EnableConfigurationProperties
public class ShpExporterApplication {
public static void main(String[] args) {
System.exit(SpringApplication.exit(SpringApplication.run(ShpExporterApplication.class, args)));
}
}

View File

@@ -0,0 +1,124 @@
package com.kamco.shpexporter.batch;
import com.kamco.shpexporter.batch.reader.GeometryConvertingRowMapper;
import com.kamco.shpexporter.batch.tasklet.GeomTypeTasklet;
import com.kamco.shpexporter.batch.writer.MapIdSwitchingWriter;
import com.kamco.shpexporter.config.ExporterProperties;
import com.kamco.shpexporter.model.InferenceResult;
import java.util.List;
import javax.sql.DataSource;
import org.geotools.api.referencing.FactoryException;
import org.geotools.api.referencing.crs.CoordinateReferenceSystem;
import org.geotools.referencing.CRS;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.job.builder.JobBuilder;
import org.springframework.batch.core.repository.JobRepository;
import org.springframework.batch.core.step.builder.StepBuilder;
import org.springframework.batch.item.database.JdbcCursorItemReader;
import org.springframework.batch.item.database.builder.JdbcCursorItemReaderBuilder;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.jdbc.core.PreparedStatementSetter;
import org.springframework.transaction.PlatformTransactionManager;
@Configuration
public class BatchJobConfig {
private final ExporterProperties properties;
public BatchJobConfig(ExporterProperties properties) {
this.properties = properties;
}
// ─── CRS Bean ──────────────────────────────────────────────────
@Bean
public CoordinateReferenceSystem coordinateReferenceSystem() throws FactoryException {
return CRS.decode(properties.getCrs());
}
// ─── Job ───────────────────────────────────────────────────────
@Bean
public Job shpExporterJob(
JobRepository jobRepository,
Step geomTypeStep,
Step generateMapIdFilesStep) {
return new JobBuilder("shpExporterJob", jobRepository)
.start(geomTypeStep)
.next(generateMapIdFilesStep)
.build();
}
// ─── Step 1: geometry 타입 확인 ────────────────────────────────
@Bean
public Step geomTypeStep(
JobRepository jobRepository,
PlatformTransactionManager transactionManager,
GeomTypeTasklet geomTypeTasklet) {
return new StepBuilder("geomTypeStep", jobRepository)
.tasklet(geomTypeTasklet, transactionManager)
.build();
}
// ─── Step 2: map_id별 shapefile 생성 ──────────────────────────
@Bean
public Step generateMapIdFilesStep(
JobRepository jobRepository,
PlatformTransactionManager transactionManager,
JdbcCursorItemReader<InferenceResult> inferenceResultReader,
MapIdSwitchingWriter mapIdSwitchingWriter) {
return new StepBuilder("generateMapIdFilesStep", jobRepository)
.<InferenceResult, InferenceResult>chunk(properties.getChunkSize(), transactionManager)
.reader(inferenceResultReader)
.writer(mapIdSwitchingWriter)
.faultTolerant()
.skipLimit(properties.getSkipLimit())
.skip(Exception.class)
.listener(mapIdSwitchingWriter) // @BeforeStep / @AfterStep 등록
.build();
}
// ─── Reader: ORDER BY map_id, uid (전체 스캔) ──────────────────
@Bean
public JdbcCursorItemReader<InferenceResult> inferenceResultReader(
DataSource dataSource,
GeometryConvertingRowMapper rowMapper) {
List<Long> batchIds = properties.getBatchIds();
String sql =
"SELECT uid, map_id, probability, before_year, after_year, "
+ " before_c, before_p, after_c, after_p, "
+ " ST_AsText(geometry) as geometry_wkt "
+ "FROM inference_results_testing "
+ "WHERE batch_id = ANY(?) "
+ " AND ST_GeometryType(geometry) IN ('ST_Polygon', 'ST_MultiPolygon') "
+ " AND ST_SRID(geometry) = 5186 "
+ " AND ST_IsValid(geometry) = true "
+ " AND ST_XMin(geometry) >= 125000 AND ST_XMax(geometry) <= 530000 "
+ " AND ST_YMin(geometry) >= -600000 AND ST_YMax(geometry) <= 988000 "
+ "ORDER BY map_id, uid";
PreparedStatementSetter pss = ps -> {
var arr = ps.getConnection().createArrayOf("bigint", batchIds.toArray());
ps.setArray(1, arr);
};
return new JdbcCursorItemReaderBuilder<InferenceResult>()
.name("inferenceResultReader")
.dataSource(dataSource)
.sql(sql)
.preparedStatementSetter(pss)
.rowMapper(rowMapper)
.fetchSize(properties.getFetchSize())
.build();
}
}

View File

@@ -0,0 +1,47 @@
package com.kamco.shpexporter.batch.reader;
import com.kamco.shpexporter.model.InferenceResult;
import com.kamco.shpexporter.service.GeometryConverter;
import java.sql.ResultSet;
import java.sql.SQLException;
import org.springframework.jdbc.core.RowMapper;
import org.springframework.stereotype.Component;
@Component
public class GeometryConvertingRowMapper implements RowMapper<InferenceResult> {
private final GeometryConverter geometryConverter;
public GeometryConvertingRowMapper(GeometryConverter geometryConverter) {
this.geometryConverter = geometryConverter;
}
@Override
public InferenceResult mapRow(ResultSet rs, int rowNum) throws SQLException {
InferenceResult r = new InferenceResult();
r.setUid(rs.getString("uid"));
r.setMapId(rs.getString("map_id"));
r.setProbability(getDoubleOrNull(rs, "probability"));
r.setBeforeYear(getLongOrNull(rs, "before_year"));
r.setAfterYear(getLongOrNull(rs, "after_year"));
r.setBeforeC(rs.getString("before_c"));
r.setBeforeP(getDoubleOrNull(rs, "before_p"));
r.setAfterC(rs.getString("after_c"));
r.setAfterP(getDoubleOrNull(rs, "after_p"));
String wkt = rs.getString("geometry_wkt");
if (wkt != null) r.setGeometry(geometryConverter.convertWKTToJTS(wkt));
return r;
}
private Long getLongOrNull(ResultSet rs, String col) throws SQLException {
long v = rs.getLong(col);
return rs.wasNull() ? null : v;
}
private Double getDoubleOrNull(ResultSet rs, String col) throws SQLException {
double v = rs.getDouble(col);
return rs.wasNull() ? null : v;
}
}

View File

@@ -0,0 +1,65 @@
package com.kamco.shpexporter.batch.tasklet;
import com.kamco.shpexporter.config.ExporterProperties;
import java.sql.Array;
import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.batch.core.StepContribution;
import org.springframework.batch.core.scope.context.ChunkContext;
import org.springframework.batch.core.step.tasklet.Tasklet;
import org.springframework.batch.repeat.RepeatStatus;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Component;
/**
* Step 1: geometry 타입 사전 확인 후 JobExecutionContext에 저장.
* MapIdSwitchingWriter가 이 값을 읽어 SimpleFeatureType을 구성합니다.
*/
@Component
public class GeomTypeTasklet implements Tasklet {
private static final Logger log = LoggerFactory.getLogger(GeomTypeTasklet.class);
private final JdbcTemplate jdbcTemplate;
private final ExporterProperties properties;
public GeomTypeTasklet(JdbcTemplate jdbcTemplate, ExporterProperties properties) {
this.jdbcTemplate = jdbcTemplate;
this.properties = properties;
}
@Override
public RepeatStatus execute(StepContribution contribution, ChunkContext chunkContext)
throws Exception {
List<Long> batchIds = properties.getBatchIds();
log.info("Validating geometry types for batch_ids: {}", batchIds);
List<String> geomTypes = jdbcTemplate.query(
con -> {
var ps = con.prepareStatement(
"SELECT DISTINCT ST_GeometryType(geometry) "
+ "FROM inference_results_testing "
+ "WHERE batch_id = ANY(?) AND geometry IS NOT NULL");
Array arr = con.createArrayOf("bigint", batchIds.toArray());
ps.setArray(1, arr);
return ps;
},
(rs, rowNum) -> rs.getString(1));
log.info("Detected geometry types: {}", geomTypes);
// MultiPolygon은 자동으로 Polygon으로 변환하므로 항상 Polygon으로 처리
String resolved = "Polygon";
log.info("Using geometry type: {}", resolved);
chunkContext.getStepContext()
.getStepExecution()
.getJobExecution()
.getExecutionContext()
.putString("geometryType", resolved);
return RepeatStatus.FINISHED;
}
}

View File

@@ -0,0 +1,244 @@
package com.kamco.shpexporter.batch.writer;
import com.kamco.shpexporter.config.ExporterProperties;
import com.kamco.shpexporter.model.InferenceResult;
import java.io.File;
import java.io.IOException;
import java.io.Serializable;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.geotools.api.data.SimpleFeatureStore;
import org.geotools.api.data.Transaction;
import org.geotools.api.feature.simple.SimpleFeature;
import org.geotools.api.feature.simple.SimpleFeatureType;
import org.geotools.api.referencing.crs.CoordinateReferenceSystem;
import org.geotools.data.DefaultTransaction;
import org.geotools.data.collection.ListFeatureCollection;
import org.geotools.data.shapefile.ShapefileDataStore;
import org.geotools.data.shapefile.ShapefileDataStoreFactory;
import org.geotools.feature.simple.SimpleFeatureBuilder;
import org.geotools.feature.simple.SimpleFeatureTypeBuilder;
import org.locationtech.jts.geom.Polygon;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.batch.core.ExitStatus;
import org.springframework.batch.core.StepExecution;
import org.springframework.batch.core.annotation.AfterStep;
import org.springframework.batch.core.annotation.BeforeStep;
import org.springframework.batch.item.Chunk;
import org.springframework.batch.item.ExecutionContext;
import org.springframework.batch.item.ItemStreamException;
import org.springframework.batch.item.ItemStreamWriter;
import org.springframework.stereotype.Component;
/**
* map_id별 Shapefile을 생성하는 Writer.
*
* <p>데이터를 map_id, uid 순서로 읽어 map_id가 바뀔 때마다 파일을 전환합니다.
* 파티셔닝 없이 단일 JDBC 커서로 전체 데이터를 스트리밍하며, 메모리에는
* chunk 1개(기본 1000건)만 유지합니다.
*
* <p>출력 경로: {output-base-dir}/{inference-id}/{map_id}/{map_id}.shp
*/
@Component
public class MapIdSwitchingWriter implements ItemStreamWriter<InferenceResult> {
private static final Logger log = LoggerFactory.getLogger(MapIdSwitchingWriter.class);
private static final int LOG_INTERVAL = 500; // 500 map_id마다 진행 로그
private final ExporterProperties properties;
private final CoordinateReferenceSystem crs;
private final ShapefileDataStoreFactory dsFactory = new ShapefileDataStoreFactory();
private SimpleFeatureType featureType;
private SimpleFeatureBuilder featureBuilder;
// 현재 열려 있는 파일 관련 상태
private String currentMapId = null;
private ShapefileDataStore currentDataStore;
private Transaction currentTransaction;
private SimpleFeatureStore currentFeatureStore;
// 통계
private int totalFiles = 0;
private long totalRecords = 0;
private long startTimeMs;
public MapIdSwitchingWriter(ExporterProperties properties, CoordinateReferenceSystem crs) {
this.properties = properties;
this.crs = crs;
}
@BeforeStep
public void beforeStep(StepExecution stepExecution) {
String geomTypeStr = stepExecution.getJobExecution()
.getExecutionContext()
.getString("geometryType", "Polygon");
Class<?> geomClass = resolveGeometryClass(geomTypeStr);
this.featureType = buildFeatureType(geomClass);
this.featureBuilder = new SimpleFeatureBuilder(featureType);
this.startTimeMs = System.currentTimeMillis();
log.info("MapIdSwitchingWriter initialized. geometryType={}, outputBase={}/{}",
geomTypeStr, properties.getOutputBaseDir(), properties.getInferenceId());
}
@Override
public void open(ExecutionContext executionContext) throws ItemStreamException {
// beforeStep에서 초기화 완료. 별도 작업 없음.
}
@Override
public void write(Chunk<? extends InferenceResult> chunk) throws Exception {
List<SimpleFeature> buffer = new ArrayList<>();
for (InferenceResult result : chunk) {
if (result.getGeometry() == null) continue;
String mapId = result.getMapId();
if (mapId == null) continue;
// map_id가 바뀌면 버퍼를 flush하고 파일을 전환합니다
if (!mapId.equals(currentMapId)) {
if (!buffer.isEmpty()) {
currentFeatureStore.addFeatures(new ListFeatureCollection(featureType, buffer));
buffer.clear();
}
if (currentMapId != null) {
commitAndClose(currentMapId);
}
openForMapId(mapId);
currentMapId = mapId;
totalFiles++;
if (totalFiles % LOG_INTERVAL == 0) {
long elapsed = Math.max((System.currentTimeMillis() - startTimeMs) / 1000, 1);
log.info("진행: map_id {}개 완료 | 총 {}건 | 경과 {}s | 속도 {}건/s",
totalFiles, totalRecords, elapsed, totalRecords / elapsed);
}
}
buffer.add(buildFeature(result));
totalRecords++;
}
// chunk 끝 - 마지막 map_id 구간 flush
if (!buffer.isEmpty()) {
currentFeatureStore.addFeatures(new ListFeatureCollection(featureType, buffer));
}
}
@AfterStep
public ExitStatus afterStep(StepExecution stepExecution) {
if (currentMapId != null) {
commitAndClose(currentMapId);
currentMapId = null;
}
long elapsed = Math.max((System.currentTimeMillis() - startTimeMs) / 1000, 1);
log.info("=== 완료: map_id {}개, 총 {}건, 소요 {}s, 평균 {}건/s ===",
totalFiles, totalRecords, elapsed, totalRecords / elapsed);
return ExitStatus.COMPLETED;
}
@Override
public void close() throws ItemStreamException {
// afterStep에서 처리. 혹시 남은 경우 안전하게 정리.
if (currentMapId != null) {
commitAndClose(currentMapId);
currentMapId = null;
}
}
@Override
public void update(ExecutionContext executionContext) throws ItemStreamException {
executionContext.putLong("totalRecords", totalRecords);
executionContext.putInt("totalFiles", totalFiles);
}
// ─── private helpers ───────────────────────────────────────────
private void openForMapId(String mapId) throws IOException {
Path dir = Paths.get(properties.getOutputBaseDir(), properties.getInferenceId(), mapId);
Files.createDirectories(dir);
File shpFile = dir.resolve(mapId + ".shp").toFile();
Map<String, Serializable> params = new HashMap<>();
params.put("url", shpFile.toURI().toURL());
params.put("create spatial index", Boolean.FALSE);
currentDataStore = (ShapefileDataStore) dsFactory.createNewDataStore(params);
currentDataStore.createSchema(featureType);
currentTransaction = new DefaultTransaction("create-" + mapId);
String typeName = currentDataStore.getTypeNames()[0];
currentFeatureStore = (SimpleFeatureStore) currentDataStore.getFeatureSource(typeName);
currentFeatureStore.setTransaction(currentTransaction);
}
private void commitAndClose(String mapId) {
try {
currentTransaction.commit();
} catch (IOException e) {
log.error("[{}] commit 실패, rollback 시도", mapId, e);
try {
currentTransaction.rollback();
} catch (IOException ignored) {}
} finally {
try { currentTransaction.close(); } catch (IOException ignored) {}
currentDataStore.dispose();
currentTransaction = null;
currentDataStore = null;
currentFeatureStore = null;
}
}
private SimpleFeature buildFeature(InferenceResult r) {
featureBuilder.add(r.getGeometry());
featureBuilder.add(r.getUid());
featureBuilder.add(r.getMapId());
featureBuilder.add(r.getProbability() != null ? String.valueOf(r.getProbability()) : "0.0");
featureBuilder.add(r.getBeforeYear() != null ? r.getBeforeYear() : 0L);
featureBuilder.add(r.getAfterYear() != null ? r.getAfterYear() : 0L);
featureBuilder.add(r.getBeforeC());
featureBuilder.add(r.getBeforeP() != null ? String.valueOf(r.getBeforeP()) : "0.0");
featureBuilder.add(r.getAfterC());
featureBuilder.add(r.getAfterP() != null ? String.valueOf(r.getAfterP()) : "0.0");
return featureBuilder.buildFeature(null);
}
private SimpleFeatureType buildFeatureType(Class<?> geomClass) {
SimpleFeatureTypeBuilder builder = new SimpleFeatureTypeBuilder();
builder.setName("inference_results");
builder.setCRS(crs);
builder.add("the_geom", geomClass);
builder.setDefaultGeometry("the_geom");
builder.add("uid", String.class);
builder.add("map_id", String.class);
builder.add("chn_dtct_p", String.class);
builder.add("cprs_yr", Long.class);
builder.add("crtr_yr", Long.class);
builder.add("bf_cls_cd", String.class);
builder.add("bf_cls_pro", String.class);
builder.add("af_cls_cd", String.class);
builder.add("af_cls_pro", String.class);
return builder.buildFeatureType();
}
private Class<?> resolveGeometryClass(String typeStr) {
try {
String name = typeStr.replace("ST_", "");
return Class.forName("org.locationtech.jts.geom." + name);
} catch (ClassNotFoundException e) {
log.warn("알 수 없는 geometry type '{}', Polygon 사용", typeStr);
return Polygon.class;
}
}
}

View File

@@ -0,0 +1,39 @@
package com.kamco.shpexporter.config;
import java.util.List;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
@Component
@ConfigurationProperties(prefix = "exporter")
public class ExporterProperties {
private String inferenceId;
private List<Long> batchIds;
private String outputBaseDir;
private String crs = "EPSG:5186";
private int chunkSize = 1000;
private int fetchSize = 1000;
private int skipLimit = 100;
public String getInferenceId() { return inferenceId; }
public void setInferenceId(String inferenceId) { this.inferenceId = inferenceId; }
public List<Long> getBatchIds() { return batchIds; }
public void setBatchIds(List<Long> batchIds) { this.batchIds = batchIds; }
public String getOutputBaseDir() { return outputBaseDir; }
public void setOutputBaseDir(String outputBaseDir) { this.outputBaseDir = outputBaseDir; }
public String getCrs() { return crs; }
public void setCrs(String crs) { this.crs = crs; }
public int getChunkSize() { return chunkSize; }
public void setChunkSize(int chunkSize) { this.chunkSize = chunkSize; }
public int getFetchSize() { return fetchSize; }
public void setFetchSize(int fetchSize) { this.fetchSize = fetchSize; }
public int getSkipLimit() { return skipLimit; }
public void setSkipLimit(int skipLimit) { this.skipLimit = skipLimit; }
}

View File

@@ -0,0 +1,47 @@
package com.kamco.shpexporter.model;
import org.locationtech.jts.geom.Geometry;
public class InferenceResult {
private String uid;
private String mapId;
private Double probability;
private Long beforeYear;
private Long afterYear;
private String beforeC;
private Double beforeP;
private String afterC;
private Double afterP;
private Geometry geometry;
public String getUid() { return uid; }
public void setUid(String uid) { this.uid = uid; }
public String getMapId() { return mapId; }
public void setMapId(String mapId) { this.mapId = mapId; }
public Double getProbability() { return probability; }
public void setProbability(Double probability) { this.probability = probability; }
public Long getBeforeYear() { return beforeYear; }
public void setBeforeYear(Long beforeYear) { this.beforeYear = beforeYear; }
public Long getAfterYear() { return afterYear; }
public void setAfterYear(Long afterYear) { this.afterYear = afterYear; }
public String getBeforeC() { return beforeC; }
public void setBeforeC(String beforeC) { this.beforeC = beforeC; }
public Double getBeforeP() { return beforeP; }
public void setBeforeP(Double beforeP) { this.beforeP = beforeP; }
public String getAfterC() { return afterC; }
public void setAfterC(String afterC) { this.afterC = afterC; }
public Double getAfterP() { return afterP; }
public void setAfterP(Double afterP) { this.afterP = afterP; }
public Geometry getGeometry() { return geometry; }
public void setGeometry(Geometry geometry) { this.geometry = geometry; }
}

View File

@@ -0,0 +1,37 @@
package com.kamco.shpexporter.service;
import org.locationtech.jts.geom.Geometry;
import org.locationtech.jts.geom.MultiPolygon;
import org.locationtech.jts.geom.Polygon;
import org.locationtech.jts.io.ParseException;
import org.locationtech.jts.io.WKTReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
@Component
public class GeometryConverter {
private static final Logger log = LoggerFactory.getLogger(GeometryConverter.class);
private final WKTReader wktReader = new WKTReader();
public Geometry convertWKTToJTS(String wkt) {
if (wkt == null || wkt.isBlank()) return null;
try {
Geometry geom = wktReader.read(wkt);
// MultiPolygon → 첫 번째 Polygon으로 자동 변환
if (geom instanceof MultiPolygon mp) {
if (mp.getNumGeometries() == 0) return null;
geom = (Polygon) mp.getGeometryN(0);
}
return geom;
} catch (ParseException e) {
log.warn("WKT parse failed: {}", e.getMessage());
return null;
}
}
}

View File

@@ -0,0 +1,30 @@
spring:
datasource:
url: jdbc:postgresql://172.16.4.56:15432/kamco_cds
username: kamco_cds
password: kamco_cds_Q!W@E#R$
driver-class-name: org.postgresql.Driver
hikari:
maximum-pool-size: 5 # cursor 1개 + 여유분. 단일 스텝이므로 많이 필요 없음
connection-timeout: 30000
idle-timeout: 600000
max-lifetime: 1800000
exporter:
inference-id: 'D5E46F60FC40B1A8BE0CD1F3547AA6'
batch-ids:
- 252
- 253
- 257
output-base-dir: '/data/model_output/export/'
crs: 'EPSG:5186'
chunk-size: 1000
fetch-size: 1000
skip-limit: 100
logging:
level:
com.kamco.shpexporter: INFO
org.springframework: WARN
pattern:
console: '%d{yyyy-MM-dd HH:mm:ss} - %msg%n'

View File

@@ -0,0 +1,13 @@
spring:
application:
name: shp-exporter-v2
profiles:
active: prod
main:
web-application-type: none
batch:
job:
enabled: false
jdbc:
initialize-schema: always
table-prefix: BATCH_