Compare commits
9 Commits
4fcc645f63
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
| b95f5007f6 | |||
| 916af9c268 | |||
| e6b3b1d8cf | |||
| c1430da2f9 | |||
| 5d6e2fc860 | |||
|
|
c7b37b23d0 | ||
|
|
e358d9def5 | ||
|
|
b23c3e2689 | ||
|
|
0f7d794a38 |
@@ -19,21 +19,21 @@ configurations {
|
||||
}
|
||||
}
|
||||
|
||||
//repositories {
|
||||
// mavenCentral()
|
||||
//}
|
||||
|
||||
def repoUrl = System.getProperty("org.gradle.project.repoUrl")
|
||||
?: System.getenv("ORG_GRADLE_PROJECT_repoUrl")
|
||||
?: "http://172.16.4.56:18100/repository/maven-public/"
|
||||
|
||||
repositories {
|
||||
maven {
|
||||
url = uri(repoUrl)
|
||||
allowInsecureProtocol = true
|
||||
}
|
||||
mavenCentral()
|
||||
}
|
||||
|
||||
//def repoUrl = System.getProperty("org.gradle.project.repoUrl")
|
||||
// ?: System.getenv("ORG_GRADLE_PROJECT_repoUrl")
|
||||
// ?: "http://172.16.4.56:18100/repository/maven-public/"
|
||||
|
||||
//repositories {
|
||||
// maven {
|
||||
// url = uri(repoUrl)
|
||||
// allowInsecureProtocol = true
|
||||
// }
|
||||
//}
|
||||
|
||||
dependencies {
|
||||
implementation 'org.springframework.boot:spring-boot-starter-batch'
|
||||
implementation 'org.springframework.boot:spring-boot-starter-jdbc'
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
distributionBase=GRADLE_USER_HOME
|
||||
distributionPath=wrapper/dists
|
||||
#distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.3-bin.zip
|
||||
distributionUrl=http\://172.16.4.56:18100/repository/gradle-distributions/gradle-8.14.3-bin.zip
|
||||
distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.3-bin.zip
|
||||
#distributionUrl=http\://172.16.4.56:18100/repository/gradle-distributions/gradle-8.14.3-bin.zip
|
||||
networkTimeout=10000
|
||||
validateDistributionUrl=true
|
||||
zipStoreBase=GRADLE_USER_HOME
|
||||
|
||||
@@ -13,6 +13,7 @@ public class DockerProperties {
|
||||
private String image;
|
||||
private String user;
|
||||
private String datasetVolume;
|
||||
private String outputVolume;
|
||||
private String imagesVolume;
|
||||
private String inputRoot;
|
||||
private String outputRoot;
|
||||
@@ -20,4 +21,5 @@ public class DockerProperties {
|
||||
private int overlapPct;
|
||||
private List<String> trainValTestRatio;
|
||||
private double keepEmptyRatio;
|
||||
private double keepEmptyRatioVal;
|
||||
}
|
||||
|
||||
@@ -71,7 +71,7 @@ public class DockerRunnerService {
|
||||
log.info("[Step 1/4] Docker 명령어 생성 중...");
|
||||
List<String> command = buildCommand(resultUid);
|
||||
log.info("[Step 1/4] Docker 명령어 생성 완료");
|
||||
log.debug(" - 명령어: {}", String.join(" ", command));
|
||||
log.info(" - 명령어: {}", String.join(" ", command));
|
||||
|
||||
try {
|
||||
// Step 2: Docker 프로세스 시작
|
||||
@@ -161,11 +161,16 @@ public class DockerRunnerService {
|
||||
|
||||
// 볼륨 마운트 (호스트:컨테이너)
|
||||
cmd.add("-v");
|
||||
cmd.add(dockerProperties.getDatasetVolume()); // 예: "/kamco-nfs/dataset:/dataset"
|
||||
cmd.add(dockerProperties.getDatasetVolume());
|
||||
log.debug(" - Dataset Volume: {}", dockerProperties.getDatasetVolume());
|
||||
|
||||
//output 마운트
|
||||
cmd.add("-v");
|
||||
cmd.add(dockerProperties.getImagesVolume()); // 예: "/kamco-nfs/images:/images"
|
||||
cmd.add(dockerProperties.getOutputVolume());
|
||||
log.debug(" - Output Volume: {}", dockerProperties.getOutputVolume());
|
||||
|
||||
cmd.add("-v");
|
||||
cmd.add(dockerProperties.getImagesVolume());
|
||||
log.debug(" - Images Volume: {}", dockerProperties.getImagesVolume());
|
||||
|
||||
// Entrypoint 및 이미지
|
||||
@@ -179,11 +184,11 @@ public class DockerRunnerService {
|
||||
|
||||
// 입출력 폴더 설정
|
||||
cmd.add("--labelling-folder");
|
||||
cmd.add("request/" + resultUid);
|
||||
cmd.add(resultUid);
|
||||
log.debug(" - Labelling Folder: request/{}", resultUid);
|
||||
|
||||
cmd.add("--output-folder");
|
||||
cmd.add("response/" + resultUid);
|
||||
cmd.add(resultUid);
|
||||
log.debug(" - Output Folder: response/{}", resultUid);
|
||||
|
||||
// 파이프라인 파라미터
|
||||
@@ -209,6 +214,10 @@ public class DockerRunnerService {
|
||||
cmd.add(String.valueOf(dockerProperties.getKeepEmptyRatio()));
|
||||
log.debug(" - Keep Empty Ratio: {}", dockerProperties.getKeepEmptyRatio());
|
||||
|
||||
cmd.add("--keep_empty_ratio_val");
|
||||
cmd.add(String.valueOf(dockerProperties.getKeepEmptyRatioVal()));
|
||||
log.debug(" - Keep Empty Ratio Val: {}", dockerProperties.getKeepEmptyRatioVal());
|
||||
|
||||
log.debug("Docker 명령어 파라미터 구성 완료");
|
||||
return cmd;
|
||||
}
|
||||
|
||||
@@ -12,3 +12,19 @@ spring:
|
||||
|
||||
training-data:
|
||||
geojson-dir: /data/dataset
|
||||
docker:
|
||||
image: kamco-cd-dataset:latest
|
||||
user: "1000:1000"
|
||||
dataset-volume: /data/dataset/request:/data/labels:ro
|
||||
output-volume: /data/dataset/response:/data/out
|
||||
images-volume: /data/images:/kamco-nfs:ro
|
||||
input-root: /data/labels
|
||||
output-root: /data/out
|
||||
patch-size: 512
|
||||
overlap-pct: 50
|
||||
train-val-test-ratio:
|
||||
- "0.7"
|
||||
- "0.2"
|
||||
- "0.1"
|
||||
keep-empty-ratio: 0.1
|
||||
keep-empty-ratio-val: 0.3
|
||||
@@ -14,10 +14,13 @@ spring:
|
||||
training-data:
|
||||
geojson-dir: /data/dataset
|
||||
docker:
|
||||
dataset-volume: /data/dataset:/dataset
|
||||
images-volume: /data/images:/kamco-nfs:ro
|
||||
input-root: /dataset
|
||||
output-root: /dataset
|
||||
image: kamco-cd-dataset:latest
|
||||
user: "1000:1000"
|
||||
dataset-volume: /data/dataset/request:/data/labels:ro
|
||||
output-volume: /data/dataset/response:/data/out
|
||||
images-volume: /data/images:/data/images
|
||||
input-root: /data/labels
|
||||
output-root: /data/out
|
||||
patch-size: 512
|
||||
overlap-pct: 50
|
||||
train-val-test-ratio:
|
||||
@@ -25,3 +28,4 @@ training-data:
|
||||
- "0.2"
|
||||
- "0.1"
|
||||
keep-empty-ratio: 0.1
|
||||
keep-empty-ratio-val: 0.3
|
||||
@@ -28,10 +28,11 @@ training-data:
|
||||
docker:
|
||||
image: kamco-cd-dataset:latest
|
||||
user: "1000:1000"
|
||||
dataset-volume: /data/dataset:/dataset
|
||||
images-volume: /data/images:/kamco-nfs:ro
|
||||
input-root: /dataset
|
||||
output-root: /dataset
|
||||
dataset-volume: /data/dataset:/data/labels:ro
|
||||
output-volume: /data/dataset:/data/out
|
||||
images-volume: /data/images:/data/images
|
||||
input-root: /data/labels
|
||||
output-root: /data/out
|
||||
patch-size: 512
|
||||
overlap-pct: 50
|
||||
train-val-test-ratio:
|
||||
@@ -39,3 +40,4 @@ training-data:
|
||||
- "0.2"
|
||||
- "0.1"
|
||||
keep-empty-ratio: 0.1
|
||||
keep-empty-ratio-val: 0.3
|
||||
@@ -1,7 +1,12 @@
|
||||
{
|
||||
"permissions": {
|
||||
"allow": [
|
||||
"WebSearch"
|
||||
"WebSearch",
|
||||
"Bash(cp /Users/d-pn-0131/dev/kamco-cd-cron/shp-exporter/gradlew /Users/d-pn-0131/dev/kamco-cd-cron/shp-exporter_v2/gradlew)",
|
||||
"Bash(cp /Users/d-pn-0131/dev/kamco-cd-cron/shp-exporter/gradlew.bat /Users/d-pn-0131/dev/kamco-cd-cron/shp-exporter_v2/gradlew.bat)",
|
||||
"Bash(cp -r /Users/d-pn-0131/dev/kamco-cd-cron/shp-exporter/gradle /Users/d-pn-0131/dev/kamco-cd-cron/shp-exporter_v2/gradle)",
|
||||
"Read(//Users/d-pn-0131/dev/kamco-cd-cron/shp-exporter_v2/**)",
|
||||
"Bash(./gradlew clean *)"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,14 +4,14 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co
|
||||
|
||||
## Project Overview
|
||||
|
||||
Spring Boot 3.5.7 CLI application that converts PostgreSQL PostGIS spatial data to ESRI shapefiles and GeoJSON formats. The application uses **Spring Batch** for memory-efficient processing of large datasets (1M+ records) and supports automatic GeoServer layer registration via REST API.
|
||||
Spring Boot 3.5.7 / Java 21 CLI application that converts PostgreSQL PostGIS spatial data to ESRI shapefiles and GeoJSON formats. The application uses **Spring Batch** for memory-efficient processing of large datasets (1M+ records) and supports automatic GeoServer layer registration via REST API.
|
||||
|
||||
**Key Features**:
|
||||
- Memory-optimized batch processing (90-95% reduction: 2-13GB → 150-200MB)
|
||||
- Chunk-based streaming with cursor pagination (fetch-size: 1000)
|
||||
- Automatic geometry validation and type conversion (MultiPolygon → Polygon)
|
||||
- Coordinate system validation (EPSG:5186 Korean 2000 / Central Belt)
|
||||
- Dual execution modes: Spring Batch (recommended) and Legacy mode
|
||||
- Three execution modes: Spring Batch (recommended), Legacy, and GeoServer registration-only
|
||||
|
||||
## Build and Run Commands
|
||||
|
||||
@@ -25,6 +25,8 @@ Spring Boot 3.5.7 CLI application that converts PostgreSQL PostGIS spatial data
|
||||
|
||||
Output: `build/libs/shp-exporter.jar` (fixed name, no version suffix)
|
||||
|
||||
> **Note**: The `Dockerfile` currently references `shp-exporter-v2.jar` in its `COPY` step, which does not match the actual build output. Update the Dockerfile if building a Docker image.
|
||||
|
||||
### Run Application
|
||||
|
||||
#### Spring Batch Mode (Recommended)
|
||||
@@ -113,6 +115,7 @@ ConverterCommandLineRunner
|
||||
→ JdbcCursorItemReader (fetch-size: 1000)
|
||||
→ FeatureConversionProcessor (InferenceResult → SimpleFeature)
|
||||
→ StreamingShapefileWriter (chunk-based append)
|
||||
→ Step 2-1: PostShapefileUpdateTasklet (post-export DB UPDATE hook)
|
||||
→ Step 3: generateGeoJsonStep (chunk-oriented, same pattern)
|
||||
→ Step 4: CreateZipTasklet (creates .zip for GeoServer)
|
||||
→ Step 5: GeoServerRegistrationTasklet (conditional, if --geoserver.enabled=true)
|
||||
@@ -379,6 +382,21 @@ public Step myNewStep(JobRepository jobRepository,
|
||||
```
|
||||
4. **Always include `BatchExecutionHistoryListener`** to track execution metrics
|
||||
|
||||
### Post-Export DB Hook (`PostShapefileUpdateTasklet`)
|
||||
|
||||
`PostShapefileUpdateTasklet` runs immediately after `generateShapefileStep` and is designed as a placeholder for running UPDATE SQL after shapefile export (e.g., marking rows as exported). The SQL body is intentionally left as a `// TODO` — add your UPDATE statement inside `execute()`:
|
||||
|
||||
```java
|
||||
// batch/tasklet/PostShapefileUpdateTasklet.java
|
||||
int updated = jdbcTemplate.update(
|
||||
"UPDATE some_table SET status = 'EXPORTED' WHERE batch_id = ANY(?)",
|
||||
ps -> {
|
||||
ps.setArray(1, ps.getConnection().createArrayOf("bigint", batchIdList.toArray()));
|
||||
});
|
||||
```
|
||||
|
||||
Job parameters available: `inferenceId` (String), `batchIds` (comma-separated String → `List<Long>`).
|
||||
|
||||
### Modifying ItemReader Configuration
|
||||
|
||||
ItemReaders are **not thread-safe**. Each step requires its own instance:
|
||||
|
||||
@@ -30,7 +30,7 @@ ENV GEOSERVER_USERNAME=""
|
||||
ENV GEOSERVER_PASSWORD=""
|
||||
|
||||
ENTRYPOINT ["java", \
|
||||
"-Xmx4g", "-Xms512m", \
|
||||
"-Xmx128g", "-Xms8g", \
|
||||
"-XX:+UseG1GC", \
|
||||
"-XX:MaxGCPauseMillis=200", \
|
||||
"-XX:G1HeapRegionSize=16m", \
|
||||
|
||||
@@ -6,12 +6,15 @@ import com.kamco.makesample.batch.processor.FeatureConversionProcessor;
|
||||
import com.kamco.makesample.batch.tasklet.CreateZipTasklet;
|
||||
import com.kamco.makesample.batch.tasklet.GeoServerRegistrationTasklet;
|
||||
import com.kamco.makesample.batch.tasklet.GeometryTypeValidationTasklet;
|
||||
import com.kamco.makesample.batch.tasklet.PostShapefileUpdateTasklet;
|
||||
import com.kamco.makesample.batch.writer.MapIdGeoJsonWriter;
|
||||
import com.kamco.makesample.batch.writer.MapIdShapefileWriter;
|
||||
import com.kamco.makesample.batch.writer.StreamingGeoJsonWriter;
|
||||
import com.kamco.makesample.batch.writer.StreamingShapefileWriter;
|
||||
import com.kamco.makesample.model.InferenceResult;
|
||||
|
||||
import java.util.Arrays;
|
||||
|
||||
import org.geotools.api.feature.simple.SimpleFeature;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
@@ -52,33 +55,35 @@ public class MergedModeJobConfig {
|
||||
/**
|
||||
* MERGED 모드 Job 정의
|
||||
*
|
||||
* @param jobRepository JobRepository
|
||||
* @param jobRepository JobRepository
|
||||
* @param validateGeometryTypeStep Geometry type 검증 Step
|
||||
* @param generateShapefileStep Shapefile 생성 Step
|
||||
* @param generateGeoJsonStep GeoJSON 생성 Step
|
||||
* @param createZipStep ZIP 생성 Step
|
||||
* @param registerToGeoServerStep GeoServer 등록 Step (merge 폴더의 shapefile만)
|
||||
* @param generateMapIdFilesStep Map ID별 파일 생성 Step (병렬 처리)
|
||||
* @param generateShapefileStep Shapefile 생성 Step
|
||||
* @param generateGeoJsonStep GeoJSON 생성 Step
|
||||
* @param createZipStep ZIP 생성 Step
|
||||
* @param registerToGeoServerStep GeoServer 등록 Step (merge 폴더의 shapefile만)
|
||||
* @param generateMapIdFilesStep Map ID별 파일 생성 Step (병렬 처리)
|
||||
* @return Job
|
||||
*/
|
||||
@Bean
|
||||
public Job mergedModeJob(
|
||||
JobRepository jobRepository,
|
||||
Step validateGeometryTypeStep,
|
||||
Step generateShapefileStep,
|
||||
Step generateGeoJsonStep,
|
||||
Step createZipStep,
|
||||
Step registerToGeoServerStep,
|
||||
Step generateMapIdFilesStep) {
|
||||
JobRepository jobRepository,
|
||||
Step validateGeometryTypeStep,
|
||||
Step generateShapefileStep,
|
||||
Step postShapefileUpdateStep,
|
||||
Step generateGeoJsonStep,
|
||||
Step createZipStep,
|
||||
Step registerToGeoServerStep,
|
||||
Step generateMapIdFilesStep) {
|
||||
|
||||
return new JobBuilder("mergedModeJob", jobRepository)
|
||||
.start(validateGeometryTypeStep)
|
||||
.next(generateShapefileStep)
|
||||
.next(generateGeoJsonStep)
|
||||
.next(createZipStep)
|
||||
.next(registerToGeoServerStep) // Conditional execution
|
||||
.next(generateMapIdFilesStep) // Map ID별 개별 파일 생성
|
||||
.build();
|
||||
.start(validateGeometryTypeStep)
|
||||
.next(generateShapefileStep)
|
||||
.next(generateGeoJsonStep)
|
||||
.next(createZipStep)
|
||||
.next(registerToGeoServerStep) // Conditional execution
|
||||
.next(generateMapIdFilesStep) // Map ID별 개별 파일 생성
|
||||
.next(postShapefileUpdateStep) // Shapefile 생성 후 UPDATE 실행
|
||||
.build();
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -86,23 +91,23 @@ public class MergedModeJobConfig {
|
||||
*
|
||||
* <p>Shapefile은 homogeneous geometry type을 요구하므로 사전 검증
|
||||
*
|
||||
* @param jobRepository JobRepository
|
||||
* @param jobRepository JobRepository
|
||||
* @param transactionManager TransactionManager
|
||||
* @param validationTasklet GeometryTypeValidationTasklet
|
||||
* @param historyListener BatchExecutionHistoryListener
|
||||
* @param validationTasklet GeometryTypeValidationTasklet
|
||||
* @param historyListener BatchExecutionHistoryListener
|
||||
* @return Step
|
||||
*/
|
||||
@Bean
|
||||
public Step validateGeometryTypeStep(
|
||||
JobRepository jobRepository,
|
||||
PlatformTransactionManager transactionManager,
|
||||
GeometryTypeValidationTasklet validationTasklet,
|
||||
BatchExecutionHistoryListener historyListener) {
|
||||
JobRepository jobRepository,
|
||||
PlatformTransactionManager transactionManager,
|
||||
GeometryTypeValidationTasklet validationTasklet,
|
||||
BatchExecutionHistoryListener historyListener) {
|
||||
|
||||
return new StepBuilder("validateGeometryTypeStep", jobRepository)
|
||||
.tasklet(validationTasklet, transactionManager)
|
||||
.listener(historyListener)
|
||||
.build();
|
||||
.tasklet(validationTasklet, transactionManager)
|
||||
.listener(historyListener)
|
||||
.build();
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -116,32 +121,54 @@ public class MergedModeJobConfig {
|
||||
* <li>Writer: StreamingShapefileWriter (chunk 단위 쓰기)
|
||||
* </ul>
|
||||
*
|
||||
* @param jobRepository JobRepository
|
||||
* @param transactionManager TransactionManager
|
||||
* @param shapefileReader ItemReader (Shapefile용)
|
||||
* @param jobRepository JobRepository
|
||||
* @param transactionManager TransactionManager
|
||||
* @param shapefileReader ItemReader (Shapefile용)
|
||||
* @param featureConversionProcessor ItemProcessor
|
||||
* @param shapefileWriter ItemWriter
|
||||
* @param chunkSize Chunk size (default: 1000)
|
||||
* @param historyListener BatchExecutionHistoryListener
|
||||
* @param shapefileWriter ItemWriter
|
||||
* @param chunkSize Chunk size (default: 1000)
|
||||
* @param historyListener BatchExecutionHistoryListener
|
||||
* @return Step
|
||||
*/
|
||||
@Bean
|
||||
public Step generateShapefileStep(
|
||||
JobRepository jobRepository,
|
||||
PlatformTransactionManager transactionManager,
|
||||
JdbcCursorItemReader<InferenceResult> shapefileReader,
|
||||
FeatureConversionProcessor featureConversionProcessor,
|
||||
StreamingShapefileWriter shapefileWriter,
|
||||
@Value("${converter.batch.chunk-size:1000}") int chunkSize,
|
||||
BatchExecutionHistoryListener historyListener) {
|
||||
JobRepository jobRepository,
|
||||
PlatformTransactionManager transactionManager,
|
||||
JdbcCursorItemReader<InferenceResult> shapefileReader,
|
||||
FeatureConversionProcessor featureConversionProcessor,
|
||||
StreamingShapefileWriter shapefileWriter,
|
||||
@Value("${converter.batch.chunk-size:1000}") int chunkSize,
|
||||
BatchExecutionHistoryListener historyListener) {
|
||||
|
||||
return new StepBuilder("generateShapefileStep", jobRepository)
|
||||
.<InferenceResult, SimpleFeature>chunk(chunkSize, transactionManager)
|
||||
.reader(shapefileReader)
|
||||
.processor(featureConversionProcessor)
|
||||
.writer(shapefileWriter)
|
||||
.listener(historyListener)
|
||||
.build();
|
||||
.<InferenceResult, SimpleFeature>chunk(chunkSize, transactionManager)
|
||||
.reader(shapefileReader)
|
||||
.processor(featureConversionProcessor)
|
||||
.writer(shapefileWriter)
|
||||
.listener(historyListener)
|
||||
.build();
|
||||
}
|
||||
|
||||
/**
|
||||
* Step 2-1: Shapefile 생성 후 UPDATE 실행
|
||||
*
|
||||
* @param jobRepository JobRepository
|
||||
* @param transactionManager TransactionManager
|
||||
* @param postShapefileUpdateTasklet PostShapefileUpdateTasklet
|
||||
* @param historyListener BatchExecutionHistoryListener
|
||||
* @return Step
|
||||
*/
|
||||
@Bean
|
||||
public Step postShapefileUpdateStep(
|
||||
JobRepository jobRepository,
|
||||
PlatformTransactionManager transactionManager,
|
||||
PostShapefileUpdateTasklet postShapefileUpdateTasklet,
|
||||
BatchExecutionHistoryListener historyListener) {
|
||||
|
||||
return new StepBuilder("postShapefileUpdateStep", jobRepository)
|
||||
.tasklet(postShapefileUpdateTasklet, transactionManager)
|
||||
.listener(historyListener)
|
||||
.build();
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -149,54 +176,54 @@ public class MergedModeJobConfig {
|
||||
*
|
||||
* <p>Shapefile과 동일한 데이터를 GeoJSON 형식으로 출력
|
||||
*
|
||||
* @param jobRepository JobRepository
|
||||
* @param transactionManager TransactionManager
|
||||
* @param geoJsonReader ItemReader (GeoJSON용 - 별도 인스턴스)
|
||||
* @param jobRepository JobRepository
|
||||
* @param transactionManager TransactionManager
|
||||
* @param geoJsonReader ItemReader (GeoJSON용 - 별도 인스턴스)
|
||||
* @param featureConversionProcessor ItemProcessor (재사용)
|
||||
* @param geoJsonWriter ItemWriter
|
||||
* @param chunkSize Chunk size
|
||||
* @param historyListener BatchExecutionHistoryListener
|
||||
* @param geoJsonWriter ItemWriter
|
||||
* @param chunkSize Chunk size
|
||||
* @param historyListener BatchExecutionHistoryListener
|
||||
* @return Step
|
||||
*/
|
||||
@Bean
|
||||
public Step generateGeoJsonStep(
|
||||
JobRepository jobRepository,
|
||||
PlatformTransactionManager transactionManager,
|
||||
JdbcCursorItemReader<InferenceResult> geoJsonReader,
|
||||
FeatureConversionProcessor featureConversionProcessor,
|
||||
StreamingGeoJsonWriter geoJsonWriter,
|
||||
@Value("${converter.batch.chunk-size:1000}") int chunkSize,
|
||||
BatchExecutionHistoryListener historyListener) {
|
||||
JobRepository jobRepository,
|
||||
PlatformTransactionManager transactionManager,
|
||||
JdbcCursorItemReader<InferenceResult> geoJsonReader,
|
||||
FeatureConversionProcessor featureConversionProcessor,
|
||||
StreamingGeoJsonWriter geoJsonWriter,
|
||||
@Value("${converter.batch.chunk-size:1000}") int chunkSize,
|
||||
BatchExecutionHistoryListener historyListener) {
|
||||
|
||||
return new StepBuilder("generateGeoJsonStep", jobRepository)
|
||||
.<InferenceResult, SimpleFeature>chunk(chunkSize, transactionManager)
|
||||
.reader(geoJsonReader)
|
||||
.processor(featureConversionProcessor)
|
||||
.writer(geoJsonWriter)
|
||||
.listener(historyListener)
|
||||
.build();
|
||||
.<InferenceResult, SimpleFeature>chunk(chunkSize, transactionManager)
|
||||
.reader(geoJsonReader)
|
||||
.processor(featureConversionProcessor)
|
||||
.writer(geoJsonWriter)
|
||||
.listener(historyListener)
|
||||
.build();
|
||||
}
|
||||
|
||||
/**
|
||||
* Step 4: ZIP 파일 생성
|
||||
*
|
||||
* @param jobRepository JobRepository
|
||||
* @param jobRepository JobRepository
|
||||
* @param transactionManager TransactionManager
|
||||
* @param createZipTasklet CreateZipTasklet
|
||||
* @param historyListener BatchExecutionHistoryListener
|
||||
* @param createZipTasklet CreateZipTasklet
|
||||
* @param historyListener BatchExecutionHistoryListener
|
||||
* @return Step
|
||||
*/
|
||||
@Bean
|
||||
public Step createZipStep(
|
||||
JobRepository jobRepository,
|
||||
PlatformTransactionManager transactionManager,
|
||||
CreateZipTasklet createZipTasklet,
|
||||
BatchExecutionHistoryListener historyListener) {
|
||||
JobRepository jobRepository,
|
||||
PlatformTransactionManager transactionManager,
|
||||
CreateZipTasklet createZipTasklet,
|
||||
BatchExecutionHistoryListener historyListener) {
|
||||
|
||||
return new StepBuilder("createZipStep", jobRepository)
|
||||
.tasklet(createZipTasklet, transactionManager)
|
||||
.listener(historyListener)
|
||||
.build();
|
||||
.tasklet(createZipTasklet, transactionManager)
|
||||
.listener(historyListener)
|
||||
.build();
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -204,23 +231,23 @@ public class MergedModeJobConfig {
|
||||
*
|
||||
* <p>Conditional execution: geoserver.enabled=true일 때만 실행
|
||||
*
|
||||
* @param jobRepository JobRepository
|
||||
* @param transactionManager TransactionManager
|
||||
* @param jobRepository JobRepository
|
||||
* @param transactionManager TransactionManager
|
||||
* @param registrationTasklet GeoServerRegistrationTasklet
|
||||
* @param historyListener BatchExecutionHistoryListener
|
||||
* @param historyListener BatchExecutionHistoryListener
|
||||
* @return Step
|
||||
*/
|
||||
@Bean
|
||||
public Step registerToGeoServerStep(
|
||||
JobRepository jobRepository,
|
||||
PlatformTransactionManager transactionManager,
|
||||
GeoServerRegistrationTasklet registrationTasklet,
|
||||
BatchExecutionHistoryListener historyListener) {
|
||||
JobRepository jobRepository,
|
||||
PlatformTransactionManager transactionManager,
|
||||
GeoServerRegistrationTasklet registrationTasklet,
|
||||
BatchExecutionHistoryListener historyListener) {
|
||||
|
||||
return new StepBuilder("registerToGeoServerStep", jobRepository)
|
||||
.tasklet(registrationTasklet, transactionManager)
|
||||
.listener(historyListener)
|
||||
.build();
|
||||
.tasklet(registrationTasklet, transactionManager)
|
||||
.listener(historyListener)
|
||||
.build();
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -229,21 +256,21 @@ public class MergedModeJobConfig {
|
||||
* <p>각 map_id마다 개별 shapefile과 geojson 파일을 순차적으로 생성합니다. SyncTaskExecutor를 명시적으로 지정하여 병렬 실행을 방지하고
|
||||
* DB connection pool 고갈 방지
|
||||
*
|
||||
* @param jobRepository JobRepository
|
||||
* @param partitioner MapIdPartitioner
|
||||
* @param jobRepository JobRepository
|
||||
* @param partitioner MapIdPartitioner
|
||||
* @param mapIdWorkerStep Worker Step (각 파티션에서 실행)
|
||||
* @return Partitioned Step
|
||||
*/
|
||||
@Bean
|
||||
public Step generateMapIdFilesStep(
|
||||
JobRepository jobRepository, MapIdPartitioner partitioner, Step mapIdWorkerStep) {
|
||||
JobRepository jobRepository, MapIdPartitioner partitioner, Step mapIdWorkerStep) {
|
||||
|
||||
return new StepBuilder("generateMapIdFilesStep", jobRepository)
|
||||
.partitioner("mapIdWorker", partitioner)
|
||||
.step(mapIdWorkerStep)
|
||||
.taskExecutor(new SyncTaskExecutor()) // 명시적으로 순차 실행 지정
|
||||
.listener(partitioner) // Register partitioner as StepExecutionListener
|
||||
.build();
|
||||
.partitioner("mapIdWorker", partitioner)
|
||||
.step(mapIdWorkerStep)
|
||||
.taskExecutor(new SyncTaskExecutor()) // 명시적으로 순차 실행 지정
|
||||
.listener(partitioner) // Register partitioner as StepExecutionListener
|
||||
.build();
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -251,39 +278,39 @@ public class MergedModeJobConfig {
|
||||
*
|
||||
* <p>각 파티션에서 실행되며, 해당 map_id의 데이터를 읽어 shapefile과 geojson을 동시에 생성합니다.
|
||||
*
|
||||
* @param jobRepository JobRepository
|
||||
* @param transactionManager TransactionManager
|
||||
* @param mapIdModeReader ItemReader (map_id별)
|
||||
* @param jobRepository JobRepository
|
||||
* @param transactionManager TransactionManager
|
||||
* @param mapIdModeReader ItemReader (map_id별)
|
||||
* @param featureConversionProcessor ItemProcessor
|
||||
* @param mapIdShapefileWriter Shapefile Writer
|
||||
* @param mapIdGeoJsonWriter GeoJSON Writer
|
||||
* @param chunkSize Chunk size
|
||||
* @param historyListener BatchExecutionHistoryListener
|
||||
* @param mapIdShapefileWriter Shapefile Writer
|
||||
* @param mapIdGeoJsonWriter GeoJSON Writer
|
||||
* @param chunkSize Chunk size
|
||||
* @param historyListener BatchExecutionHistoryListener
|
||||
* @return Worker Step
|
||||
*/
|
||||
@Bean
|
||||
public Step mapIdWorkerStep(
|
||||
JobRepository jobRepository,
|
||||
PlatformTransactionManager transactionManager,
|
||||
JdbcCursorItemReader<InferenceResult> mapIdModeReader,
|
||||
FeatureConversionProcessor featureConversionProcessor,
|
||||
MapIdShapefileWriter mapIdShapefileWriter,
|
||||
MapIdGeoJsonWriter mapIdGeoJsonWriter,
|
||||
@Value("${converter.batch.chunk-size:1000}") int chunkSize,
|
||||
BatchExecutionHistoryListener historyListener) {
|
||||
JobRepository jobRepository,
|
||||
PlatformTransactionManager transactionManager,
|
||||
JdbcCursorItemReader<InferenceResult> mapIdModeReader,
|
||||
FeatureConversionProcessor featureConversionProcessor,
|
||||
MapIdShapefileWriter mapIdShapefileWriter,
|
||||
MapIdGeoJsonWriter mapIdGeoJsonWriter,
|
||||
@Value("${converter.batch.chunk-size:1000}") int chunkSize,
|
||||
BatchExecutionHistoryListener historyListener) {
|
||||
|
||||
// CompositeItemWriter로 shapefile과 geojson 동시 생성
|
||||
CompositeItemWriter<SimpleFeature> compositeWriter = new CompositeItemWriter<>();
|
||||
compositeWriter.setDelegates(Arrays.asList(mapIdShapefileWriter, mapIdGeoJsonWriter));
|
||||
|
||||
return new StepBuilder("mapIdWorkerStep", jobRepository)
|
||||
.<InferenceResult, SimpleFeature>chunk(chunkSize, transactionManager)
|
||||
.reader(mapIdModeReader)
|
||||
.processor(featureConversionProcessor)
|
||||
.writer(compositeWriter)
|
||||
.stream(mapIdShapefileWriter)
|
||||
.stream(mapIdGeoJsonWriter)
|
||||
.listener(historyListener)
|
||||
.build();
|
||||
.<InferenceResult, SimpleFeature>chunk(chunkSize, transactionManager)
|
||||
.reader(mapIdModeReader)
|
||||
.processor(featureConversionProcessor)
|
||||
.writer(compositeWriter)
|
||||
.stream(mapIdShapefileWriter)
|
||||
.stream(mapIdGeoJsonWriter)
|
||||
.listener(historyListener)
|
||||
.build();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -139,6 +139,8 @@ public class GeometryTypeValidationTasklet implements Tasklet {
|
||||
SELECT COUNT(*) as valid_count
|
||||
FROM inference_results_testing
|
||||
WHERE batch_id = ANY(?)
|
||||
AND after_c IS NOT NULL
|
||||
AND after_p IS NOT NULL
|
||||
AND geometry IS NOT NULL
|
||||
AND ST_GeometryType(geometry) IN ('ST_Polygon', 'ST_MultiPolygon')
|
||||
AND ST_SRID(geometry) = 5186
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
package com.kamco.makesample.batch.tasklet;
|
||||
|
||||
import java.util.List;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.batch.core.StepContribution;
|
||||
import org.springframework.batch.core.configuration.annotation.StepScope;
|
||||
import org.springframework.batch.core.scope.context.ChunkContext;
|
||||
import org.springframework.batch.core.step.tasklet.Tasklet;
|
||||
import org.springframework.batch.repeat.RepeatStatus;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* Shapefile 생성 완료 후 실행되는 UPDATE Tasklet
|
||||
*
|
||||
* <p>Job Flow 상 generateShapefileStep 이후에 실행됩니다.
|
||||
*
|
||||
* <p>실행할 SQL은 이 클래스의 execute() 메서드 안에 작성하세요.
|
||||
*/
|
||||
@Component
|
||||
@StepScope
|
||||
public class PostShapefileUpdateTasklet implements Tasklet {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(PostShapefileUpdateTasklet.class);
|
||||
|
||||
private final JdbcTemplate jdbcTemplate;
|
||||
|
||||
@Value("#{jobParameters['inferenceId']}")
|
||||
private String inferenceId;
|
||||
|
||||
@Value("#{jobParameters['batchIds']}")
|
||||
private String batchIds;
|
||||
|
||||
public PostShapefileUpdateTasklet(JdbcTemplate jdbcTemplate) {
|
||||
this.jdbcTemplate = jdbcTemplate;
|
||||
}
|
||||
|
||||
@Override
|
||||
public RepeatStatus execute(StepContribution contribution, ChunkContext chunkContext)
|
||||
throws Exception {
|
||||
|
||||
log.info(
|
||||
"Executing post-shapefile UPDATE for inferenceId={}, batchIds={}", inferenceId, batchIds);
|
||||
|
||||
List<Long> batchIdList =
|
||||
List.of(batchIds.split(",")).stream().map(Long::parseLong).toList();
|
||||
|
||||
// TODO: 실행할 UPDATE SQL을 여기에 작성하세요.
|
||||
// 예시:
|
||||
// int updated = jdbcTemplate.update(
|
||||
// "UPDATE some_table SET status = 'EXPORTED', inference_id = ? WHERE batch_id = ANY(?)",
|
||||
// ps -> {
|
||||
// ps.setString(1, inferenceId);
|
||||
// ps.setArray(2, ps.getConnection().createArrayOf("bigint", batchIdList.toArray()));
|
||||
// });
|
||||
// log.info("Updated {} rows", updated);
|
||||
|
||||
log.info("Post-shapefile UPDATE completed");
|
||||
|
||||
return RepeatStatus.FINISHED;
|
||||
}
|
||||
}
|
||||
@@ -10,13 +10,11 @@ import java.nio.file.Paths;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import org.geotools.api.data.SimpleFeatureStore;
|
||||
import org.geotools.api.data.FeatureWriter;
|
||||
import org.geotools.api.data.Transaction;
|
||||
import org.geotools.api.feature.simple.SimpleFeature;
|
||||
import org.geotools.api.feature.simple.SimpleFeatureType;
|
||||
import org.geotools.api.referencing.crs.CoordinateReferenceSystem;
|
||||
import org.geotools.data.DefaultTransaction;
|
||||
import org.geotools.data.collection.ListFeatureCollection;
|
||||
import org.geotools.data.shapefile.ShapefileDataStore;
|
||||
import org.geotools.data.shapefile.ShapefileDataStoreFactory;
|
||||
import org.slf4j.Logger;
|
||||
@@ -69,8 +67,7 @@ public class StreamingShapefileWriter implements ItemStreamWriter<SimpleFeature>
|
||||
private String outputPath;
|
||||
|
||||
private ShapefileDataStore dataStore;
|
||||
private Transaction transaction;
|
||||
private SimpleFeatureStore featureStore;
|
||||
private FeatureWriter<SimpleFeatureType, SimpleFeature> featureWriter;
|
||||
private SimpleFeatureType featureType;
|
||||
|
||||
private int chunkCount = 0;
|
||||
@@ -145,13 +142,9 @@ public class StreamingShapefileWriter implements ItemStreamWriter<SimpleFeature>
|
||||
dataStore = (ShapefileDataStore) factory.createNewDataStore(params);
|
||||
dataStore.createSchema(featureType);
|
||||
|
||||
// Transaction 시작
|
||||
transaction = new DefaultTransaction("create");
|
||||
|
||||
// FeatureStore 가져오기
|
||||
// FeatureWriter를 append 모드로 직접 열기 (Diff 누적 없이 파일에 직접 씀)
|
||||
String typeName = dataStore.getTypeNames()[0];
|
||||
featureStore = (SimpleFeatureStore) dataStore.getFeatureSource(typeName);
|
||||
featureStore.setTransaction(transaction);
|
||||
featureWriter = dataStore.getFeatureWriterAppend(typeName, Transaction.AUTO_COMMIT);
|
||||
|
||||
startTimeMs = System.currentTimeMillis();
|
||||
log.info("ShapefileDataStore initialized successfully");
|
||||
@@ -172,10 +165,13 @@ public class StreamingShapefileWriter implements ItemStreamWriter<SimpleFeature>
|
||||
int itemCount = items.size();
|
||||
totalRecordCount += itemCount;
|
||||
|
||||
// FeatureStore에 추가 - GeoTools ShapefileDataStore는 Diff 없이 파일에 직접 씀
|
||||
// 트랜잭션은 afterStep()에서 단일 커밋 (per-chunk 커밋 시 setTransaction()이 .shx 재스캔 → O(n²))
|
||||
ListFeatureCollection collection = new ListFeatureCollection(featureType, items);
|
||||
featureStore.addFeatures(collection);
|
||||
// FeatureWriter로 직접 append - Diff 누적 없이 O(1) per record
|
||||
for (SimpleFeature feature : items) {
|
||||
SimpleFeature newFeature = featureWriter.next();
|
||||
newFeature.setAttributes(feature.getAttributes());
|
||||
newFeature.setDefaultGeometry(feature.getDefaultGeometry());
|
||||
featureWriter.write();
|
||||
}
|
||||
|
||||
if (chunkCount % LOG_INTERVAL_CHUNKS == 0) {
|
||||
logProgress();
|
||||
@@ -191,15 +187,10 @@ public class StreamingShapefileWriter implements ItemStreamWriter<SimpleFeature>
|
||||
chunkCount);
|
||||
|
||||
try {
|
||||
if (transaction != null) {
|
||||
transaction.commit();
|
||||
log.info("Final transaction committed successfully");
|
||||
}
|
||||
} catch (IOException e) {
|
||||
log.error("Failed to commit final transaction", e);
|
||||
throw new ItemStreamException("Failed to commit shapefile transaction", e);
|
||||
} finally {
|
||||
cleanup();
|
||||
} catch (Exception e) {
|
||||
log.error("Failed to close shapefile writer", e);
|
||||
throw new ItemStreamException("Failed to close shapefile writer", e);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -212,23 +203,13 @@ public class StreamingShapefileWriter implements ItemStreamWriter<SimpleFeature>
|
||||
public void onError(Exception exception, Chunk<? extends SimpleFeature> chunk) {
|
||||
log.error("Error writing chunk #{}: {}", chunkCount, exception.getMessage(), exception);
|
||||
|
||||
try {
|
||||
if (transaction != null) {
|
||||
transaction.rollback();
|
||||
log.info("Transaction rolled back due to error");
|
||||
}
|
||||
cleanup();
|
||||
|
||||
// 부분 파일 삭제
|
||||
File shpFile = new File(outputPath);
|
||||
if (shpFile.exists()) {
|
||||
shpFile.delete();
|
||||
log.info("Deleted partial shapefile: {}", outputPath);
|
||||
}
|
||||
|
||||
} catch (IOException e) {
|
||||
log.error("Failed to rollback transaction", e);
|
||||
} finally {
|
||||
cleanup();
|
||||
// 부분 파일 삭제
|
||||
File shpFile = new File(outputPath);
|
||||
if (shpFile.exists()) {
|
||||
shpFile.delete();
|
||||
log.info("Deleted partial shapefile: {}", outputPath);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -264,13 +245,13 @@ public class StreamingShapefileWriter implements ItemStreamWriter<SimpleFeature>
|
||||
}
|
||||
|
||||
private void cleanup() {
|
||||
if (transaction != null) {
|
||||
if (featureWriter != null) {
|
||||
try {
|
||||
transaction.close();
|
||||
featureWriter.close();
|
||||
} catch (IOException e) {
|
||||
log.warn("Failed to close transaction", e);
|
||||
log.warn("Failed to close feature writer", e);
|
||||
}
|
||||
transaction = null;
|
||||
featureWriter = null;
|
||||
}
|
||||
|
||||
if (dataStore != null) {
|
||||
|
||||
129
shp-exporter_v2/README.md
Normal file
129
shp-exporter_v2/README.md
Normal file
@@ -0,0 +1,129 @@
|
||||
# shp-exporter-v2
|
||||
|
||||
`inference_results_testing` 테이블에서 공간 데이터를 읽어 **map_id 단위로 Shapefile(.shp)을 생성**하는 Spring Batch 애플리케이션입니다.
|
||||
|
||||
---
|
||||
|
||||
## 요구사항
|
||||
|
||||
| 항목 | 버전 |
|
||||
|------|------|
|
||||
| Java | 21 |
|
||||
| Gradle | Wrapper 사용 (별도 설치 불필요) |
|
||||
| DB | PostgreSQL + PostGIS (`inference_results_testing` 테이블) |
|
||||
|
||||
---
|
||||
|
||||
## 설정
|
||||
|
||||
`src/main/resources/application-prod.yml` 에서 아래 항목을 환경에 맞게 수정합니다.
|
||||
|
||||
```yaml
|
||||
spring:
|
||||
datasource:
|
||||
url: jdbc:postgresql://<HOST>:<PORT>/<DB>
|
||||
username: <USERNAME>
|
||||
password: <PASSWORD>
|
||||
|
||||
exporter:
|
||||
inference-id: 'D5E46F60FC40B1A8BE0CD1F3547AA6' # 추출 대상 inference ID
|
||||
batch-ids: # 추출 대상 batch_id 목록
|
||||
- 252
|
||||
- 253
|
||||
- 257
|
||||
output-base-dir: '/data/model_output/export/' # Shapefile 저장 경로 (디렉토리)
|
||||
crs: 'EPSG:5186' # 출력 좌표계
|
||||
chunk-size: 1000 # 청크 단위 (기본값 유지 권장)
|
||||
fetch-size: 1000 # DB cursor fetch 크기
|
||||
skip-limit: 100 # 오류 허용 건수 (초과 시 Job 실패)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 빌드
|
||||
|
||||
```bash
|
||||
./gradlew bootJar
|
||||
```
|
||||
|
||||
빌드 결과물: `build/libs/shp-exporter-v2.jar`
|
||||
|
||||
---
|
||||
|
||||
## 실행
|
||||
|
||||
### 방법 1 — Gradle로 직접 실행 (개발/테스트)
|
||||
|
||||
```bash
|
||||
./gradlew bootRun
|
||||
```
|
||||
|
||||
### 방법 2 — JAR 실행 (운영)
|
||||
|
||||
```bash
|
||||
java \
|
||||
-Xmx128g -Xms8g \
|
||||
-XX:+UseG1GC \
|
||||
-XX:MaxGCPauseMillis=200 \
|
||||
-XX:G1HeapRegionSize=16m \
|
||||
-XX:+ParallelRefProcEnabled \
|
||||
-jar build/libs/shp-exporter-v2.jar
|
||||
```
|
||||
|
||||
> 대용량 데이터 처리를 위해 힙 메모리를 충분히 확보하는 것을 권장합니다.
|
||||
|
||||
### 방법 3 — 설정을 커맨드라인으로 오버라이드
|
||||
|
||||
```bash
|
||||
java -jar build/libs/shp-exporter-v2.jar \
|
||||
--exporter.batch-ids=252,253,257 \
|
||||
--exporter.output-base-dir=/data/model_output/export/ \
|
||||
--exporter.inference-id=D5E46F60FC40B1A8BE0CD1F3547AA6
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 출력 구조
|
||||
|
||||
```
|
||||
{output-base-dir}/
|
||||
{map_id}/
|
||||
{map_id}.shp
|
||||
{map_id}.dbf
|
||||
{map_id}.shx
|
||||
{map_id}.prj
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Job 처리 흐름
|
||||
|
||||
```
|
||||
Step 1 (geomTypeStep)
|
||||
└─ DB에서 geometry 타입 확인
|
||||
|
||||
Step 2 (generateMapIdFilesStep)
|
||||
└─ inference_results_testing 조회 (batch_id 필터, ORDER BY map_id, uid)
|
||||
└─ map_id가 바뀔 때마다 Shapefile 파일 분리 저장
|
||||
```
|
||||
|
||||
**데이터 필터 조건 (고정)**
|
||||
- `batch_id = ANY(batch-ids 목록)`
|
||||
- geometry 타입: `ST_Polygon` 또는 `ST_MultiPolygon`
|
||||
- SRID: `5186`
|
||||
- 유효한 geometry (`ST_IsValid = true`)
|
||||
- 좌표 범위: X `125000~530000`, Y `-600000~988000` (EPSG:5186 한반도 범위)
|
||||
|
||||
---
|
||||
|
||||
## 로그 확인
|
||||
|
||||
정상 실행 시 아래와 같은 로그가 출력됩니다.
|
||||
|
||||
```
|
||||
=== shp-exporter-v2 시작 ===
|
||||
inference-id : D5E46F60FC40B1A8BE0CD1F3547AA6
|
||||
batch-ids : [252, 253, 257]
|
||||
output : /data/model_output/export/
|
||||
Job 완료: ExitStatus [COMPLETED]
|
||||
```
|
||||
67
shp-exporter_v2/build.gradle
Normal file
67
shp-exporter_v2/build.gradle
Normal file
@@ -0,0 +1,67 @@
|
||||
plugins {
|
||||
id 'java'
|
||||
id 'org.springframework.boot' version '3.5.7'
|
||||
id 'io.spring.dependency-management' version '1.1.7'
|
||||
}
|
||||
|
||||
group = 'com.kamco'
|
||||
version = '2.0.0'
|
||||
|
||||
java {
|
||||
toolchain {
|
||||
languageVersion = JavaLanguageVersion.of(21)
|
||||
}
|
||||
}
|
||||
|
||||
repositories {
|
||||
mavenCentral()
|
||||
maven { url 'https://repo.osgeo.org/repository/release/' }
|
||||
maven { url 'https://repo.osgeo.org/repository/geotools-releases/' }
|
||||
}
|
||||
|
||||
ext {
|
||||
geoToolsVersion = '30.0'
|
||||
}
|
||||
|
||||
configurations.all {
|
||||
exclude group: 'javax.media', module: 'jai_core'
|
||||
}
|
||||
|
||||
bootJar {
|
||||
archiveFileName = "shp-exporter-v2.jar"
|
||||
}
|
||||
|
||||
jar {
|
||||
enabled = false
|
||||
}
|
||||
|
||||
dependencies {
|
||||
implementation 'org.springframework.boot:spring-boot-starter'
|
||||
implementation 'org.springframework.boot:spring-boot-starter-jdbc'
|
||||
implementation 'org.springframework.boot:spring-boot-starter-batch'
|
||||
implementation 'org.springframework.boot:spring-boot-starter-validation'
|
||||
|
||||
implementation 'org.postgresql:postgresql'
|
||||
implementation 'com.zaxxer:HikariCP'
|
||||
implementation 'net.postgis:postgis-jdbc:2.5.1'
|
||||
|
||||
implementation 'org.locationtech.jts:jts-core:1.19.0'
|
||||
|
||||
implementation "org.geotools:gt-shapefile:${geoToolsVersion}"
|
||||
implementation "org.geotools:gt-referencing:${geoToolsVersion}"
|
||||
implementation "org.geotools:gt-epsg-hsql:${geoToolsVersion}"
|
||||
|
||||
testImplementation 'org.springframework.boot:spring-boot-starter-test'
|
||||
testRuntimeOnly 'org.junit.platform:junit-platform-launcher'
|
||||
}
|
||||
|
||||
tasks.named('test') {
|
||||
useJUnitPlatform()
|
||||
}
|
||||
|
||||
bootRun {
|
||||
jvmArgs = ['-Xmx128g', '-Xms8g', '-XX:+UseG1GC',
|
||||
'-XX:MaxGCPauseMillis=200',
|
||||
'-XX:G1HeapRegionSize=16m',
|
||||
'-XX:+ParallelRefProcEnabled']
|
||||
}
|
||||
BIN
shp-exporter_v2/gradle/wrapper/gradle-wrapper.jar
vendored
Executable file
BIN
shp-exporter_v2/gradle/wrapper/gradle-wrapper.jar
vendored
Executable file
Binary file not shown.
8
shp-exporter_v2/gradle/wrapper/gradle-wrapper.properties
vendored
Executable file
8
shp-exporter_v2/gradle/wrapper/gradle-wrapper.properties
vendored
Executable file
@@ -0,0 +1,8 @@
|
||||
distributionBase=GRADLE_USER_HOME
|
||||
distributionPath=wrapper/dists
|
||||
distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.3-bin.zip
|
||||
#distributionUrl=http\://172.16.4.56:18100/repository/gradle-distributions/gradle-8.14.3-bin.zip
|
||||
networkTimeout=10000
|
||||
validateDistributionUrl=true
|
||||
zipStoreBase=GRADLE_USER_HOME
|
||||
zipStorePath=wrapper/dists
|
||||
251
shp-exporter_v2/gradlew
vendored
Executable file
251
shp-exporter_v2/gradlew
vendored
Executable file
@@ -0,0 +1,251 @@
|
||||
#!/bin/sh
|
||||
|
||||
#
|
||||
# Copyright © 2015-2021 the original authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# https://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
|
||||
##############################################################################
|
||||
#
|
||||
# Gradle start up script for POSIX generated by Gradle.
|
||||
#
|
||||
# Important for running:
|
||||
#
|
||||
# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is
|
||||
# noncompliant, but you have some other compliant shell such as ksh or
|
||||
# bash, then to run this script, type that shell name before the whole
|
||||
# command line, like:
|
||||
#
|
||||
# ksh Gradle
|
||||
#
|
||||
# Busybox and similar reduced shells will NOT work, because this script
|
||||
# requires all of these POSIX shell features:
|
||||
# * functions;
|
||||
# * expansions «$var», «${var}», «${var:-default}», «${var+SET}»,
|
||||
# «${var#prefix}», «${var%suffix}», and «$( cmd )»;
|
||||
# * compound commands having a testable exit status, especially «case»;
|
||||
# * various built-in commands including «command», «set», and «ulimit».
|
||||
#
|
||||
# Important for patching:
|
||||
#
|
||||
# (2) This script targets any POSIX shell, so it avoids extensions provided
|
||||
# by Bash, Ksh, etc; in particular arrays are avoided.
|
||||
#
|
||||
# The "traditional" practice of packing multiple parameters into a
|
||||
# space-separated string is a well documented source of bugs and security
|
||||
# problems, so this is (mostly) avoided, by progressively accumulating
|
||||
# options in "$@", and eventually passing that to Java.
|
||||
#
|
||||
# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS,
|
||||
# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly;
|
||||
# see the in-line comments for details.
|
||||
#
|
||||
# There are tweaks for specific operating systems such as AIX, CygWin,
|
||||
# Darwin, MinGW, and NonStop.
|
||||
#
|
||||
# (3) This script is generated from the Groovy template
|
||||
# https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
|
||||
# within the Gradle project.
|
||||
#
|
||||
# You can find Gradle at https://github.com/gradle/gradle/.
|
||||
#
|
||||
##############################################################################
|
||||
|
||||
# Attempt to set APP_HOME
|
||||
|
||||
# Resolve links: $0 may be a link
|
||||
app_path=$0
|
||||
|
||||
# Need this for daisy-chained symlinks.
|
||||
while
|
||||
APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path
|
||||
[ -h "$app_path" ]
|
||||
do
|
||||
ls=$( ls -ld "$app_path" )
|
||||
link=${ls#*' -> '}
|
||||
case $link in #(
|
||||
/*) app_path=$link ;; #(
|
||||
*) app_path=$APP_HOME$link ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# This is normally unused
|
||||
# shellcheck disable=SC2034
|
||||
APP_BASE_NAME=${0##*/}
|
||||
# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036)
|
||||
APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s\n' "$PWD" ) || exit
|
||||
|
||||
# Use the maximum available, or set MAX_FD != -1 to use that value.
|
||||
MAX_FD=maximum
|
||||
|
||||
warn () {
|
||||
echo "$*"
|
||||
} >&2
|
||||
|
||||
die () {
|
||||
echo
|
||||
echo "$*"
|
||||
echo
|
||||
exit 1
|
||||
} >&2
|
||||
|
||||
# OS specific support (must be 'true' or 'false').
|
||||
cygwin=false
|
||||
msys=false
|
||||
darwin=false
|
||||
nonstop=false
|
||||
case "$( uname )" in #(
|
||||
CYGWIN* ) cygwin=true ;; #(
|
||||
Darwin* ) darwin=true ;; #(
|
||||
MSYS* | MINGW* ) msys=true ;; #(
|
||||
NONSTOP* ) nonstop=true ;;
|
||||
esac
|
||||
|
||||
CLASSPATH="\\\"\\\""
|
||||
|
||||
|
||||
# Determine the Java command to use to start the JVM.
|
||||
if [ -n "$JAVA_HOME" ] ; then
|
||||
if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
|
||||
# IBM's JDK on AIX uses strange locations for the executables
|
||||
JAVACMD=$JAVA_HOME/jre/sh/java
|
||||
else
|
||||
JAVACMD=$JAVA_HOME/bin/java
|
||||
fi
|
||||
if [ ! -x "$JAVACMD" ] ; then
|
||||
die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
|
||||
|
||||
Please set the JAVA_HOME variable in your environment to match the
|
||||
location of your Java installation."
|
||||
fi
|
||||
else
|
||||
JAVACMD=java
|
||||
if ! command -v java >/dev/null 2>&1
|
||||
then
|
||||
die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
|
||||
|
||||
Please set the JAVA_HOME variable in your environment to match the
|
||||
location of your Java installation."
|
||||
fi
|
||||
fi
|
||||
|
||||
# Increase the maximum file descriptors if we can.
|
||||
if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
|
||||
case $MAX_FD in #(
|
||||
max*)
|
||||
# In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked.
|
||||
# shellcheck disable=SC2039,SC3045
|
||||
MAX_FD=$( ulimit -H -n ) ||
|
||||
warn "Could not query maximum file descriptor limit"
|
||||
esac
|
||||
case $MAX_FD in #(
|
||||
'' | soft) :;; #(
|
||||
*)
|
||||
# In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked.
|
||||
# shellcheck disable=SC2039,SC3045
|
||||
ulimit -n "$MAX_FD" ||
|
||||
warn "Could not set maximum file descriptor limit to $MAX_FD"
|
||||
esac
|
||||
fi
|
||||
|
||||
# Collect all arguments for the java command, stacking in reverse order:
|
||||
# * args from the command line
|
||||
# * the main class name
|
||||
# * -classpath
|
||||
# * -D...appname settings
|
||||
# * --module-path (only if needed)
|
||||
# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables.
|
||||
|
||||
# For Cygwin or MSYS, switch paths to Windows format before running java
|
||||
if "$cygwin" || "$msys" ; then
|
||||
APP_HOME=$( cygpath --path --mixed "$APP_HOME" )
|
||||
CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" )
|
||||
|
||||
JAVACMD=$( cygpath --unix "$JAVACMD" )
|
||||
|
||||
# Now convert the arguments - kludge to limit ourselves to /bin/sh
|
||||
for arg do
|
||||
if
|
||||
case $arg in #(
|
||||
-*) false ;; # don't mess with options #(
|
||||
/?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath
|
||||
[ -e "$t" ] ;; #(
|
||||
*) false ;;
|
||||
esac
|
||||
then
|
||||
arg=$( cygpath --path --ignore --mixed "$arg" )
|
||||
fi
|
||||
# Roll the args list around exactly as many times as the number of
|
||||
# args, so each arg winds up back in the position where it started, but
|
||||
# possibly modified.
|
||||
#
|
||||
# NB: a `for` loop captures its iteration list before it begins, so
|
||||
# changing the positional parameters here affects neither the number of
|
||||
# iterations, nor the values presented in `arg`.
|
||||
shift # remove old arg
|
||||
set -- "$@" "$arg" # push replacement arg
|
||||
done
|
||||
fi
|
||||
|
||||
|
||||
# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
|
||||
DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
|
||||
|
||||
# Collect all arguments for the java command:
|
||||
# * DEFAULT_JVM_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments,
|
||||
# and any embedded shellness will be escaped.
|
||||
# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be
|
||||
# treated as '${Hostname}' itself on the command line.
|
||||
|
||||
set -- \
|
||||
"-Dorg.gradle.appname=$APP_BASE_NAME" \
|
||||
-classpath "$CLASSPATH" \
|
||||
-jar "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" \
|
||||
"$@"
|
||||
|
||||
# Stop when "xargs" is not available.
|
||||
if ! command -v xargs >/dev/null 2>&1
|
||||
then
|
||||
die "xargs is not available"
|
||||
fi
|
||||
|
||||
# Use "xargs" to parse quoted args.
|
||||
#
|
||||
# With -n1 it outputs one arg per line, with the quotes and backslashes removed.
|
||||
#
|
||||
# In Bash we could simply go:
|
||||
#
|
||||
# readarray ARGS < <( xargs -n1 <<<"$var" ) &&
|
||||
# set -- "${ARGS[@]}" "$@"
|
||||
#
|
||||
# but POSIX shell has neither arrays nor command substitution, so instead we
|
||||
# post-process each arg (as a line of input to sed) to backslash-escape any
|
||||
# character that might be a shell metacharacter, then use eval to reverse
|
||||
# that process (while maintaining the separation between arguments), and wrap
|
||||
# the whole thing up as a single "set" statement.
|
||||
#
|
||||
# This will of course break if any of these variables contains a newline or
|
||||
# an unmatched quote.
|
||||
#
|
||||
|
||||
eval "set -- $(
|
||||
printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" |
|
||||
xargs -n1 |
|
||||
sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' |
|
||||
tr '\n' ' '
|
||||
)" '"$@"'
|
||||
|
||||
exec "$JAVACMD" "$@"
|
||||
94
shp-exporter_v2/gradlew.bat
vendored
Executable file
94
shp-exporter_v2/gradlew.bat
vendored
Executable file
@@ -0,0 +1,94 @@
|
||||
@rem
|
||||
@rem Copyright 2015 the original author or authors.
|
||||
@rem
|
||||
@rem Licensed under the Apache License, Version 2.0 (the "License");
|
||||
@rem you may not use this file except in compliance with the License.
|
||||
@rem You may obtain a copy of the License at
|
||||
@rem
|
||||
@rem https://www.apache.org/licenses/LICENSE-2.0
|
||||
@rem
|
||||
@rem Unless required by applicable law or agreed to in writing, software
|
||||
@rem distributed under the License is distributed on an "AS IS" BASIS,
|
||||
@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
@rem See the License for the specific language governing permissions and
|
||||
@rem limitations under the License.
|
||||
@rem
|
||||
@rem SPDX-License-Identifier: Apache-2.0
|
||||
@rem
|
||||
|
||||
@if "%DEBUG%"=="" @echo off
|
||||
@rem ##########################################################################
|
||||
@rem
|
||||
@rem Gradle startup script for Windows
|
||||
@rem
|
||||
@rem ##########################################################################
|
||||
|
||||
@rem Set local scope for the variables with windows NT shell
|
||||
if "%OS%"=="Windows_NT" setlocal
|
||||
|
||||
set DIRNAME=%~dp0
|
||||
if "%DIRNAME%"=="" set DIRNAME=.
|
||||
@rem This is normally unused
|
||||
set APP_BASE_NAME=%~n0
|
||||
set APP_HOME=%DIRNAME%
|
||||
|
||||
@rem Resolve any "." and ".." in APP_HOME to make it shorter.
|
||||
for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi
|
||||
|
||||
@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
|
||||
set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m"
|
||||
|
||||
@rem Find java.exe
|
||||
if defined JAVA_HOME goto findJavaFromJavaHome
|
||||
|
||||
set JAVA_EXE=java.exe
|
||||
%JAVA_EXE% -version >NUL 2>&1
|
||||
if %ERRORLEVEL% equ 0 goto execute
|
||||
|
||||
echo. 1>&2
|
||||
echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2
|
||||
echo. 1>&2
|
||||
echo Please set the JAVA_HOME variable in your environment to match the 1>&2
|
||||
echo location of your Java installation. 1>&2
|
||||
|
||||
goto fail
|
||||
|
||||
:findJavaFromJavaHome
|
||||
set JAVA_HOME=%JAVA_HOME:"=%
|
||||
set JAVA_EXE=%JAVA_HOME%/bin/java.exe
|
||||
|
||||
if exist "%JAVA_EXE%" goto execute
|
||||
|
||||
echo. 1>&2
|
||||
echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2
|
||||
echo. 1>&2
|
||||
echo Please set the JAVA_HOME variable in your environment to match the 1>&2
|
||||
echo location of your Java installation. 1>&2
|
||||
|
||||
goto fail
|
||||
|
||||
:execute
|
||||
@rem Setup the command line
|
||||
|
||||
set CLASSPATH=
|
||||
|
||||
|
||||
@rem Execute Gradle
|
||||
"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" -jar "%APP_HOME%\gradle\wrapper\gradle-wrapper.jar" %*
|
||||
|
||||
:end
|
||||
@rem End local scope for the variables with windows NT shell
|
||||
if %ERRORLEVEL% equ 0 goto mainEnd
|
||||
|
||||
:fail
|
||||
rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
|
||||
rem the _cmd.exe /c_ return code!
|
||||
set EXIT_CODE=%ERRORLEVEL%
|
||||
if %EXIT_CODE% equ 0 set EXIT_CODE=1
|
||||
if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE%
|
||||
exit /b %EXIT_CODE%
|
||||
|
||||
:mainEnd
|
||||
if "%OS%"=="Windows_NT" endlocal
|
||||
|
||||
:omega
|
||||
1
shp-exporter_v2/settings.gradle
Normal file
1
shp-exporter_v2/settings.gradle
Normal file
@@ -0,0 +1 @@
|
||||
rootProject.name = 'shp-exporter-v2'
|
||||
@@ -0,0 +1,50 @@
|
||||
package com.kamco.shpexporter;
|
||||
|
||||
import com.kamco.shpexporter.config.ExporterProperties;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.batch.core.Job;
|
||||
import org.springframework.batch.core.JobParameters;
|
||||
import org.springframework.batch.core.JobParametersBuilder;
|
||||
import org.springframework.batch.core.launch.JobLauncher;
|
||||
import org.springframework.boot.CommandLineRunner;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@Component
|
||||
public class ExporterRunner implements CommandLineRunner {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(ExporterRunner.class);
|
||||
|
||||
private final JobLauncher jobLauncher;
|
||||
private final Job shpExporterJob;
|
||||
private final ExporterProperties properties;
|
||||
|
||||
public ExporterRunner(JobLauncher jobLauncher, Job shpExporterJob,
|
||||
ExporterProperties properties) {
|
||||
this.jobLauncher = jobLauncher;
|
||||
this.shpExporterJob = shpExporterJob;
|
||||
this.properties = properties;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run(String... args) throws Exception {
|
||||
if (properties.getBatchIds() == null || properties.getBatchIds().isEmpty()) {
|
||||
log.error("exporter.batch-ids 가 설정되지 않았습니다.");
|
||||
System.exit(1);
|
||||
}
|
||||
|
||||
log.info("=== shp-exporter-v2 시작 ===");
|
||||
log.info("inference-id : {}", properties.getInferenceId());
|
||||
log.info("batch-ids : {}", properties.getBatchIds());
|
||||
log.info("output : {}", properties.getOutputBaseDir());
|
||||
|
||||
JobParameters params = new JobParametersBuilder()
|
||||
.addString("inferenceId", properties.getInferenceId())
|
||||
.addString("batchIds", properties.getBatchIds().toString())
|
||||
.addLong("timestamp", System.currentTimeMillis())
|
||||
.toJobParameters();
|
||||
|
||||
var result = jobLauncher.run(shpExporterJob, params);
|
||||
log.info("Job 완료: {}", result.getExitStatus());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
package com.kamco.shpexporter;
|
||||
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
import org.springframework.boot.context.properties.EnableConfigurationProperties;
|
||||
|
||||
@SpringBootApplication
|
||||
@EnableConfigurationProperties
|
||||
public class ShpExporterApplication {
|
||||
|
||||
public static void main(String[] args) {
|
||||
System.exit(SpringApplication.exit(SpringApplication.run(ShpExporterApplication.class, args)));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
package com.kamco.shpexporter.batch;
|
||||
|
||||
import com.kamco.shpexporter.batch.reader.GeometryConvertingRowMapper;
|
||||
import com.kamco.shpexporter.batch.tasklet.GeomTypeTasklet;
|
||||
import com.kamco.shpexporter.batch.writer.MapIdSwitchingWriter;
|
||||
import com.kamco.shpexporter.config.ExporterProperties;
|
||||
import com.kamco.shpexporter.model.InferenceResult;
|
||||
import java.util.List;
|
||||
import javax.sql.DataSource;
|
||||
import org.geotools.api.referencing.FactoryException;
|
||||
import org.geotools.api.referencing.crs.CoordinateReferenceSystem;
|
||||
import org.geotools.referencing.CRS;
|
||||
import org.springframework.batch.core.Job;
|
||||
import org.springframework.batch.core.Step;
|
||||
import org.springframework.batch.core.job.builder.JobBuilder;
|
||||
import org.springframework.batch.core.repository.JobRepository;
|
||||
import org.springframework.batch.core.step.builder.StepBuilder;
|
||||
import org.springframework.batch.item.database.JdbcCursorItemReader;
|
||||
import org.springframework.batch.item.database.builder.JdbcCursorItemReaderBuilder;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.jdbc.core.PreparedStatementSetter;
|
||||
import org.springframework.transaction.PlatformTransactionManager;
|
||||
|
||||
@Configuration
|
||||
public class BatchJobConfig {
|
||||
|
||||
private final ExporterProperties properties;
|
||||
|
||||
public BatchJobConfig(ExporterProperties properties) {
|
||||
this.properties = properties;
|
||||
}
|
||||
|
||||
// ─── CRS Bean ──────────────────────────────────────────────────
|
||||
|
||||
@Bean
|
||||
public CoordinateReferenceSystem coordinateReferenceSystem() throws FactoryException {
|
||||
return CRS.decode(properties.getCrs());
|
||||
}
|
||||
|
||||
// ─── Job ───────────────────────────────────────────────────────
|
||||
|
||||
@Bean
|
||||
public Job shpExporterJob(
|
||||
JobRepository jobRepository,
|
||||
Step geomTypeStep,
|
||||
Step generateMapIdFilesStep) {
|
||||
|
||||
return new JobBuilder("shpExporterJob", jobRepository)
|
||||
.start(geomTypeStep)
|
||||
.next(generateMapIdFilesStep)
|
||||
.build();
|
||||
}
|
||||
|
||||
// ─── Step 1: geometry 타입 확인 ────────────────────────────────
|
||||
|
||||
@Bean
|
||||
public Step geomTypeStep(
|
||||
JobRepository jobRepository,
|
||||
PlatformTransactionManager transactionManager,
|
||||
GeomTypeTasklet geomTypeTasklet) {
|
||||
|
||||
return new StepBuilder("geomTypeStep", jobRepository)
|
||||
.tasklet(geomTypeTasklet, transactionManager)
|
||||
.build();
|
||||
}
|
||||
|
||||
// ─── Step 2: map_id별 shapefile 생성 ──────────────────────────
|
||||
|
||||
@Bean
|
||||
public Step generateMapIdFilesStep(
|
||||
JobRepository jobRepository,
|
||||
PlatformTransactionManager transactionManager,
|
||||
JdbcCursorItemReader<InferenceResult> inferenceResultReader,
|
||||
MapIdSwitchingWriter mapIdSwitchingWriter) {
|
||||
|
||||
return new StepBuilder("generateMapIdFilesStep", jobRepository)
|
||||
.<InferenceResult, InferenceResult>chunk(properties.getChunkSize(), transactionManager)
|
||||
.reader(inferenceResultReader)
|
||||
.writer(mapIdSwitchingWriter)
|
||||
.faultTolerant()
|
||||
.skipLimit(properties.getSkipLimit())
|
||||
.skip(Exception.class)
|
||||
.listener(mapIdSwitchingWriter) // @BeforeStep / @AfterStep 등록
|
||||
.build();
|
||||
}
|
||||
|
||||
// ─── Reader: ORDER BY map_id, uid (전체 스캔) ──────────────────
|
||||
|
||||
@Bean
|
||||
public JdbcCursorItemReader<InferenceResult> inferenceResultReader(
|
||||
DataSource dataSource,
|
||||
GeometryConvertingRowMapper rowMapper) {
|
||||
|
||||
List<Long> batchIds = properties.getBatchIds();
|
||||
|
||||
String sql =
|
||||
"SELECT uid, map_id, probability, before_year, after_year, "
|
||||
+ " before_c, before_p, after_c, after_p, "
|
||||
+ " ST_AsText(geometry) as geometry_wkt "
|
||||
+ "FROM inference_results_testing "
|
||||
+ "WHERE batch_id = ANY(?) "
|
||||
+ " AND ST_GeometryType(geometry) IN ('ST_Polygon', 'ST_MultiPolygon') "
|
||||
+ " AND ST_SRID(geometry) = 5186 "
|
||||
+ " AND ST_IsValid(geometry) = true "
|
||||
+ " AND after_c IS NOT NULL "
|
||||
+ " AND after_p IS NOT NULL "
|
||||
+ "ORDER BY map_id, uid";
|
||||
|
||||
PreparedStatementSetter pss = ps -> {
|
||||
var arr = ps.getConnection().createArrayOf("bigint", batchIds.toArray());
|
||||
ps.setArray(1, arr);
|
||||
};
|
||||
|
||||
return new JdbcCursorItemReaderBuilder<InferenceResult>()
|
||||
.name("inferenceResultReader")
|
||||
.dataSource(dataSource)
|
||||
.sql(sql)
|
||||
.preparedStatementSetter(pss)
|
||||
.rowMapper(rowMapper)
|
||||
.fetchSize(properties.getFetchSize())
|
||||
.build();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
package com.kamco.shpexporter.batch.reader;
|
||||
|
||||
import com.kamco.shpexporter.model.InferenceResult;
|
||||
import com.kamco.shpexporter.service.GeometryConverter;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import org.springframework.jdbc.core.RowMapper;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@Component
|
||||
public class GeometryConvertingRowMapper implements RowMapper<InferenceResult> {
|
||||
|
||||
private final GeometryConverter geometryConverter;
|
||||
|
||||
public GeometryConvertingRowMapper(GeometryConverter geometryConverter) {
|
||||
this.geometryConverter = geometryConverter;
|
||||
}
|
||||
|
||||
@Override
|
||||
public InferenceResult mapRow(ResultSet rs, int rowNum) throws SQLException {
|
||||
InferenceResult r = new InferenceResult();
|
||||
r.setUid(rs.getString("uid"));
|
||||
r.setMapId(rs.getString("map_id"));
|
||||
r.setProbability(getDoubleOrNull(rs, "probability"));
|
||||
r.setBeforeYear(getLongOrNull(rs, "before_year"));
|
||||
r.setAfterYear(getLongOrNull(rs, "after_year"));
|
||||
r.setBeforeC(rs.getString("before_c"));
|
||||
r.setBeforeP(getDoubleOrNull(rs, "before_p"));
|
||||
r.setAfterC(rs.getString("after_c"));
|
||||
r.setAfterP(getDoubleOrNull(rs, "after_p"));
|
||||
|
||||
String wkt = rs.getString("geometry_wkt");
|
||||
if (wkt != null) r.setGeometry(geometryConverter.convertWKTToJTS(wkt));
|
||||
|
||||
return r;
|
||||
}
|
||||
|
||||
private Long getLongOrNull(ResultSet rs, String col) throws SQLException {
|
||||
long v = rs.getLong(col);
|
||||
return rs.wasNull() ? null : v;
|
||||
}
|
||||
|
||||
private Double getDoubleOrNull(ResultSet rs, String col) throws SQLException {
|
||||
double v = rs.getDouble(col);
|
||||
return rs.wasNull() ? null : v;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
package com.kamco.shpexporter.batch.tasklet;
|
||||
|
||||
import com.kamco.shpexporter.config.ExporterProperties;
|
||||
import java.sql.Array;
|
||||
import java.util.List;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.batch.core.StepContribution;
|
||||
import org.springframework.batch.core.scope.context.ChunkContext;
|
||||
import org.springframework.batch.core.step.tasklet.Tasklet;
|
||||
import org.springframework.batch.repeat.RepeatStatus;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* Step 1: geometry 타입 사전 확인 후 JobExecutionContext에 저장.
|
||||
* MapIdSwitchingWriter가 이 값을 읽어 SimpleFeatureType을 구성합니다.
|
||||
*/
|
||||
@Component
|
||||
public class GeomTypeTasklet implements Tasklet {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(GeomTypeTasklet.class);
|
||||
|
||||
private final JdbcTemplate jdbcTemplate;
|
||||
private final ExporterProperties properties;
|
||||
|
||||
public GeomTypeTasklet(JdbcTemplate jdbcTemplate, ExporterProperties properties) {
|
||||
this.jdbcTemplate = jdbcTemplate;
|
||||
this.properties = properties;
|
||||
}
|
||||
|
||||
@Override
|
||||
public RepeatStatus execute(StepContribution contribution, ChunkContext chunkContext)
|
||||
throws Exception {
|
||||
|
||||
List<Long> batchIds = properties.getBatchIds();
|
||||
log.info("Validating geometry types for batch_ids: {}", batchIds);
|
||||
|
||||
List<String> geomTypes = jdbcTemplate.query(
|
||||
con -> {
|
||||
var ps = con.prepareStatement(
|
||||
"SELECT DISTINCT ST_GeometryType(geometry) "
|
||||
+ "FROM inference_results_testing "
|
||||
+ "WHERE batch_id = ANY(?) AND geometry IS NOT NULL");
|
||||
Array arr = con.createArrayOf("bigint", batchIds.toArray());
|
||||
ps.setArray(1, arr);
|
||||
return ps;
|
||||
},
|
||||
(rs, rowNum) -> rs.getString(1));
|
||||
|
||||
log.info("Detected geometry types: {}", geomTypes);
|
||||
|
||||
// MultiPolygon은 자동으로 Polygon으로 변환하므로 항상 Polygon으로 처리
|
||||
String resolved = "Polygon";
|
||||
log.info("Using geometry type: {}", resolved);
|
||||
|
||||
// 전체 map_id 수 및 레코드 수 사전 집계
|
||||
long[] counts = new long[]{0, 0};
|
||||
jdbcTemplate.query(
|
||||
con -> {
|
||||
var ps = con.prepareStatement(
|
||||
"SELECT COUNT(DISTINCT map_id), COUNT(*) "
|
||||
+ "FROM inference_results_testing "
|
||||
+ "WHERE batch_id = ANY(?) "
|
||||
+ " AND ST_GeometryType(geometry) IN ('ST_Polygon', 'ST_MultiPolygon') "
|
||||
+ " AND ST_SRID(geometry) = 5186 "
|
||||
+ " AND ST_IsValid(geometry) = true "
|
||||
+ " AND after_c IS NOT NULL "
|
||||
+ " AND after_p IS NOT NULL");
|
||||
Array arr = con.createArrayOf("bigint", batchIds.toArray());
|
||||
ps.setArray(1, arr);
|
||||
return ps;
|
||||
},
|
||||
rs -> {
|
||||
counts[0] = rs.getLong(1);
|
||||
counts[1] = rs.getLong(2);
|
||||
});
|
||||
|
||||
log.info("처리 대상: map_id {}개, 레코드 {}건", counts[0], counts[1]);
|
||||
|
||||
var jobCtx = chunkContext.getStepContext()
|
||||
.getStepExecution()
|
||||
.getJobExecution()
|
||||
.getExecutionContext();
|
||||
jobCtx.putString("geometryType", resolved);
|
||||
jobCtx.putLong("totalMapIds", counts[0]);
|
||||
jobCtx.putLong("totalExpectedRecords", counts[1]);
|
||||
|
||||
return RepeatStatus.FINISHED;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,267 @@
|
||||
package com.kamco.shpexporter.batch.writer;
|
||||
|
||||
import com.kamco.shpexporter.config.ExporterProperties;
|
||||
import com.kamco.shpexporter.model.InferenceResult;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.Serializable;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import org.geotools.api.data.SimpleFeatureStore;
|
||||
import org.geotools.api.data.Transaction;
|
||||
import org.geotools.api.feature.simple.SimpleFeature;
|
||||
import org.geotools.api.feature.simple.SimpleFeatureType;
|
||||
import org.geotools.api.referencing.crs.CoordinateReferenceSystem;
|
||||
import org.geotools.data.DefaultTransaction;
|
||||
import org.geotools.data.collection.ListFeatureCollection;
|
||||
import org.geotools.data.shapefile.ShapefileDataStore;
|
||||
import org.geotools.data.shapefile.ShapefileDataStoreFactory;
|
||||
import org.geotools.feature.simple.SimpleFeatureBuilder;
|
||||
import org.geotools.feature.simple.SimpleFeatureTypeBuilder;
|
||||
import org.locationtech.jts.geom.Polygon;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.batch.core.ExitStatus;
|
||||
import org.springframework.batch.core.StepExecution;
|
||||
import org.springframework.batch.core.annotation.AfterStep;
|
||||
import org.springframework.batch.core.annotation.BeforeStep;
|
||||
import org.springframework.batch.item.Chunk;
|
||||
import org.springframework.batch.item.ExecutionContext;
|
||||
import org.springframework.batch.item.ItemStreamException;
|
||||
import org.springframework.batch.item.ItemStreamWriter;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* map_id별 Shapefile을 생성하는 Writer.
|
||||
*
|
||||
* <p>데이터를 map_id, uid 순서로 읽어 map_id가 바뀔 때마다 파일을 전환합니다.
|
||||
* 파티셔닝 없이 단일 JDBC 커서로 전체 데이터를 스트리밍하며, 메모리에는
|
||||
* chunk 1개(기본 1000건)만 유지합니다.
|
||||
*
|
||||
* <p>출력 경로: {output-base-dir}/{inference-id}/{map_id}/{map_id}.shp
|
||||
*/
|
||||
@Component
|
||||
public class MapIdSwitchingWriter implements ItemStreamWriter<InferenceResult> {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(MapIdSwitchingWriter.class);
|
||||
private static final int LOG_INTERVAL_DEFAULT = 100; // 기본 진행 로그 간격
|
||||
|
||||
private final ExporterProperties properties;
|
||||
private final CoordinateReferenceSystem crs;
|
||||
private final ShapefileDataStoreFactory dsFactory = new ShapefileDataStoreFactory();
|
||||
|
||||
private SimpleFeatureType featureType;
|
||||
private SimpleFeatureBuilder featureBuilder;
|
||||
|
||||
// 현재 열려 있는 파일 관련 상태
|
||||
private String currentMapId = null;
|
||||
private ShapefileDataStore currentDataStore;
|
||||
private Transaction currentTransaction;
|
||||
private SimpleFeatureStore currentFeatureStore;
|
||||
|
||||
// 통계
|
||||
private int totalFiles = 0;
|
||||
private long totalRecords = 0;
|
||||
private long startTimeMs;
|
||||
private long totalMapIds = 0; // GeomTypeTasklet에서 사전 집계
|
||||
private int logInterval = LOG_INTERVAL_DEFAULT;
|
||||
|
||||
public MapIdSwitchingWriter(ExporterProperties properties, CoordinateReferenceSystem crs) {
|
||||
this.properties = properties;
|
||||
this.crs = crs;
|
||||
}
|
||||
|
||||
@BeforeStep
|
||||
public void beforeStep(StepExecution stepExecution) {
|
||||
var jobCtx = stepExecution.getJobExecution().getExecutionContext();
|
||||
String geomTypeStr = jobCtx.getString("geometryType", "Polygon");
|
||||
this.totalMapIds = jobCtx.getLong("totalMapIds", 0L);
|
||||
long totalExpectedRecords = jobCtx.getLong("totalExpectedRecords", 0L);
|
||||
|
||||
// 전체의 10% 단위로 로그 (최소 1, 최대 500)
|
||||
if (totalMapIds > 0) {
|
||||
logInterval = (int) Math.min(500, Math.max(1, totalMapIds / 10));
|
||||
}
|
||||
|
||||
Class<?> geomClass = resolveGeometryClass(geomTypeStr);
|
||||
this.featureType = buildFeatureType(geomClass);
|
||||
this.featureBuilder = new SimpleFeatureBuilder(featureType);
|
||||
this.startTimeMs = System.currentTimeMillis();
|
||||
|
||||
log.info("MapIdSwitchingWriter initialized. geometryType={}, outputBase={}/{}, 처리 대상: map_id {}개 / 레코드 {}건",
|
||||
geomTypeStr, properties.getOutputBaseDir(), properties.getInferenceId(),
|
||||
totalMapIds, totalExpectedRecords);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void open(ExecutionContext executionContext) throws ItemStreamException {
|
||||
// beforeStep에서 초기화 완료. 별도 작업 없음.
|
||||
}
|
||||
|
||||
@Override
|
||||
public void write(Chunk<? extends InferenceResult> chunk) throws Exception {
|
||||
List<SimpleFeature> buffer = new ArrayList<>();
|
||||
|
||||
for (InferenceResult result : chunk) {
|
||||
if (result.getGeometry() == null) continue;
|
||||
|
||||
String mapId = result.getMapId();
|
||||
if (mapId == null) continue;
|
||||
|
||||
// map_id가 바뀌면 버퍼를 flush하고 파일을 전환합니다
|
||||
if (!mapId.equals(currentMapId)) {
|
||||
if (!buffer.isEmpty()) {
|
||||
currentFeatureStore.addFeatures(new ListFeatureCollection(featureType, buffer));
|
||||
buffer.clear();
|
||||
}
|
||||
if (currentMapId != null) {
|
||||
commitAndClose(currentMapId);
|
||||
}
|
||||
openForMapId(mapId);
|
||||
currentMapId = mapId;
|
||||
totalFiles++;
|
||||
|
||||
if (totalFiles % logInterval == 0) {
|
||||
long elapsed = Math.max((System.currentTimeMillis() - startTimeMs) / 1000, 1);
|
||||
double rate = (double) totalFiles / elapsed; // map_id/s 기준 ETA 계산
|
||||
String progress = totalMapIds > 0
|
||||
? String.format("%d / %d (%.1f%%)", totalFiles, totalMapIds, totalFiles * 100.0 / totalMapIds)
|
||||
: String.valueOf(totalFiles);
|
||||
String eta = (totalMapIds > 0 && rate > 0)
|
||||
? formatSeconds((long) ((totalMapIds - totalFiles) / rate))
|
||||
: "N/A";
|
||||
log.info("진행: map_id {} 완료 | 총 {}건 | 경과 {}s | 속도 {}건/s | 남은시간 {} | thread={}",
|
||||
progress, totalRecords, elapsed, totalRecords / elapsed,
|
||||
eta, Thread.currentThread().getName());
|
||||
}
|
||||
}
|
||||
|
||||
buffer.add(buildFeature(result));
|
||||
totalRecords++;
|
||||
}
|
||||
|
||||
// chunk 끝 - 마지막 map_id 구간 flush
|
||||
if (!buffer.isEmpty()) {
|
||||
currentFeatureStore.addFeatures(new ListFeatureCollection(featureType, buffer));
|
||||
}
|
||||
}
|
||||
|
||||
@AfterStep
|
||||
public ExitStatus afterStep(StepExecution stepExecution) {
|
||||
if (currentMapId != null) {
|
||||
commitAndClose(currentMapId);
|
||||
currentMapId = null;
|
||||
}
|
||||
|
||||
long elapsed = Math.max((System.currentTimeMillis() - startTimeMs) / 1000, 1);
|
||||
log.info("=== 완료: map_id {}개, 총 {}건, 소요 {}s, 평균 {}건/s ===",
|
||||
totalFiles, totalRecords, elapsed, totalRecords / elapsed);
|
||||
|
||||
return ExitStatus.COMPLETED;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() throws ItemStreamException {
|
||||
// afterStep에서 처리. 혹시 남은 경우 안전하게 정리.
|
||||
if (currentMapId != null) {
|
||||
commitAndClose(currentMapId);
|
||||
currentMapId = null;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void update(ExecutionContext executionContext) throws ItemStreamException {
|
||||
executionContext.putLong("totalRecords", totalRecords);
|
||||
executionContext.putInt("totalFiles", totalFiles);
|
||||
}
|
||||
|
||||
// ─── private helpers ───────────────────────────────────────────
|
||||
|
||||
private void openForMapId(String mapId) throws IOException {
|
||||
Path dir = Paths.get(properties.getOutputBaseDir(), properties.getInferenceId(), mapId);
|
||||
Files.createDirectories(dir);
|
||||
File shpFile = dir.resolve(mapId + ".shp").toFile();
|
||||
|
||||
Map<String, Serializable> params = new HashMap<>();
|
||||
params.put("url", shpFile.toURI().toURL());
|
||||
params.put("create spatial index", Boolean.FALSE);
|
||||
|
||||
currentDataStore = (ShapefileDataStore) dsFactory.createNewDataStore(params);
|
||||
currentDataStore.createSchema(featureType);
|
||||
|
||||
currentTransaction = new DefaultTransaction("create-" + mapId);
|
||||
String typeName = currentDataStore.getTypeNames()[0];
|
||||
currentFeatureStore = (SimpleFeatureStore) currentDataStore.getFeatureSource(typeName);
|
||||
currentFeatureStore.setTransaction(currentTransaction);
|
||||
}
|
||||
|
||||
private void commitAndClose(String mapId) {
|
||||
try {
|
||||
currentTransaction.commit();
|
||||
} catch (IOException e) {
|
||||
log.error("[{}] commit 실패, rollback 시도", mapId, e);
|
||||
try {
|
||||
currentTransaction.rollback();
|
||||
} catch (IOException ignored) {}
|
||||
} finally {
|
||||
try { currentTransaction.close(); } catch (IOException ignored) {}
|
||||
currentDataStore.dispose();
|
||||
currentTransaction = null;
|
||||
currentDataStore = null;
|
||||
currentFeatureStore = null;
|
||||
}
|
||||
}
|
||||
|
||||
private SimpleFeature buildFeature(InferenceResult r) {
|
||||
featureBuilder.add(r.getGeometry());
|
||||
featureBuilder.add(r.getUid());
|
||||
featureBuilder.add(r.getMapId());
|
||||
featureBuilder.add(r.getProbability() != null ? String.valueOf(r.getProbability()) : "0.0");
|
||||
featureBuilder.add(r.getBeforeYear() != null ? r.getBeforeYear() : 0L);
|
||||
featureBuilder.add(r.getAfterYear() != null ? r.getAfterYear() : 0L);
|
||||
featureBuilder.add(r.getBeforeC());
|
||||
featureBuilder.add(r.getBeforeP() != null ? String.valueOf(r.getBeforeP()) : "0.0");
|
||||
featureBuilder.add(r.getAfterC());
|
||||
featureBuilder.add(r.getAfterP() != null ? String.valueOf(r.getAfterP()) : "0.0");
|
||||
return featureBuilder.buildFeature(null);
|
||||
}
|
||||
|
||||
private SimpleFeatureType buildFeatureType(Class<?> geomClass) {
|
||||
SimpleFeatureTypeBuilder builder = new SimpleFeatureTypeBuilder();
|
||||
builder.setName("inference_results");
|
||||
builder.setCRS(crs);
|
||||
builder.add("the_geom", geomClass);
|
||||
builder.setDefaultGeometry("the_geom");
|
||||
builder.add("uid", String.class);
|
||||
builder.add("map_id", String.class);
|
||||
builder.add("chn_dtct_p", String.class);
|
||||
builder.add("cprs_yr", Long.class);
|
||||
builder.add("crtr_yr", Long.class);
|
||||
builder.add("bf_cls_cd", String.class);
|
||||
builder.add("bf_cls_pro", String.class);
|
||||
builder.add("af_cls_cd", String.class);
|
||||
builder.add("af_cls_pro", String.class);
|
||||
return builder.buildFeatureType();
|
||||
}
|
||||
|
||||
private static String formatSeconds(long seconds) {
|
||||
if (seconds < 60) return seconds + "s";
|
||||
if (seconds < 3600) return String.format("%dm %ds", seconds / 60, seconds % 60);
|
||||
return String.format("%dh %dm", seconds / 3600, (seconds % 3600) / 60);
|
||||
}
|
||||
|
||||
private Class<?> resolveGeometryClass(String typeStr) {
|
||||
try {
|
||||
String name = typeStr.replace("ST_", "");
|
||||
return Class.forName("org.locationtech.jts.geom." + name);
|
||||
} catch (ClassNotFoundException e) {
|
||||
log.warn("알 수 없는 geometry type '{}', Polygon 사용", typeStr);
|
||||
return Polygon.class;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
package com.kamco.shpexporter.config;
|
||||
|
||||
import java.util.List;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@Component
|
||||
@ConfigurationProperties(prefix = "exporter")
|
||||
public class ExporterProperties {
|
||||
|
||||
private String inferenceId;
|
||||
private List<Long> batchIds;
|
||||
private String outputBaseDir;
|
||||
private String crs = "EPSG:5186";
|
||||
private int chunkSize = 1000;
|
||||
private int fetchSize = 1000;
|
||||
private int skipLimit = 100;
|
||||
|
||||
public String getInferenceId() { return inferenceId; }
|
||||
public void setInferenceId(String inferenceId) { this.inferenceId = inferenceId; }
|
||||
|
||||
public List<Long> getBatchIds() { return batchIds; }
|
||||
public void setBatchIds(List<Long> batchIds) { this.batchIds = batchIds; }
|
||||
|
||||
public String getOutputBaseDir() { return outputBaseDir; }
|
||||
public void setOutputBaseDir(String outputBaseDir) { this.outputBaseDir = outputBaseDir; }
|
||||
|
||||
public String getCrs() { return crs; }
|
||||
public void setCrs(String crs) { this.crs = crs; }
|
||||
|
||||
public int getChunkSize() { return chunkSize; }
|
||||
public void setChunkSize(int chunkSize) { this.chunkSize = chunkSize; }
|
||||
|
||||
public int getFetchSize() { return fetchSize; }
|
||||
public void setFetchSize(int fetchSize) { this.fetchSize = fetchSize; }
|
||||
|
||||
public int getSkipLimit() { return skipLimit; }
|
||||
public void setSkipLimit(int skipLimit) { this.skipLimit = skipLimit; }
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
package com.kamco.shpexporter.model;
|
||||
|
||||
import org.locationtech.jts.geom.Geometry;
|
||||
|
||||
public class InferenceResult {
|
||||
|
||||
private String uid;
|
||||
private String mapId;
|
||||
private Double probability;
|
||||
private Long beforeYear;
|
||||
private Long afterYear;
|
||||
private String beforeC;
|
||||
private Double beforeP;
|
||||
private String afterC;
|
||||
private Double afterP;
|
||||
private Geometry geometry;
|
||||
|
||||
public String getUid() { return uid; }
|
||||
public void setUid(String uid) { this.uid = uid; }
|
||||
|
||||
public String getMapId() { return mapId; }
|
||||
public void setMapId(String mapId) { this.mapId = mapId; }
|
||||
|
||||
public Double getProbability() { return probability; }
|
||||
public void setProbability(Double probability) { this.probability = probability; }
|
||||
|
||||
public Long getBeforeYear() { return beforeYear; }
|
||||
public void setBeforeYear(Long beforeYear) { this.beforeYear = beforeYear; }
|
||||
|
||||
public Long getAfterYear() { return afterYear; }
|
||||
public void setAfterYear(Long afterYear) { this.afterYear = afterYear; }
|
||||
|
||||
public String getBeforeC() { return beforeC; }
|
||||
public void setBeforeC(String beforeC) { this.beforeC = beforeC; }
|
||||
|
||||
public Double getBeforeP() { return beforeP; }
|
||||
public void setBeforeP(Double beforeP) { this.beforeP = beforeP; }
|
||||
|
||||
public String getAfterC() { return afterC; }
|
||||
public void setAfterC(String afterC) { this.afterC = afterC; }
|
||||
|
||||
public Double getAfterP() { return afterP; }
|
||||
public void setAfterP(Double afterP) { this.afterP = afterP; }
|
||||
|
||||
public Geometry getGeometry() { return geometry; }
|
||||
public void setGeometry(Geometry geometry) { this.geometry = geometry; }
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
package com.kamco.shpexporter.service;
|
||||
|
||||
import org.locationtech.jts.geom.Geometry;
|
||||
import org.locationtech.jts.geom.MultiPolygon;
|
||||
import org.locationtech.jts.geom.Polygon;
|
||||
import org.locationtech.jts.io.ParseException;
|
||||
import org.locationtech.jts.io.WKTReader;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@Component
|
||||
public class GeometryConverter {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(GeometryConverter.class);
|
||||
|
||||
private final WKTReader wktReader = new WKTReader();
|
||||
|
||||
public Geometry convertWKTToJTS(String wkt) {
|
||||
if (wkt == null || wkt.isBlank()) return null;
|
||||
|
||||
try {
|
||||
Geometry geom = wktReader.read(wkt);
|
||||
|
||||
// MultiPolygon → 첫 번째 Polygon으로 자동 변환
|
||||
if (geom instanceof MultiPolygon mp) {
|
||||
if (mp.getNumGeometries() == 0) return null;
|
||||
geom = (Polygon) mp.getGeometryN(0);
|
||||
}
|
||||
|
||||
return geom;
|
||||
} catch (ParseException e) {
|
||||
log.warn("WKT parse failed: {}", e.getMessage());
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
30
shp-exporter_v2/src/main/resources/application-prod.yml
Normal file
30
shp-exporter_v2/src/main/resources/application-prod.yml
Normal file
@@ -0,0 +1,30 @@
|
||||
spring:
|
||||
datasource:
|
||||
url: jdbc:postgresql://172.16.4.56:15432/kamco_cds
|
||||
username: kamco_cds
|
||||
password: kamco_cds_Q!W@E#R$
|
||||
driver-class-name: org.postgresql.Driver
|
||||
hikari:
|
||||
maximum-pool-size: 5 # cursor 1개 + 여유분. 단일 스텝이므로 많이 필요 없음
|
||||
connection-timeout: 30000
|
||||
idle-timeout: 600000
|
||||
max-lifetime: 1800000
|
||||
|
||||
exporter:
|
||||
inference-id: 'D5E46F60FC40B1A8BE0CD1F3547AA6'
|
||||
batch-ids:
|
||||
- 252
|
||||
- 253
|
||||
- 257
|
||||
output-base-dir: '/data/model_output/export/'
|
||||
crs: 'EPSG:5186'
|
||||
chunk-size: 1000
|
||||
fetch-size: 1000
|
||||
skip-limit: 100
|
||||
|
||||
logging:
|
||||
level:
|
||||
com.kamco.shpexporter: INFO
|
||||
org.springframework: WARN
|
||||
pattern:
|
||||
console: '%d{yyyy-MM-dd HH:mm:ss} - %msg%n'
|
||||
13
shp-exporter_v2/src/main/resources/application.yml
Normal file
13
shp-exporter_v2/src/main/resources/application.yml
Normal file
@@ -0,0 +1,13 @@
|
||||
spring:
|
||||
application:
|
||||
name: shp-exporter-v2
|
||||
profiles:
|
||||
active: prod
|
||||
main:
|
||||
web-application-type: none
|
||||
batch:
|
||||
job:
|
||||
enabled: false
|
||||
jdbc:
|
||||
initialize-schema: always
|
||||
table-prefix: BATCH_
|
||||
Reference in New Issue
Block a user