51 Commits

Author SHA1 Message Date
a5267d8065 Merge pull request 'select-dataset-list api solarPanelCnt 추가, spotless 적용' (#181) from feat/training_260324 into develop
Reviewed-on: #181
2026-04-02 17:45:16 +09:00
71d9835b03 select-dataset-list api solarPanelCnt 추가, spotless 적용 2026-04-02 17:44:27 +09:00
39f39a4f0c Merge pull request 'ModelType enum G4 추가' (#180) from feat/training_260324 into develop
Reviewed-on: #180
2026-04-02 16:55:14 +09:00
1df7142544 ModelType enum G4 추가 2026-04-02 16:54:25 +09:00
d99e18b38c val nan 일때 오류 수정, spotless 적용 2026-04-02 14:41:13 +09:00
d6aa612494 Merge pull request 'val nan 일때 오류 수정' (#179) from feat/training_260324 into develop
Reviewed-on: #179
2026-04-02 14:26:13 +09:00
8def356323 val nan 일때 오류 수정 2026-04-02 14:19:42 +09:00
dean
6cc9b54ba9 welcome 2026-04-01 08:55:17 +09:00
dean
50ad05b53b welcome 2026-03-31 17:44:08 +09:00
dean
822dbb252f test 2026-03-31 17:15:45 +09:00
dean
5e04df73af test 2026-03-31 17:15:29 +09:00
650b1695f0 Merge pull request '미사용 목록 주석 추가, 학습데이터 삭제 테스트' (#178) from feat/training_260324 into develop
Reviewed-on: #178
2026-03-27 18:02:34 +09:00
960e4215e0 미사용 목록 주석 추가, 학습데이터 삭제 테스트 2026-03-27 18:02:13 +09:00
08a220db4d Merge pull request '미사용 목록 주석 추가, 납품데이터 폴더 벨리데이션 위치 옮김' (#177) from feat/training_260324 into develop
Reviewed-on: #177
2026-03-27 17:04:25 +09:00
a9b49faa6c 미사용 목록 주석 추가, 납품데이터 폴더 벨리데이션 위치 옮김 2026-03-27 17:03:52 +09:00
b760e9874c Merge pull request '납품데이터셋 업로드 geojson 파일 수정' (#176) from feat/training_260324 into develop
Reviewed-on: #176
2026-03-27 10:56:57 +09:00
8698bf61d1 납품데이터셋 업로드 geojson 파일 수정 2026-03-27 10:56:38 +09:00
680e137284 Merge pull request '납품데이터셋 업로드 geojson 파일 수정' (#175) from feat/training_260324 into develop
Reviewed-on: #175
2026-03-27 10:18:15 +09:00
f4a81a34d6 납품데이터셋 업로드 geojson 파일 수정 2026-03-27 10:17:58 +09:00
479ad710e0 Merge pull request '납품데이터셋 업로드 geojson 파일 수정' (#174) from feat/training_260324 into develop
Reviewed-on: #174
2026-03-27 10:15:32 +09:00
3cb9840248 납품데이터셋 업로드 geojson 파일 수정 2026-03-27 10:14:44 +09:00
fc9543f195 Merge pull request 'feat/training_260324' (#173) from feat/training_260324 into develop
Reviewed-on: #173
2026-03-27 10:05:24 +09:00
73d0e03b08 납품데이터셋 업로드 geojson 파일 수정 2026-03-27 10:04:56 +09:00
50c965cb79 학습결과 파일 베스트 에폭 제외 삭제 추가, 납품데이터 등록 비동기 수정 2026-03-27 09:32:49 +09:00
dean
abca9467d8 docker 2026-03-26 20:07:37 +09:00
dean
4ed03f6e94 docker 2026-03-26 11:26:46 +09:00
04eddfce54 학습결과 파일 베스트 에폭 제외 삭제 2026-03-25 17:56:36 +09:00
f49b7cc850 Merge pull request '납품데이터 등록 수정' (#172) from feat/training_260324 into develop
Reviewed-on: #172
2026-03-25 12:52:25 +09:00
888c0e314b 납품데이터 등록 수정 2026-03-25 12:50:28 +09:00
6c043b0031 Merge pull request '납품데이터 등록 수정' (#171) from feat/training_260324 into develop
Reviewed-on: #171
2026-03-25 12:38:56 +09:00
531da09c5f 납품데이터 등록 수정 2026-03-25 12:37:27 +09:00
4da2a1f0d7 Merge pull request 'spotlessApply 적용' (#170) from feat/training_260324 into develop
Reviewed-on: #170
2026-03-25 12:33:04 +09:00
50b3f1ba62 spotlessApply 적용 2026-03-25 12:32:50 +09:00
f1f88c83e1 Merge pull request '납품 데이터 등록 api 추가' (#169) from feat/training_260324 into develop
Reviewed-on: #169
2026-03-25 12:30:51 +09:00
ff478452a6 납품 데이터 등록 api 추가 2026-03-25 12:30:30 +09:00
bf77725ef8 Merge pull request '납품 폴더 조회 api 추가' (#168) from feat/training_260324 into develop
Reviewed-on: #168
2026-03-24 17:30:18 +09:00
dec0f26999 납품 폴더 조회 api 추가 2026-03-24 17:30:04 +09:00
dfd4a42379 Merge pull request '납품 폴더 조회 api 추가' (#167) from feat/training_260324 into develop
Reviewed-on: #167
2026-03-24 17:19:39 +09:00
79272137ab 납품 폴더 조회 api 추가 2026-03-24 17:19:06 +09:00
73ea6176b4 Merge pull request '시스템 사용율 모니터링 기능 로그 수정' (#166) from feat/training_260303 into develop
Reviewed-on: #166
2026-03-19 17:38:45 +09:00
26caf505b9 Merge pull request '시스템 사용율 모니터링 기능 추가' (#165) from feat/training_260303 into develop
Reviewed-on: #165
2026-03-19 17:09:08 +09:00
bd54854bc6 Merge pull request '시스템 사용율 모니터링 기능 테스트' (#164) from feat/training_260303 into develop
Reviewed-on: #164
2026-03-19 16:52:44 +09:00
ca3d115d0e Merge pull request '시스템 사용율 모니터링 기능 테스트' (#163) from feat/training_260303 into develop
Reviewed-on: #163
2026-03-19 16:36:56 +09:00
831ba3e616 Merge pull request '시스템 사용율 모니터링 기능 테스트' (#162) from feat/training_260303 into develop
Reviewed-on: #162
2026-03-19 16:27:19 +09:00
a4b5e20db2 Merge pull request '시스템 사용율 모니터링 기능 테스트' (#161) from feat/training_260303 into develop
Reviewed-on: #161
2026-03-19 16:12:45 +09:00
da260f35ea Merge pull request '시스템 사용율 모니터링 기능 테스트' (#160) from feat/training_260303 into develop
Reviewed-on: #160
2026-03-19 15:46:40 +09:00
6cf81bf60f Merge pull request '시스템 사용율 모니터링 기능 테스트' (#159) from feat/training_260303 into develop
Reviewed-on: #159
2026-03-19 15:41:34 +09:00
ed95829a34 Merge pull request '시스템 사용율 모니터링 기능 테스트' (#158) from feat/training_260303 into develop
Reviewed-on: #158
2026-03-19 15:37:47 +09:00
52ffe53815 Merge pull request 'feat/training_260303' (#157) from feat/training_260303 into develop
Reviewed-on: #157
2026-03-19 15:25:37 +09:00
5887a954ea Merge pull request '시스템 사용율 모니터링 기능 테스트' (#156) from feat/training_260303 into develop
Reviewed-on: #156
2026-03-19 15:24:38 +09:00
dean
72bc2fd47b test 2026-03-17 21:02:21 +09:00
68 changed files with 1967 additions and 379 deletions

View File

@@ -125,6 +125,7 @@ public class CommonCodeService {
return commonCodeCoreService.getCode(parentCodeCd, childCodeCd); return commonCodeCoreService.getCode(parentCodeCd, childCodeCd);
} }
// TODO 미사용시작
/** /**
* 공통코드 이름 조회 * 공통코드 이름 조회
* *
@@ -136,6 +137,8 @@ public class CommonCodeService {
return commonCodeCoreService.getCode(parentCodeCd, childCodeCd); return commonCodeCoreService.getCode(parentCodeCd, childCodeCd);
} }
// TODO 미사용 끝
public List<CodeDto> getTypeCode(String type) { public List<CodeDto> getTypeCode(String type) {
return Enums.getCodes(type); return Enums.getCodes(type);
} }

View File

@@ -1,3 +1,4 @@
// TODO 미사용시작
package com.kamco.cd.training.common.download; package com.kamco.cd.training.common.download;
import com.kamco.cd.training.common.download.dto.DownloadSpec; import com.kamco.cd.training.common.download.dto.DownloadSpec;
@@ -46,3 +47,4 @@ public class DownloadExecutor {
.body(body); .body(body);
} }
} }
// TODO 미사용 끝

View File

@@ -1,3 +1,4 @@
// TODO 미사용시작
package com.kamco.cd.training.common.download; package com.kamco.cd.training.common.download;
import org.springframework.util.AntPathMatcher; import org.springframework.util.AntPathMatcher;
@@ -17,3 +18,4 @@ public final class DownloadPaths {
return false; return false;
} }
} }
// TODO 미사용 끝

View File

@@ -1,3 +1,4 @@
// TODO 미사용시작
package com.kamco.cd.training.common.enums; package com.kamco.cd.training.common.enums;
import com.kamco.cd.training.common.utils.enums.CodeExpose; import com.kamco.cd.training.common.utils.enums.CodeExpose;
@@ -17,3 +18,4 @@ public enum DeployTargetType implements EnumType {
private final String id; private final String id;
private final String text; private final String text;
} }
// TODO 미사용 끝

View File

@@ -1,3 +1,4 @@
// TODO 미사용시작
package com.kamco.cd.training.common.enums; package com.kamco.cd.training.common.enums;
import com.kamco.cd.training.common.utils.enums.CodeExpose; import com.kamco.cd.training.common.utils.enums.CodeExpose;
@@ -25,3 +26,4 @@ public enum ModelMngStatusType implements EnumType {
return desc; return desc;
} }
} }
// TODO 미사용 끝

View File

@@ -12,7 +12,8 @@ import lombok.Getter;
public enum ModelType implements EnumType { public enum ModelType implements EnumType {
G1("G1"), G1("G1"),
G2("G2"), G2("G2"),
G3("G3"); G3("G3"),
G4("G4");
private String desc; private String desc;

View File

@@ -1,3 +1,4 @@
// TODO 미사용시작
package com.kamco.cd.training.common.enums; package com.kamco.cd.training.common.enums;
import com.kamco.cd.training.common.utils.enums.CodeExpose; import com.kamco.cd.training.common.utils.enums.CodeExpose;
@@ -18,3 +19,4 @@ public enum ProcessStepType implements EnumType {
private final String id; private final String id;
private final String text; private final String text;
} }
// TODO 미사용 끝

View File

@@ -20,4 +20,18 @@ public class AsyncConfig {
executor.initialize(); executor.initialize();
return executor; return executor;
} }
@Bean("datasetExecutor")
public Executor datasetExecutor() {
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
executor.setCorePoolSize(5);
executor.setMaxPoolSize(10);
executor.setQueueCapacity(100);
executor.setThreadNamePrefix("dataset-");
executor.initialize();
return executor;
}
} }

View File

@@ -24,7 +24,7 @@ public class OpenApiConfig {
@Value("${swagger.dev-url:https://kamco.training-dev-api.gs.dabeeo.com}") @Value("${swagger.dev-url:https://kamco.training-dev-api.gs.dabeeo.com}")
private String devUrl; private String devUrl;
@Value("${swagger.prod-url:https://api.training-kamco.com}") @Value("${swagger.prod-url:https://api.train-kamco.com}")
private String prodUrl; private String prodUrl;
@Bean @Bean
@@ -53,10 +53,10 @@ public class OpenApiConfig {
} else if ("prod".equals(profile)) { } else if ("prod".equals(profile)) {
// servers.add(new Server().url(prodUrl).description("운영 서버")); // servers.add(new Server().url(prodUrl).description("운영 서버"));
servers.add(new Server().url("http://localhost:" + localPort).description("로컬 서버")); servers.add(new Server().url("http://localhost:" + localPort).description("로컬 서버"));
servers.add(new Server().url(devUrl).description("개발 서버")); servers.add(new Server().url(prodUrl).description("개발 서버"));
} else { } else {
servers.add(new Server().url("http://localhost:" + localPort).description("로컬 서버")); servers.add(new Server().url("http://localhost:" + localPort).description("로컬 서버"));
servers.add(new Server().url(devUrl).description("개발 서버")); servers.add(new Server().url(devUrl).description("운영 서버"));
// servers.add(new Server().url(prodUrl).description("운영 서버")); // servers.add(new Server().url(prodUrl).description("운영 서버"));
} }

View File

@@ -57,7 +57,7 @@ public class StartupLogger {
""" """
╔════════════════════════════════════════════════════════════════════════════════╗ ╔════════════════════════════════════════════════════════════════════════════════╗
║ 🚀 APPLICATION STARTUP INFORMATION 2 ║ 🚀 APPLICATION STARTUP INFORMATION 3
╠════════════════════════════════════════════════════════════════════════════════╣ ╠════════════════════════════════════════════════════════════════════════════════╣
║ PROFILE CONFIGURATION ║ ║ PROFILE CONFIGURATION ║
╠────────────────────────────────────────────────────────────────────────────────╣ ╠────────────────────────────────────────────────────────────────────────────────╣

View File

@@ -16,6 +16,12 @@ public class ApiLogFilter extends OncePerRequestFilter {
protected void doFilterInternal( protected void doFilterInternal(
HttpServletRequest request, HttpServletResponse response, FilterChain filterChain) HttpServletRequest request, HttpServletResponse response, FilterChain filterChain)
throws ServletException, IOException { throws ServletException, IOException {
String uri = request.getRequestURI();
if (uri.contains("/download/")) {
filterChain.doFilter(request, response);
return;
}
ContentCachingRequestWrapper wrappedRequest = new ContentCachingRequestWrapper(request); ContentCachingRequestWrapper wrappedRequest = new ContentCachingRequestWrapper(request);
ContentCachingResponseWrapper wrappedResponse = new ContentCachingResponseWrapper(response); ContentCachingResponseWrapper wrappedResponse = new ContentCachingResponseWrapper(response);

View File

@@ -2,10 +2,14 @@ package com.kamco.cd.training.dataset;
import com.kamco.cd.training.config.api.ApiResponseDto; import com.kamco.cd.training.config.api.ApiResponseDto;
import com.kamco.cd.training.dataset.dto.DatasetDto; import com.kamco.cd.training.dataset.dto.DatasetDto;
import com.kamco.cd.training.dataset.dto.DatasetDto.AddDeliveriesReq;
import com.kamco.cd.training.dataset.dto.DatasetObjDto; import com.kamco.cd.training.dataset.dto.DatasetObjDto;
import com.kamco.cd.training.dataset.dto.DatasetObjDto.DatasetClass; import com.kamco.cd.training.dataset.dto.DatasetObjDto.DatasetClass;
import com.kamco.cd.training.dataset.dto.DatasetObjDto.DatasetStorage; import com.kamco.cd.training.dataset.dto.DatasetObjDto.DatasetStorage;
import com.kamco.cd.training.dataset.service.DatasetAsyncService;
import com.kamco.cd.training.dataset.service.DatasetService; import com.kamco.cd.training.dataset.service.DatasetService;
import com.kamco.cd.training.model.dto.FileDto.FoldersDto;
import com.kamco.cd.training.model.dto.FileDto.SrchFoldersDto;
import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.media.Content; import io.swagger.v3.oas.annotations.media.Content;
@@ -33,6 +37,7 @@ import org.springframework.web.bind.annotation.*;
public class DatasetApiController { public class DatasetApiController {
private final DatasetService datasetService; private final DatasetService datasetService;
private final DatasetAsyncService datasetAsyncService;
@Operation(summary = "학습데이터 관리 목록 조회", description = "학습데이터 목록을 조회합니다.") @Operation(summary = "학습데이터 관리 목록 조회", description = "학습데이터 목록을 조회합니다.")
@ApiResponses( @ApiResponses(
@@ -248,4 +253,48 @@ public class DatasetApiController {
String path = datasetService.getFilePathByUUIDPathType(uuid, pathType); String path = datasetService.getFilePathByUUIDPathType(uuid, pathType);
return datasetService.getFilePathByFile(path); return datasetService.getFilePathByFile(path);
} }
@Operation(summary = "납품 폴더 조회", description = "납품 폴더 조회 API")
@ApiResponses(
value = {
@ApiResponse(
responseCode = "200",
description = "조회 성공",
content =
@Content(
mediaType = "application/json",
schema = @Schema(implementation = FoldersDto.class))),
@ApiResponse(responseCode = "404", description = "조회 오류", content = @Content),
@ApiResponse(responseCode = "500", description = "서버 오류", content = @Content)
})
@PostMapping("/folder-list")
public ApiResponseDto<FoldersDto> getDir(@RequestBody SrchFoldersDto srchDto) throws IOException {
return ApiResponseDto.createOK(datasetService.getFolderAll(srchDto));
}
@Operation(summary = "납품 학습데이터셋 등록", description = "납품 학습데이터셋 등록 API")
@ApiResponses(
value = {
@ApiResponse(
responseCode = "200",
description = "등록 성공",
content =
@Content(
mediaType = "application/json",
schema = @Schema(implementation = String.class))),
@ApiResponse(responseCode = "404", description = "조회 오류", content = @Content),
@ApiResponse(responseCode = "500", description = "서버 오류", content = @Content)
})
@PostMapping("/deliveries")
public ApiResponseDto<String> insertDeliveriesDataset(@RequestBody AddDeliveriesReq req) {
// 폴더 구조 검증
DatasetService.validateTrainValTestDirs(req.getFilePath());
// 파일 개수 검증
DatasetService.validateDirFileCount(req.getFilePath());
datasetAsyncService.insertDeliveriesDatasetAsync(req);
return ApiResponseDto.createOK("ok");
}
} }

View File

@@ -145,6 +145,7 @@ public class DatasetDto {
} }
} }
// TODO 미사용시작
@Schema(name = "DatasetDetailReq", description = "데이터셋 상세 조회 요청") @Schema(name = "DatasetDetailReq", description = "데이터셋 상세 조회 요청")
@Getter @Getter
@Setter @Setter
@@ -157,6 +158,8 @@ public class DatasetDto {
private Long datasetId; private Long datasetId;
} }
// TODO 미사용 끝
@Schema(name = "DatasetRegisterReq", description = "데이터셋 등록 요청") @Schema(name = "DatasetRegisterReq", description = "데이터셋 등록 요청")
@Getter @Getter
@Setter @Setter
@@ -251,6 +254,7 @@ public class DatasetDto {
private Long wasteCnt; private Long wasteCnt;
private Long landCoverCnt; private Long landCoverCnt;
private Integer solarPanelCnt;
public SelectDataSet( public SelectDataSet(
String modelNo, String modelNo,
@@ -305,6 +309,29 @@ public class DatasetDto {
this.containerCnt = containerCnt; this.containerCnt = containerCnt;
} }
public SelectDataSet(
String modelNo,
Long datasetId,
UUID uuid,
String dataType,
String title,
Long roundNo,
Integer compareYyyy,
Integer targetYyyy,
String memo,
Integer solarPanelCnt) {
this.datasetId = datasetId;
this.uuid = uuid;
this.dataType = dataType;
this.dataTypeName = getDataTypeName(dataType);
this.title = title;
this.roundNo = roundNo;
this.compareYyyy = compareYyyy;
this.targetYyyy = targetYyyy;
this.memo = memo;
this.solarPanelCnt = solarPanelCnt;
}
public String getDataTypeName(String groupTitleCd) { public String getDataTypeName(String groupTitleCd) {
LearnDataType type = Enums.fromId(LearnDataType.class, groupTitleCd); LearnDataType type = Enums.fromId(LearnDataType.class, groupTitleCd);
return type == null ? null : type.getText(); return type == null ? null : type.getText();
@@ -532,4 +559,28 @@ public class DatasetDto {
private Long totalObjectCount; private Long totalObjectCount;
private String datasetPath; private String datasetPath;
} }
@Getter
@Setter
@NoArgsConstructor
@AllArgsConstructor
public static class AddDeliveriesReq {
@Schema(description = "경로", example = "/")
private String filePath;
@Schema(description = "제목", example = "")
private String title;
@Schema(description = "메모", example = "")
private String memo;
@Schema(description = "비교년도", example = "")
private Integer compareYyyy;
@Schema(description = "기준년도", example = "")
private Integer targetYyyy;
@Schema(description = "회차", example = "")
private Long roundNo;
}
} }

View File

@@ -1,3 +1,4 @@
// TODO 미사용시작
package com.kamco.cd.training.dataset.dto; package com.kamco.cd.training.dataset.dto;
import com.kamco.cd.training.common.utils.interfaces.JsonFormatDttm; import com.kamco.cd.training.common.utils.interfaces.JsonFormatDttm;
@@ -72,6 +73,7 @@ public class MapSheetDto {
private List<Long> itemIds; private List<Long> itemIds;
} }
// TODO 미사용시작
@Schema(name = "MapSheetCheckReq", description = "도엽 번호 유효성 검증 요청") @Schema(name = "MapSheetCheckReq", description = "도엽 번호 유효성 검증 요청")
@Getter @Getter
@Setter @Setter
@@ -101,3 +103,4 @@ public class MapSheetDto {
private boolean duplicate; private boolean duplicate;
} }
} }
// TODO 미사용 끝

View File

@@ -0,0 +1,124 @@
package com.kamco.cd.training.dataset.service;
import com.kamco.cd.training.common.enums.LearnDataRegister;
import com.kamco.cd.training.dataset.dto.DatasetDto.AddDeliveriesReq;
import com.kamco.cd.training.dataset.dto.DatasetDto.DatasetMngRegDto;
import com.kamco.cd.training.postgres.core.DatasetCoreService;
import java.util.UUID;
import lombok.RequiredArgsConstructor;
import lombok.extern.log4j.Log4j2;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
@Service
@Log4j2
@RequiredArgsConstructor
public class DatasetAsyncService {
private final DatasetService datasetService;
private final DatasetCoreService datasetCoreService;
private static final String LOG_PREFIX = "[납품 데이터셋]";
/**
* 납품 데이터셋 등록 비동기 업로드 처리 1) 데이터셋 구조/파일 검증 2) UID 생성 및 마스터 데이터 저장 3) 상태를 UPLOADING으로 변경 4) 실제
* 데이터(train/val/test) 등록 5) 완료 시 COMPLETED 상태로 변경 6) 실패 시 상태를 UPLOAD_FAILED로 변경 후 데이터 정리(삭제)
*
* @param req
*/
@Async("datasetExecutor")
public void insertDeliveriesDatasetAsync(AddDeliveriesReq req) {
long startTime = System.currentTimeMillis();
log.info("{} 업로드 시작 ==========", LOG_PREFIX);
log.info(
"{} filePath={}, targetYyyy={}, compareYyyy={}, roundNo={}",
LOG_PREFIX,
req.getFilePath(),
req.getTargetYyyy(),
req.getCompareYyyy(),
req.getRoundNo());
Long datasetUid = null;
try {
// ===== 1. UID 생성 =====
String uid = UUID.randomUUID().toString().replace("-", "").toUpperCase();
log.info("{} 생성된 UID: {}", LOG_PREFIX, uid);
// ===== 2. 마스터 데이터 생성 =====
String title = req.getTitle();
if (title == null || title.isBlank()) {
Integer compareYyyy = req.getCompareYyyy();
Integer targetYyyy = req.getTargetYyyy();
if (compareYyyy != null && targetYyyy != null) {
title = compareYyyy + "-" + targetYyyy;
} else {
title = null;
}
}
DatasetMngRegDto datasetMngRegDto = new DatasetMngRegDto();
datasetMngRegDto.setUid(uid);
datasetMngRegDto.setDataType("DELIVER");
datasetMngRegDto.setCompareYyyy(req.getCompareYyyy() == null ? 0 : req.getCompareYyyy());
datasetMngRegDto.setTargetYyyy(req.getTargetYyyy() == null ? 0 : req.getTargetYyyy());
datasetMngRegDto.setRoundNo(req.getRoundNo());
datasetMngRegDto.setTitle(title);
datasetMngRegDto.setMemo(req.getMemo());
datasetMngRegDto.setDatasetPath(req.getFilePath());
// 마스터 저장
datasetUid = datasetCoreService.insertDatasetMngData(datasetMngRegDto);
log.info("{} 마스터 저장 완료. datasetUid={}", LOG_PREFIX, datasetUid);
// ===== 3. 상태 변경 (업로드중) =====
datasetCoreService.updateDatasetUploadStatus(datasetUid, LearnDataRegister.UPLOADING);
log.info("{} 상태 변경 → UPLOADING. datasetUid={}", LOG_PREFIX, datasetUid);
// ===== 4. 데이터 등록 =====
long insertStart = System.currentTimeMillis();
// 납품 데이터 obj 등록
datasetService.insertDeliveriesDataset(req, datasetUid);
log.info(
"{} 데이터 등록 완료. datasetUid={}, 소요시간={} ms",
LOG_PREFIX,
datasetUid,
System.currentTimeMillis() - insertStart);
// ===== 5. 상태 변경 (완료) =====
datasetCoreService.updateDatasetUploadStatus(datasetUid, LearnDataRegister.COMPLETED);
log.info("{} 상태 변경 → COMPLETED. datasetUid={}", LOG_PREFIX, datasetUid);
log.info(
"{} 업로드 완료. 총 소요시간={} ms ==========", LOG_PREFIX, System.currentTimeMillis() - startTime);
} catch (Exception e) {
log.error(
"{} 업로드 실패. datasetUid={}, filePath={}", LOG_PREFIX, datasetUid, req.getFilePath(), e);
if (datasetUid != null) {
try {
// ===== 실패 처리 =====
datasetCoreService.updateDatasetUploadStatus(datasetUid, LearnDataRegister.UPLOAD_FAILED);
log.error("{} 상태 변경 → 업로드 실패. datasetUid={}", LOG_PREFIX, datasetUid);
// 실패 시 데이터 정리
datasetCoreService.deleteAllDatasetObj(datasetUid);
log.error("{} 데이터 정리 완료. datasetUid={}", LOG_PREFIX, datasetUid);
} catch (Exception ex) {
log.error("{} 실패 후 정리 작업 중 오류. datasetUid={}", LOG_PREFIX, datasetUid, ex);
}
}
}
}
}

View File

@@ -0,0 +1,199 @@
package com.kamco.cd.training.dataset.service;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.kamco.cd.training.dataset.dto.DatasetObjDto.DatasetObjRegDto;
import com.kamco.cd.training.postgres.core.DatasetCoreService;
import java.nio.file.Paths;
import java.util.List;
import java.util.Map;
import lombok.RequiredArgsConstructor;
import lombok.extern.log4j.Log4j2;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
@Service
@Log4j2
@RequiredArgsConstructor
public class DatasetBatchService {
private final DatasetCoreService datasetCoreService;
private final ObjectMapper mapper;
/**
* 배치 단위 데이터 저장
*
* <p>- 전달받은 데이터 목록을 순회하며 개별 insert 처리 - batch 단위로 트랜잭션 관리
*/
@Transactional
public void saveBatch(List<Map<String, Object>> batch, Long datasetUid, String type) {
for (Map<String, Object> map : batch) {
try {
insertTrainTestData(map, datasetUid, type);
} catch (Exception e) {
log.error("파일 단위 실패. skip. file={}", batch, e);
continue;
}
}
}
/**
* 단일 데이터 처리 및 insert DTO 생성
*
* <p>처리 흐름: 1) 경로/JSON 데이터 추출 2) 파일명에서 연도 및 도엽번호 파싱 3) label JSON → feature 단위 분리 4) feature별 DTO
* 생성 후 DB insert
*/
private void insertTrainTestData(Map<String, Object> map, Long datasetUid, String subDir) {
String comparePath = (String) map.get("input1");
String targetPath = (String) map.get("input2");
String labelPath = (String) map.get("label");
String geojsonPath = (String) map.get("geojson_path");
Object labelJson = map.get("label-json");
// JSON 파싱
JsonNode json;
try {
json = parseJson(labelJson);
if (json == null) {
log.warn("json null. skip. file={}", labelJson);
return;
}
} catch (Exception e) {
// 실패하면 skip, 다음 진행
log.error("GeoJSON 파싱 실패. skip. file={}", geojsonPath, e);
return;
}
// 파일명 파싱
String fileName = Paths.get(comparePath).getFileName().toString();
String[] fileNameStr = fileName.split("_");
if (fileNameStr.length < 4) {
log.error("파일명 파싱 실패: {}", fileName);
return;
// throw new IllegalArgumentException("잘못된 파일명 형식: " + fileName);
}
int compareYyyy = 0;
int targetYyyy = 0;
try {
compareYyyy = parseInt(fileNameStr[1], "compareYyyy", fileName);
targetYyyy = parseInt(fileNameStr[2], "targetYyyy", fileName);
} catch (Exception e) {
log.error("기준년도 파싱 실패: {}", fileName);
return;
}
String mapSheetNum = fileNameStr[3];
// JSON 유효성 체크
JsonNode featuresNode = json.path("features");
// 2. 비어있는지 확인
if (featuresNode.isEmpty()) {
log.warn("features empty. skip. file={}", geojsonPath);
return;
}
if (!featuresNode.isArray()) {
log.warn("features array 아님. skip. file={}", geojsonPath);
return; // skip
}
if (featuresNode.isMissingNode() || !featuresNode.isArray() || featuresNode.isEmpty()) {
return; // skip
}
ObjectNode base = mapper.createObjectNode();
base.put("type", "FeatureCollection");
for (JsonNode feature : featuresNode) {
try {
JsonNode prop = feature.path("properties");
String compareClassCd = prop.path("before").asText(null);
String targetClassCd = prop.path("after").asText(null);
// null 방어
if (compareClassCd == null || targetClassCd == null) {
log.warn("class 값 없음. skip. file={}", fileName);
continue;
}
ArrayNode arr = mapper.createArrayNode();
arr.add(feature);
ObjectNode root = base.deepCopy();
root.set("features", arr);
DatasetObjRegDto objRegDto =
DatasetObjRegDto.builder()
.datasetUid(datasetUid)
.compareYyyy(compareYyyy)
.compareClassCd(compareClassCd)
.targetYyyy(targetYyyy)
.targetClassCd(targetClassCd)
.comparePath(comparePath)
.targetPath(targetPath)
.labelPath(labelPath)
.mapSheetNum(mapSheetNum)
.geojson(root)
.geojsonPath(geojsonPath)
.fileName(fileName)
.build();
// 데이터 타입별 insert
insertByType(subDir, objRegDto);
} catch (Exception e) {
// 개별 feature skip
log.error("feature 처리 실패. skip. file={}", fileName, e);
}
}
}
/** 데이터 타입별 insert 처리 - type 값에 따라 대상 테이블 분기 - 잘못된 타입 입력 시 예외 발생 */
private void insertByType(String type, DatasetObjRegDto dto) {
switch (type) {
case "train" -> datasetCoreService.insertDatasetObj(dto);
case "val" -> datasetCoreService.insertDatasetValObj(dto);
case "test" -> datasetCoreService.insertDatasetTestObj(dto);
default -> throw new IllegalArgumentException("잘못된 타입: " + type);
}
}
/**
* label_json → JsonNode 변환
*
* <p>- JsonNode면 그대로 사용 - 문자열이면 파싱 수행 - 실패 시 로그 후 예외 발생
*/
private JsonNode parseJson(Object labelJson) {
try {
if (labelJson instanceof JsonNode jn) {
return jn;
}
return mapper.readTree(labelJson.toString());
} catch (Exception e) {
log.error("label_json parse error: {}", labelJson, e);
return null;
}
}
/**
* 문자열 → 정수 변환
*
* <p>- 파싱 실패 시 어떤 필드/파일에서 발생했는지 로그 기록 - 잘못된 데이터는 즉시 예외 처리
*/
private int parseInt(String value, String field, String fileName) {
try {
return Integer.parseInt(value);
} catch (NumberFormatException e) {
log.error("{} 파싱 실패. fileName={}, value={}", field, fileName, value);
throw new IllegalArgumentException(field + " 파싱 실패: " + fileName);
}
}
}

View File

@@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.node.ObjectNode;
import com.kamco.cd.training.common.enums.LearnDataRegister;
import com.kamco.cd.training.common.enums.LearnDataType; import com.kamco.cd.training.common.enums.LearnDataType;
import com.kamco.cd.training.common.exception.CustomApiException; import com.kamco.cd.training.common.exception.CustomApiException;
import com.kamco.cd.training.common.service.FormatStorage; import com.kamco.cd.training.common.service.FormatStorage;
@@ -11,6 +12,7 @@ import com.kamco.cd.training.common.utils.FIleChecker;
import com.kamco.cd.training.config.api.ApiResponseDto.ApiResponseCode; import com.kamco.cd.training.config.api.ApiResponseDto.ApiResponseCode;
import com.kamco.cd.training.config.api.ApiResponseDto.ResponseObj; import com.kamco.cd.training.config.api.ApiResponseDto.ResponseObj;
import com.kamco.cd.training.dataset.dto.DatasetDto; import com.kamco.cd.training.dataset.dto.DatasetDto;
import com.kamco.cd.training.dataset.dto.DatasetDto.AddDeliveriesReq;
import com.kamco.cd.training.dataset.dto.DatasetDto.AddReq; import com.kamco.cd.training.dataset.dto.DatasetDto.AddReq;
import com.kamco.cd.training.dataset.dto.DatasetDto.DatasetMngRegDto; import com.kamco.cd.training.dataset.dto.DatasetDto.DatasetMngRegDto;
import com.kamco.cd.training.dataset.dto.DatasetObjDto; import com.kamco.cd.training.dataset.dto.DatasetObjDto;
@@ -18,8 +20,11 @@ import com.kamco.cd.training.dataset.dto.DatasetObjDto.DatasetClass;
import com.kamco.cd.training.dataset.dto.DatasetObjDto.DatasetObjRegDto; import com.kamco.cd.training.dataset.dto.DatasetObjDto.DatasetObjRegDto;
import com.kamco.cd.training.dataset.dto.DatasetObjDto.DatasetStorage; import com.kamco.cd.training.dataset.dto.DatasetObjDto.DatasetStorage;
import com.kamco.cd.training.dataset.dto.DatasetObjDto.SearchReq; import com.kamco.cd.training.dataset.dto.DatasetObjDto.SearchReq;
import com.kamco.cd.training.model.dto.FileDto.FoldersDto;
import com.kamco.cd.training.model.dto.FileDto.SrchFoldersDto;
import com.kamco.cd.training.postgres.core.DatasetCoreService; import com.kamco.cd.training.postgres.core.DatasetCoreService;
import jakarta.validation.Valid; import jakarta.validation.Valid;
import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.nio.file.Files; import java.nio.file.Files;
@@ -27,6 +32,8 @@ import java.nio.file.Path;
import java.nio.file.Paths; import java.nio.file.Paths;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet; import java.util.HashSet;
import java.util.List; import java.util.List;
@@ -37,7 +44,6 @@ import java.util.stream.Collectors;
import java.util.stream.Stream; import java.util.stream.Stream;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.io.InputStreamResource; import org.springframework.core.io.InputStreamResource;
import org.springframework.core.io.Resource; import org.springframework.core.io.Resource;
import org.springframework.data.domain.Page; import org.springframework.data.domain.Page;
@@ -51,13 +57,10 @@ import org.springframework.transaction.annotation.Transactional;
@Slf4j @Slf4j
@Service @Service
@RequiredArgsConstructor @RequiredArgsConstructor
@Transactional
public class DatasetService { public class DatasetService {
private final DatasetCoreService datasetCoreService; private final DatasetCoreService datasetCoreService;
private final DatasetBatchService datasetBatchService;
@Value("${file.dataset-dir}")
private String datasetDir;
private static final List<String> LABEL_DIRS = List.of("label-json", "label", "input1", "input2"); private static final List<String> LABEL_DIRS = List.of("label-json", "label", "input1", "input2");
private static final List<String> REQUIRED_DIRS = Arrays.asList("train", "val", "test"); private static final List<String> REQUIRED_DIRS = Arrays.asList("train", "val", "test");
@@ -84,6 +87,7 @@ public class DatasetService {
return datasetCoreService.getOneByUuid(id); return datasetCoreService.getOneByUuid(id);
} }
// TODO 미사용시작
/** /**
* 데이터셋 등록 * 데이터셋 등록
* *
@@ -98,6 +102,7 @@ public class DatasetService {
return saved.getId(); return saved.getId();
} }
// TODO 미사용 끝
/** /**
* 데이터셋 수정 * 데이터셋 수정
* *
@@ -231,7 +236,7 @@ public class DatasetService {
return new ResponseObj(ApiResponseCode.INTERNAL_SERVER_ERROR, e.getMessage()); return new ResponseObj(ApiResponseCode.INTERNAL_SERVER_ERROR, e.getMessage());
} }
datasetCoreService.updateDatasetUploadStatus(datasetUid); datasetCoreService.updateDatasetUploadStatus(datasetUid, LearnDataRegister.COMPLETED);
return new ResponseObj(ApiResponseCode.OK, "업로드 성공하였습니다."); return new ResponseObj(ApiResponseCode.OK, "업로드 성공하였습니다.");
} }
@@ -365,7 +370,12 @@ public class DatasetService {
// 폴더별 처리 // 폴더별 처리
if ("label-json".equals(dirName)) { if ("label-json".equals(dirName)) {
// json 파일이면 파싱 // json 파일이면 파싱
data.put("label-json", readJson(path)); try {
data.put("label-json", readJson(path));
} catch (Exception e) {
log.error("파일 JSON 읽기 실패. skip. file={}", path, e);
return; // skip
}
data.put("geojson_path", path.toAbsolutePath().toString()); data.put("geojson_path", path.toAbsolutePath().toString());
} else { } else {
data.put(dirName, path.toAbsolutePath().toString()); data.put(dirName, path.toAbsolutePath().toString());
@@ -506,4 +516,164 @@ public class DatasetService {
} }
} }
} }
/**
* 폴더 조회
*
* @param srchDto 폴더 경로
* @return 폴더 리스트
* @throws IOException
*/
public FoldersDto getFolderAll(SrchFoldersDto srchDto) throws IOException {
File dir = new File(srchDto.getDirPath() == null ? "/" : srchDto.getDirPath());
// 존재 + 디렉토리 체크
if (!dir.exists() || !dir.isDirectory()) {
throw new CustomApiException("BAD_REQUEST", HttpStatus.BAD_REQUEST, "잘못된 경로입니다.");
}
// 권한 없을때
if (!dir.canRead()) {
throw new CustomApiException(
ApiResponseCode.FORBIDDEN.getId(), HttpStatus.FORBIDDEN, "디렉토리에 접근할 권한이 없습니다.");
}
String canonicalPath = dir.getCanonicalPath();
File[] files = dir.listFiles();
if (files == null) {
return new FoldersDto(canonicalPath, 0, 0, Collections.emptyList());
}
List<FIleChecker.Folder> folders = new ArrayList<>();
int folderTotCnt = 0;
int folderErrTotCnt = 0;
for (File f : files) {
// 숨김 제외
if (f.isHidden()) continue;
if (f.isDirectory()) {
// 폴더 개수 증가
folderTotCnt++;
// 폴더 유효성 여부 (기본 true, 이후 검증 로직으로 변경 가능)
boolean isValid = true;
// 유효하지 않은 폴더 카운트 증가
if (!isValid) folderErrTotCnt++;
// 현재 폴더 이름 (ex: train, images 등)
String folderNm = f.getName();
// 부모 경로 (ex: /data/datasets)
String parentPath = f.getParent();
// 부모 폴더 이름 (ex: datasets)
String parentFolderNm = new File(parentPath).getName();
// 전체 절대 경로 (ex: /data/datasets/train)
String fullPath = f.getAbsolutePath();
// 폴더 깊이 (경로 기준 depth)
// ex: /a/b/c → depth = 3
int depth = f.toPath().getNameCount();
// 하위 폴더 개수
long childCnt = FIleChecker.getChildFolderCount(f);
// 마지막 수정 시간 (문자열 포맷)
String lastModified = FIleChecker.getLastModified(f);
// Folder DTO 생성 및 리스트에 추가
folders.add(
new FIleChecker.Folder(
folderNm, // 폴더명
parentFolderNm, // 부모 폴더명
parentPath, // 부모 경로
fullPath, // 전체 경로
depth, // 깊이
childCnt, // 하위 폴더 개수
lastModified, // 수정일시
isValid // 유효성 여부
));
}
}
// 폴더 정렬
folders.sort(
Comparator.comparing(FIleChecker.Folder::getFolderNm, String.CASE_INSENSITIVE_ORDER));
return new FoldersDto(canonicalPath, folderTotCnt, folderErrTotCnt, folders);
}
/**
* 납품 데이터 등록
*
* @param req 폴더경로, 메모
* @return 성공/실패 여부0
*/
public void insertDeliveriesDataset(AddDeliveriesReq req, Long datasetUid) {
long startTime = System.currentTimeMillis();
// 처리
processType(req.getFilePath(), datasetUid, "train");
processType(req.getFilePath(), datasetUid, "val");
processType(req.getFilePath(), datasetUid, "test");
log.info("========== 전체 완료. 총 소요시간: {} ms ==========", System.currentTimeMillis() - startTime);
}
/**
* 납품 데이터 등록 처리
*
* @param path
* @param datasetUid
* @param type
*/
private void processType(String path, Long datasetUid, String type) {
long start = System.currentTimeMillis();
log.info("[납품 데이터 등록 처리][{}] 시작", type.toUpperCase());
List<Map<String, Object>> list = getUnzipDatasetFiles(path, type);
int batchSize = 1000;
int total = list.size();
int processed = 0;
for (int i = 0; i < total; i += batchSize) {
List<Map<String, Object>> batch = list.subList(i, Math.min(i + batchSize, total));
try {
log.info("[납품 데이터 등록 처리][{}] batch 시작: {} ~ {}", type, i, i + batch.size());
datasetBatchService.saveBatch(batch, datasetUid, type);
processed += batch.size();
} catch (Exception e) {
log.error("batch 실패 row 데이터: {}", batch);
log.error(
"[납품 데이터 등록 처리][{}] batch 실패. range: {} ~ {}, datasetUid={}",
type,
i,
i + batch.size(),
datasetUid,
e);
}
}
log.info(
"[납품 데이터 등록 처리][{}] 완료. 총 {}건, 소요시간: {} ms",
type,
total,
System.currentTimeMillis() - start);
}
} }

View File

@@ -1,3 +1,4 @@
// TODO 미사용시작
package com.kamco.cd.training.dataset.service; package com.kamco.cd.training.dataset.service;
import com.kamco.cd.training.dataset.dto.MapSheetDto; import com.kamco.cd.training.dataset.dto.MapSheetDto;
@@ -39,3 +40,4 @@ public class MapSheetService {
log.info("도엽 삭제 완료 - 개수: {}", deleteReq.getItemIds().size()); log.info("도엽 삭제 완료 - 개수: {}", deleteReq.getItemIds().size());
} }
} }
// TODO 미사용 끝

View File

@@ -11,9 +11,9 @@ import com.kamco.cd.training.model.dto.ModelTrainDetailDto.ModelTrainMetrics;
import com.kamco.cd.training.model.dto.ModelTrainDetailDto.ModelValidationMetrics; import com.kamco.cd.training.model.dto.ModelTrainDetailDto.ModelValidationMetrics;
import com.kamco.cd.training.model.dto.ModelTrainDetailDto.TransferDetailDto; import com.kamco.cd.training.model.dto.ModelTrainDetailDto.TransferDetailDto;
import com.kamco.cd.training.model.dto.ModelTrainMngDto.Basic; import com.kamco.cd.training.model.dto.ModelTrainMngDto.Basic;
import com.kamco.cd.training.model.dto.ModelTrainMngDto.CleanupResult;
import com.kamco.cd.training.model.dto.ModelTrainMngDto.ModelProgressStepDto; import com.kamco.cd.training.model.dto.ModelTrainMngDto.ModelProgressStepDto;
import com.kamco.cd.training.model.service.ModelTrainDetailService; import com.kamco.cd.training.model.service.ModelTrainDetailService;
import com.kamco.cd.training.model.service.ModelTrainMngService;
import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.enums.ParameterIn; import io.swagger.v3.oas.annotations.enums.ParameterIn;
@@ -34,6 +34,7 @@ import lombok.RequiredArgsConstructor;
import org.apache.coyote.BadRequestException; import org.apache.coyote.BadRequestException;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.ResponseEntity; import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMapping;
@@ -45,10 +46,9 @@ import org.springframework.web.bind.annotation.RestController;
@RequestMapping("/api/models") @RequestMapping("/api/models")
public class ModelTrainDetailApiController { public class ModelTrainDetailApiController {
private final ModelTrainDetailService modelTrainDetailService; private final ModelTrainDetailService modelTrainDetailService;
private final ModelTrainMngService modelTrainMngService;
private final RangeDownloadResponder rangeDownloadResponder; private final RangeDownloadResponder rangeDownloadResponder;
@Value("${train.docker.responseDir}") @Value("${train.docker.response_dir}")
private String responseDir; private String responseDir;
@Operation(summary = "모델학습관리> 모델관리 > 상세정보탭 > 학습 진행정보", description = "학습 진행정보, 모델학습 정보 API") @Operation(summary = "모델학습관리> 모델관리 > 상세정보탭 > 학습 진행정보", description = "학습 진행정보, 모델학습 정보 API")
@@ -326,4 +326,44 @@ public class ModelTrainDetailApiController {
UUID uuid) { UUID uuid) {
return ApiResponseDto.ok(modelTrainDetailService.findModelTrainProgressInfo(uuid)); return ApiResponseDto.ok(modelTrainDetailService.findModelTrainProgressInfo(uuid));
} }
@Operation(
summary = "모델관리 > 모델 상세 > best epoch 제외 삭제 될 파일 미리보기",
description = "best epoch 제외 삭제 될 파일 미리보기 API")
@ApiResponses(
value = {
@ApiResponse(
responseCode = "200",
description = "조회 성공",
content =
@Content(
mediaType = "application/json",
schema = @Schema(implementation = CleanupResult.class))),
@ApiResponse(responseCode = "404", description = "데이터셋을 찾을 수 없음", content = @Content),
@ApiResponse(responseCode = "500", description = "서버 오류", content = @Content)
})
@GetMapping("/{uuid}/cleanup/preview")
public ApiResponseDto<CleanupResult> previewCleanup(
@Parameter(description = "모델 uuid") @PathVariable UUID uuid) {
return ApiResponseDto.ok(modelTrainDetailService.previewCleanup(uuid));
}
@Operation(summary = "모델관리 > 모델 상세 > best epoch 제외 삭제", description = "best epoch 제외 파일 삭제 API")
@ApiResponses(
value = {
@ApiResponse(
responseCode = "200",
description = "삭제 성공",
content =
@Content(
mediaType = "application/json",
schema = @Schema(implementation = CleanupResult.class))),
@ApiResponse(responseCode = "404", description = "데이터셋을 찾을 수 없음", content = @Content),
@ApiResponse(responseCode = "500", description = "서버 오류", content = @Content)
})
@DeleteMapping("/{uuid}/cleanup")
public ApiResponseDto<CleanupResult> cleanup(
@Parameter(description = "모델 uuid") @PathVariable UUID uuid) {
return ApiResponseDto.ok(modelTrainDetailService.cleanup(uuid));
}
} }

View File

@@ -1,6 +1,7 @@
package com.kamco.cd.training.model; package com.kamco.cd.training.model;
import com.kamco.cd.training.common.dto.MonitorDto; import com.kamco.cd.training.common.dto.MonitorDto;
import com.kamco.cd.training.common.enums.ModelType;
import com.kamco.cd.training.common.service.SystemMonitorService; import com.kamco.cd.training.common.service.SystemMonitorService;
import com.kamco.cd.training.config.api.ApiResponseDto; import com.kamco.cd.training.config.api.ApiResponseDto;
import com.kamco.cd.training.dataset.dto.DatasetDto; import com.kamco.cd.training.dataset.dto.DatasetDto;
@@ -143,9 +144,9 @@ public class ModelTrainMngApiController {
@Parameter( @Parameter(
description = "모델 구분", description = "모델 구분",
example = "", example = "",
schema = @Schema(allowableValues = {"G1", "G2", "G3"})) schema = @Schema(allowableValues = {"G1", "G2", "G3", "G4"}))
@RequestParam @RequestParam
String modelType, ModelType modelType,
@Parameter( @Parameter(
description = "선택 구분", description = "선택 구분",
example = "", example = "",
@@ -153,7 +154,7 @@ public class ModelTrainMngApiController {
@RequestParam @RequestParam
String selectType) { String selectType) {
DatasetReq req = new DatasetReq(); DatasetReq req = new DatasetReq();
req.setModelNo(modelType); req.setModelNo(modelType.getId());
req.setDataType(selectType); req.setDataType(selectType);
return ApiResponseDto.ok(modelTrainMngService.getDatasetSelectList(req)); return ApiResponseDto.ok(modelTrainMngService.getDatasetSelectList(req));
} }

View File

@@ -0,0 +1,160 @@
package com.kamco.cd.training.model.dto;
import com.kamco.cd.training.common.utils.FIleChecker;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.constraints.NotNull;
import java.util.List;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
public class FileDto {
@Getter
@Setter
@NoArgsConstructor
@AllArgsConstructor
public static class SrchFoldersDto {
@Schema(description = "디렉토리경로(ROOT:/)", example = "")
@NotNull
private String dirPath = "/";
}
@Getter
@Setter
@NoArgsConstructor
@AllArgsConstructor
public static class SrchFilesDto {
@Schema(description = "디렉토리경로", example = "D:\\kamco\\2022\\캠코_2021_2022_34602060_D1")
@NotNull
private String dirPath;
@Schema(description = "전체(*), cpg,dbf,geojson등", example = "*")
@NotNull
private String extension;
@Schema(description = "파일명(name), 최종수정일(date)", example = "name")
@NotNull
private String sortType;
@Schema(description = "파일시작위치", example = "1")
@NotNull
private Integer startPos;
@Schema(description = "파일종료위치", example = "100")
@NotNull
private Integer endPos;
}
// TODO 미사용시작
@Getter
@Setter
@NoArgsConstructor
@AllArgsConstructor
public static class SrchFilesDepthDto extends SrchFilesDto {
@Schema(description = "최대폴더Depth", example = "5")
@NotNull
private Integer maxDepth;
}
@Schema(name = "FolderDto", description = "폴더 정보")
@Getter
public static class FolderDto {
private final String folderNm;
private final String parentFolderNm;
private final String parentPath;
private final String fullPath;
private final int depth;
private final long childCnt;
private final String lastModified;
private final Boolean isValid;
public FolderDto(
String folderNm,
String parentFolderNm,
String parentPath,
String fullPath,
int depth,
long childCnt,
String lastModified,
Boolean isValid) {
this.folderNm = folderNm;
this.parentFolderNm = parentFolderNm;
this.parentPath = parentPath;
this.fullPath = fullPath;
this.depth = depth;
this.childCnt = childCnt;
this.lastModified = lastModified;
this.isValid = isValid;
}
}
// TODO 미사용 끝
@Schema(name = "FoldersDto", description = "폴더목록 정보")
@Getter
public static class FoldersDto {
private final String dirPath;
private final int folderTotCnt;
private final int folderErrTotCnt;
private final List<FIleChecker.Folder> folders;
public FoldersDto(
String dirPath, int folderTotCnt, int folderErrTotCnt, List<FIleChecker.Folder> folders) {
this.dirPath = dirPath;
this.folderTotCnt = folderTotCnt;
this.folderErrTotCnt = folderErrTotCnt;
this.folders = folders;
}
}
@Schema(name = "File Basic", description = "파일 기본 정보")
@Getter
public static class Basic {
private final String fileNm;
private final String parentFolderNm;
private final String parentPath;
private final String fullPath;
private final String extension;
private final long fileSize;
private final String lastModified;
public Basic(
String fileNm,
String parentFolderNm,
String parentPath,
String fullPath,
String extension,
long fileSize,
String lastModified) {
this.fileNm = fileNm;
this.parentFolderNm = parentFolderNm;
this.parentPath = parentPath;
this.fullPath = fullPath;
this.extension = extension;
this.fileSize = fileSize;
this.lastModified = lastModified;
}
}
@Schema(name = "FilesDto", description = "파일 목록 정보")
@Getter
public static class FilesDto {
private final String dirPath;
private final int fileTotCnt;
private final long fileTotSize;
private final List<FIleChecker.Basic> files;
public FilesDto(
String dirPath, int fileTotCnt, long fileTotSize, List<FIleChecker.Basic> files) {
this.dirPath = dirPath;
this.fileTotCnt = fileTotCnt;
this.fileTotSize = fileTotSize;
this.files = files;
}
}
}

View File

@@ -48,6 +48,7 @@ public class ModelTrainMngDto {
private ZonedDateTime packingEndDttm; private ZonedDateTime packingEndDttm;
private Long beforeModelId; private Long beforeModelId;
private Integer bestEpoch;
public String getStatusName() { public String getStatusName() {
if (this.statusCd == null || this.statusCd.isBlank()) return null; if (this.statusCd == null || this.statusCd.isBlank()) return null;
@@ -327,4 +328,26 @@ public class ModelTrainMngDto {
@JsonFormatDttm private ZonedDateTime endTime; @JsonFormatDttm private ZonedDateTime endTime;
private boolean isError; private boolean isError;
} }
@Getter
@Setter
public static class CleanupResult {
// cleanup 대상 전체 파일 수 (삭제 대상 + 유지 파일 포함)
private int totalCount;
// 실제로 삭제된 파일 개수
private int deletedCount;
// 삭제 실패한 파일 개수
private int failedCount;
// 삭제 실패한 파일명 목록
private List<String> failedFiles;
// 유지된 파일명 (best epoch 기준)
private String keptFile;
// 삭제 될 파일
private List<String> deleteTargets;
}
} }

View File

@@ -1,6 +1,8 @@
package com.kamco.cd.training.model.service; package com.kamco.cd.training.model.service;
import com.kamco.cd.training.common.enums.ModelType; import com.kamco.cd.training.common.enums.ModelType;
import com.kamco.cd.training.common.enums.TrainStatusType;
import com.kamco.cd.training.common.exception.CustomApiException;
import com.kamco.cd.training.dataset.dto.DatasetDto.DatasetReq; import com.kamco.cd.training.dataset.dto.DatasetDto.DatasetReq;
import com.kamco.cd.training.dataset.dto.DatasetDto.SelectTransferDataSet; import com.kamco.cd.training.dataset.dto.DatasetDto.SelectTransferDataSet;
import com.kamco.cd.training.model.dto.ModelConfigDto; import com.kamco.cd.training.model.dto.ModelConfigDto;
@@ -14,15 +16,31 @@ import com.kamco.cd.training.model.dto.ModelTrainDetailDto.ModelTrainMetrics;
import com.kamco.cd.training.model.dto.ModelTrainDetailDto.ModelValidationMetrics; import com.kamco.cd.training.model.dto.ModelTrainDetailDto.ModelValidationMetrics;
import com.kamco.cd.training.model.dto.ModelTrainDetailDto.TransferDetailDto; import com.kamco.cd.training.model.dto.ModelTrainDetailDto.TransferDetailDto;
import com.kamco.cd.training.model.dto.ModelTrainDetailDto.TransferHyperSummary; import com.kamco.cd.training.model.dto.ModelTrainDetailDto.TransferHyperSummary;
import com.kamco.cd.training.model.dto.ModelTrainMngDto;
import com.kamco.cd.training.model.dto.ModelTrainMngDto.Basic; import com.kamco.cd.training.model.dto.ModelTrainMngDto.Basic;
import com.kamco.cd.training.model.dto.ModelTrainMngDto.CleanupResult;
import com.kamco.cd.training.model.dto.ModelTrainMngDto.ModelProgressStepDto; import com.kamco.cd.training.model.dto.ModelTrainMngDto.ModelProgressStepDto;
import com.kamco.cd.training.postgres.core.ModelTrainDetailCoreService; import com.kamco.cd.training.postgres.core.ModelTrainDetailCoreService;
import com.kamco.cd.training.postgres.core.ModelTrainMngCoreService; import com.kamco.cd.training.postgres.core.ModelTrainMngCoreService;
import java.io.IOException;
import java.nio.file.AccessDeniedException;
import java.nio.file.FileVisitOption;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.LinkOption;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.EnumSet;
import java.util.List; import java.util.List;
import java.util.UUID; import java.util.UUID;
import java.util.stream.Stream;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.annotation.Transactional;
@@ -35,6 +53,9 @@ public class ModelTrainDetailService {
private final ModelTrainDetailCoreService modelTrainDetailCoreService; private final ModelTrainDetailCoreService modelTrainDetailCoreService;
private final ModelTrainMngCoreService mngCoreService; private final ModelTrainMngCoreService mngCoreService;
@Value("${train.docker.response_dir}")
private String responseDir;
/** /**
* 모델 상세정보 요약 * 모델 상세정보 요약
* *
@@ -63,6 +84,7 @@ public class ModelTrainDetailService {
return modelTrainDetailCoreService.findByModelByUUID(uuid); return modelTrainDetailCoreService.findByModelByUUID(uuid);
} }
// TODO 미사용시작
/** /**
* 전이학습 모델선택 정보 * 전이학습 모델선택 정보
* *
@@ -129,6 +151,8 @@ public class ModelTrainDetailService {
return transferDetailDto; return transferDetailDto;
} }
// TODO 미사용 끝
public List<ModelTrainMetrics> getModelTrainMetricResult(UUID uuid) { public List<ModelTrainMetrics> getModelTrainMetricResult(UUID uuid) {
return modelTrainDetailCoreService.getModelTrainMetricResult(uuid); return modelTrainDetailCoreService.getModelTrainMetricResult(uuid);
} }
@@ -152,4 +176,262 @@ public class ModelTrainDetailService {
public List<ModelProgressStepDto> findModelTrainProgressInfo(UUID uuid) { public List<ModelProgressStepDto> findModelTrainProgressInfo(UUID uuid) {
return modelTrainDetailCoreService.findModelTrainProgressInfo(uuid); return modelTrainDetailCoreService.findModelTrainProgressInfo(uuid);
} }
/**
* 삭제될 파일목록 및 유지될 파일 목록
*
* @param uuid
* @return
*/
public CleanupResult previewCleanup(UUID uuid) {
CleanupResult result = new CleanupResult();
// ===== 모델 조회 =====
ModelTrainMngDto.Basic model = modelTrainDetailCoreService.findByModelByUUID(uuid);
if (model == null) {
throw new CustomApiException(
"NOT_FOUND_DATA", HttpStatus.NOT_FOUND, "모델을 찾을 수 없습니다. UUID: " + uuid);
}
Path dir = Paths.get(responseDir, model.getUuid().toString()).toAbsolutePath().normalize();
if (!Files.exists(dir) || !Files.isDirectory(dir)) {
throw new CustomApiException("NOT_FOUND_DATA", HttpStatus.NOT_FOUND, "디렉토리가 없습니다.");
}
if (!Files.isReadable(dir)) {
throw new CustomApiException("FORBIDDEN", HttpStatus.FORBIDDEN, "디렉토리 읽기 권한이 없습니다.");
}
try (Stream<Path> stream = Files.list(dir)) {
List<Path> files = stream.toList();
if (files.isEmpty()) {
throw new CustomApiException("NOT_FOUND_DATA", HttpStatus.NOT_FOUND, "파일이 없습니다.");
}
// ===== keep 파일 찾기 =====
Path keep =
files.stream()
.filter(
p -> {
String name = p.getFileName().toString();
return name.endsWith(".zip") && name.contains(model.getUuid().toString());
})
.findFirst()
.orElseThrow(
() ->
new CustomApiException(
"NOT_FOUND_DATA", HttpStatus.NOT_FOUND, "zip 파일이 없습니다."));
log.info("유지 파일: {}", keep.getFileName());
// ===== 결과 세팅 =====
result.setTotalCount(files.size());
result.setKeptFile(keep.getFileName().toString());
// ===== 삭제 대상 =====
List<String> deleteTargets =
files.stream()
.filter(
p -> !p.toAbsolutePath().normalize().equals(keep.toAbsolutePath().normalize()))
.map(p -> p.getFileName().toString())
.toList();
result.setDeleteTargets(deleteTargets);
log.info(
"previewCleanup 완료. total={}, deleteTargets={}",
result.getTotalCount(),
deleteTargets.size());
return result;
} catch (IOException e) {
log.error("파일 목록 조회 실패: {}", dir, e);
throw new CustomApiException(
"INTERNAL_SERVER_ERROR", HttpStatus.INTERNAL_SERVER_ERROR, "파일 목록 조회 실패");
}
}
public CleanupResult cleanup(UUID uuid) {
// ===== 모델 조회 =====
ModelTrainMngDto.Basic model = modelTrainDetailCoreService.findByModelByUUID(uuid);
if (model == null) {
throw new CustomApiException(
"NOT_FOUND_DATA", HttpStatus.NOT_FOUND, "모델을 찾을 수 없습니다. UUID: " + uuid);
}
if (!TrainStatusType.COMPLETED.getId().equals(model.getStep2Status())) {
throw new CustomApiException("CONFLICT", HttpStatus.CONFLICT, "테스트가 완료되지 않았습니다.");
}
// ===== 경로 =====
Path dir = Paths.get(responseDir, model.getUuid().toString()).toAbsolutePath().normalize();
return executeCleanup(model, dir);
}
/**
* 베스트 에폭 제외 파일 삭제, 베스트 에폭 zip 파일만 남김
*
* @param model model 정보
* @param dir response 폴더 경로
* @return 삭제 정보
*/
public CleanupResult executeCleanup(ModelTrainMngDto.Basic model, Path dir) {
CleanupResult result = new CleanupResult();
if (!Files.exists(dir) || !Files.isDirectory(dir)) {
throw new CustomApiException("NOT_FOUND_DATA", HttpStatus.NOT_FOUND, "디렉토리가 없습니다.");
}
if (!Files.isReadable(dir)) {
throw new CustomApiException("FORBIDDEN", HttpStatus.FORBIDDEN, "디렉토리 읽기 권한이 없습니다.");
}
if (!Files.isWritable(dir)) {
throw new CustomApiException("FORBIDDEN", HttpStatus.FORBIDDEN, "디렉토리 삭제 권한이 없습니다.");
}
int bestEpoch = model.getBestEpoch();
if (bestEpoch <= 0) {
throw new CustomApiException(
"BAD_REQUEST", HttpStatus.BAD_REQUEST, "잘못된 bestEpoch 값 입니다. : " + bestEpoch);
}
log.info("cleanup 시작. dir={}, bestEpoch={}", dir, bestEpoch);
try (Stream<Path> stream = Files.list(dir)) {
List<Path> files = stream.toList();
if (files.isEmpty()) {
throw new CustomApiException("NOT_FOUND_DATA", HttpStatus.NOT_FOUND, "파일이 없습니다.");
}
// ===== keep 파일 찾기 =====
Path keep = null;
for (Path p : files) {
String name = p.getFileName().toString();
if (name.endsWith(".zip") && name.contains(model.getUuid().toString())) {
keep = p;
break;
}
}
if (keep == null) {
throw new CustomApiException("NOT_FOUND_DATA", HttpStatus.NOT_FOUND, "zip 파일이 없습니다.");
}
log.info("유지 파일: {}", keep.getFileName());
result.setTotalCount(files.size());
result.setKeptFile(keep.getFileName().toString());
int deletedCount = 0;
List<String> failed = new ArrayList<>();
// ===== 삭제 =====
for (Path p : files) {
if (p.equals(keep)) {
continue;
}
try {
// 심볼릭 링크 → 링크만 삭제
if (Files.isSymbolicLink(p)) {
Files.deleteIfExists(p);
log.info("심볼릭 링크 삭제: {}", p.getFileName());
}
// 디렉토리 → 재귀 삭제
else if (Files.isDirectory(p, LinkOption.NOFOLLOW_LINKS)) {
log.info("디렉토리 재귀 삭제: {}", p.getFileName());
deleteDirectory(p);
}
// 일반 파일
else {
Files.deleteIfExists(p);
log.info("파일 삭제: {}", p.getFileName());
}
deletedCount++;
} catch (AccessDeniedException e) {
failed.add(p.getFileName().toString());
log.error("권한 없음: {}", p.getFileName(), e);
} catch (IOException e) {
failed.add(p.getFileName().toString());
log.error("삭제 실패: {}", p.getFileName(), e);
}
}
result.setDeletedCount(deletedCount);
result.setFailedCount(failed.size());
result.setFailedFiles(failed);
} catch (IOException e) {
log.error("파일 목록 조회 실패: {}", dir, e);
throw new CustomApiException(
"INTERNAL_SERVER_ERROR", HttpStatus.INTERNAL_SERVER_ERROR, "파일 목록 조회 실패");
}
log.info(
"cleanup 완료. total={}, deleted={}, failed={}",
result.getTotalCount(),
result.getDeletedCount(),
result.getFailedCount());
return result;
}
// 디렉토리 재귀 삭제
private void deleteDirectory(Path dir) throws IOException {
if (!Files.exists(dir, LinkOption.NOFOLLOW_LINKS)) {
return;
}
// dir 자체가 심볼릭 링크면 링크만 삭제
if (Files.isSymbolicLink(dir)) {
Files.delete(dir);
return;
}
Files.walkFileTree(
dir,
EnumSet.noneOf(FileVisitOption.class), // NOFOLLOW_LINKS
Integer.MAX_VALUE,
new SimpleFileVisitor<>() {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
throws IOException {
Files.delete(file); // 링크면 링크만 삭제됨
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult postVisitDirectory(Path directory, IOException exc)
throws IOException {
if (exc != null) throw exc;
Files.delete(directory);
return FileVisitResult.CONTINUE;
}
});
}
} }

View File

@@ -3,6 +3,7 @@ package com.kamco.cd.training.model.service;
import com.kamco.cd.training.common.dto.HyperParam; import com.kamco.cd.training.common.dto.HyperParam;
import com.kamco.cd.training.common.enums.HyperParamSelectType; import com.kamco.cd.training.common.enums.HyperParamSelectType;
import com.kamco.cd.training.common.enums.ModelType; import com.kamco.cd.training.common.enums.ModelType;
import com.kamco.cd.training.common.enums.TrainStatusType;
import com.kamco.cd.training.common.enums.TrainType; import com.kamco.cd.training.common.enums.TrainType;
import com.kamco.cd.training.common.exception.CustomApiException; import com.kamco.cd.training.common.exception.CustomApiException;
import com.kamco.cd.training.dataset.dto.DatasetDto.DatasetReq; import com.kamco.cd.training.dataset.dto.DatasetDto.DatasetReq;
@@ -14,10 +15,21 @@ import com.kamco.cd.training.model.dto.ModelTrainMngDto.SearchReq;
import com.kamco.cd.training.postgres.core.HyperParamCoreService; import com.kamco.cd.training.postgres.core.HyperParamCoreService;
import com.kamco.cd.training.postgres.core.ModelTrainMngCoreService; import com.kamco.cd.training.postgres.core.ModelTrainMngCoreService;
import com.kamco.cd.training.train.service.TrainJobService; import com.kamco.cd.training.train.service.TrainJobService;
import java.io.IOException;
import java.nio.file.FileVisitOption;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.LinkOption;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.EnumSet;
import java.util.List; import java.util.List;
import java.util.UUID; import java.util.UUID;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.data.domain.Page; import org.springframework.data.domain.Page;
import org.springframework.http.HttpStatus; import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
@@ -32,6 +44,16 @@ public class ModelTrainMngService {
private final ModelTrainMngCoreService modelTrainMngCoreService; private final ModelTrainMngCoreService modelTrainMngCoreService;
private final HyperParamCoreService hyperParamCoreService; private final HyperParamCoreService hyperParamCoreService;
private final TrainJobService trainJobService; private final TrainJobService trainJobService;
private final ModelTrainDetailService modelTrainDetailService;
@Value("${train.docker.base_path}")
private String basePath;
@Value("${train.docker.response_dir}")
private String responseDir;
@Value("${train.docker.symbolic_link_dir}")
private String symbolicDir;
/** /**
* 모델학습 조회 * 모델학습 조회
@@ -46,11 +68,195 @@ public class ModelTrainMngService {
/** /**
* 모델학습 삭제 * 모델학습 삭제
* *
* @param uuid * <p>순서: 1. tmp 구조 검증 (예외 발생 가능) 2. DB 삭제 (트랜잭션) 3. 파일 삭제 (실패해도 로그만)
*/ */
@Transactional @Transactional
public void deleteModelTrain(UUID uuid) { public void deleteModelTrain(UUID uuid) {
log.info("deleteModelTrain 시작. uuid={}", uuid);
// ===== 1. 모델 조회 =====
ModelTrainMngDto.Basic model = modelTrainMngCoreService.findModelByUuid(uuid);
if (model == null) {
throw new CustomApiException("NOT_FOUND", HttpStatus.NOT_FOUND, "모델 없음");
}
// ===== 2. 경로 생성 =====
Path tmpBase = Path.of(symbolicDir).toAbsolutePath().normalize();
Path tmp = tmpBase.resolve(model.getRequestPath()).normalize();
Path responseBase = Paths.get(responseDir).toAbsolutePath().normalize();
Path response = responseBase.resolve(model.getUuid().toString()).normalize();
// ===== 3. 경로 탈출 방지 =====
if (!tmp.startsWith(tmpBase)) {
throw new CustomApiException("INVALID_PATH", HttpStatus.BAD_REQUEST, "잘못된 tmp 경로");
}
if (!response.startsWith(responseBase)) {
throw new CustomApiException("INVALID_PATH", HttpStatus.BAD_REQUEST, "잘못된 response 경로");
}
// ===== 4. 상태 로그 =====
log.info(
"tmp 상태: exists={}, isDir={}, isSymlink={}",
Files.exists(tmp, LinkOption.NOFOLLOW_LINKS),
Files.isDirectory(tmp, LinkOption.NOFOLLOW_LINKS),
Files.isSymbolicLink(tmp));
log.info(
"response 상태: exists={}, isDir={}, isSymlink={}",
Files.exists(response, LinkOption.NOFOLLOW_LINKS),
Files.isDirectory(response, LinkOption.NOFOLLOW_LINKS),
Files.isSymbolicLink(response));
// ===== 5. tmp 구조 검증 =====
validateTmpStructure(tmp);
// ===== 6. DB 삭제 =====
modelTrainMngCoreService.deleteModel(uuid); modelTrainMngCoreService.deleteModel(uuid);
log.info("DB 삭제 완료. uuid={}", uuid);
// ===== 7. tmp 삭제 =====
log.info("tmp 삭제 시작: {}", tmp);
try {
deleteTmpDirectory(tmp);
log.info("tmp 삭제 완료: {}", tmp);
} catch (Exception e) {
log.error("tmp 삭제 실패 (DB는 이미 삭제됨): {}", tmp, e);
}
// ===== 8. response 삭제 =====
log.info("response 삭제 시작: {}", response);
try {
// 테스트 완료되었으면 베스트 에폭은 삭제안함
if (TrainStatusType.COMPLETED.getId().equals(model.getStep2Status())) {
modelTrainDetailService.executeCleanup(model, response);
} else {
deleteResponseDirectory(response);
}
log.info("response 삭제 완료: {}", response);
} catch (Exception e) {
log.error("response 삭제 실패 (DB는 이미 삭제됨): {}", response, e);
}
log.info("deleteModelTrain 완료. uuid={}", uuid);
}
/** tmp 디렉토리 삭제 */
private void deleteTmpDirectory(Path dir) throws IOException {
if (!Files.exists(dir, LinkOption.NOFOLLOW_LINKS)) {
log.warn("삭제 대상 없음: {}", dir);
return;
}
Files.walkFileTree(
dir,
EnumSet.noneOf(FileVisitOption.class),
Integer.MAX_VALUE,
new SimpleFileVisitor<>() {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
throws IOException {
Files.delete(file);
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult postVisitDirectory(Path directory, IOException exc)
throws IOException {
if (exc != null) {
throw exc;
}
Files.delete(directory);
return FileVisitResult.CONTINUE;
}
});
}
/** response 디렉토리 삭제 */
private void deleteResponseDirectory(Path dir) throws IOException {
if (!Files.exists(dir, LinkOption.NOFOLLOW_LINKS)) {
log.warn("삭제 대상 없음: {}", dir);
return;
}
Files.walkFileTree(
dir,
EnumSet.noneOf(FileVisitOption.class),
Integer.MAX_VALUE,
new SimpleFileVisitor<>() {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
throws IOException {
Files.delete(file);
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult postVisitDirectory(Path directory, IOException exc)
throws IOException {
if (exc != null) {
throw exc;
}
Files.delete(directory);
return FileVisitResult.CONTINUE;
}
});
}
/** tmp 내부 구조 검증 - 내부는 반드시 symlink만 허용 */
private void validateTmpStructure(Path dir) {
if (!Files.exists(dir, LinkOption.NOFOLLOW_LINKS)) {
return;
}
try {
Files.walkFileTree(
dir,
EnumSet.noneOf(FileVisitOption.class),
Integer.MAX_VALUE,
new SimpleFileVisitor<>() {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
throws IOException {
// 파일은 전부 허용 (일반 + symlink)
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult preVisitDirectory(Path directory, BasicFileAttributes attrs)
throws IOException {
// 루트 제외 + symlink 디렉토리 금지
if (!directory.equals(dir) && Files.isSymbolicLink(directory)) {
log.error("tmp 내부에 symlink 디렉토리 존재: {}", directory);
throw new CustomApiException(
"BAD_REQUEST", HttpStatus.BAD_REQUEST, "tmp 내부에 symlink 디렉토리는 허용되지 않습니다.");
}
return FileVisitResult.CONTINUE;
}
});
} catch (IOException e) {
throw new CustomApiException(
"INTERNAL_SERVER_ERROR", HttpStatus.INTERNAL_SERVER_ERROR, "tmp 구조 검증 실패");
}
} }
/** /**
@@ -118,6 +324,8 @@ public class ModelTrainMngService {
public List<SelectDataSet> getDatasetSelectList(DatasetReq req) { public List<SelectDataSet> getDatasetSelectList(DatasetReq req) {
if (req.getModelNo().equals(ModelType.G1.getId())) { if (req.getModelNo().equals(ModelType.G1.getId())) {
return modelTrainMngCoreService.getDatasetSelectG1List(req); return modelTrainMngCoreService.getDatasetSelectG1List(req);
} else if (req.getModelNo().equals(ModelType.G4.getId())) {
return modelTrainMngCoreService.getDatasetSelectG4List(req);
} else { } else {
return modelTrainMngCoreService.getDatasetSelectG2G3List(req); return modelTrainMngCoreService.getDatasetSelectG2G3List(req);
} }

View File

@@ -1,6 +1,5 @@
package com.kamco.cd.training.postgres.core; package com.kamco.cd.training.postgres.core;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.kamco.cd.training.common.enums.LearnDataRegister; import com.kamco.cd.training.common.enums.LearnDataRegister;
import com.kamco.cd.training.common.enums.LearnDataType; import com.kamco.cd.training.common.enums.LearnDataType;
import com.kamco.cd.training.common.exception.NotFoundException; import com.kamco.cd.training.common.exception.NotFoundException;
@@ -30,9 +29,9 @@ import org.springframework.stereotype.Service;
@Slf4j @Slf4j
public class DatasetCoreService public class DatasetCoreService
implements BaseCoreService<DatasetDto.Basic, Long, DatasetDto.SearchReq> { implements BaseCoreService<DatasetDto.Basic, Long, DatasetDto.SearchReq> {
private final DatasetRepository datasetRepository; private final DatasetRepository datasetRepository;
private final DatasetObjRepository datasetObjRepository; private final DatasetObjRepository datasetObjRepository;
private final ObjectMapper objectMapper;
/** /**
* 학습 데이터 삭제 * 학습 데이터 삭제
@@ -96,6 +95,7 @@ public class DatasetCoreService
return search(searchReq); return search(searchReq);
} }
// TODO 미사용시작
/** /**
* 학습데이터 등록 * 학습데이터 등록
* *
@@ -130,6 +130,7 @@ public class DatasetCoreService
return savedEntity.toDto(); return savedEntity.toDto();
} }
// TODO 미사용 끝
/** /**
* 학습 데이터 수정 * 학습 데이터 수정
* *
@@ -221,7 +222,6 @@ public class DatasetCoreService
return datasetRepository.insertDatasetMngData(mngRegDto); return datasetRepository.insertDatasetMngData(mngRegDto);
} }
@Transactional
public void insertDatasetObj(DatasetObjRegDto objRegDto) { public void insertDatasetObj(DatasetObjRegDto objRegDto) {
datasetObjRepository.insertDatasetObj(objRegDto); datasetObjRepository.insertDatasetObj(objRegDto);
} }
@@ -234,13 +234,20 @@ public class DatasetCoreService
datasetObjRepository.insertDatasetTestObj(objRegDto); datasetObjRepository.insertDatasetTestObj(objRegDto);
} }
public void updateDatasetUploadStatus(Long datasetUid) { /**
* 학습데이터셋 마스터 상태 변경
*
* @param datasetUid 학습데이터셋 마스터 id
* @param register 상태
*/
@Transactional
public void updateDatasetUploadStatus(Long datasetUid, LearnDataRegister register) {
DatasetEntity entity = DatasetEntity entity =
datasetRepository datasetRepository
.findById(datasetUid) .findById(datasetUid)
.orElseThrow(() -> new NotFoundException("데이터셋을 찾을 수 없습니다. ID: " + datasetUid)); .orElseThrow(() -> new NotFoundException("데이터셋을 찾을 수 없습니다. ID: " + datasetUid));
entity.setStatus(LearnDataRegister.COMPLETED.getId()); entity.setStatus(register.getId());
} }
public void insertDatasetValObj(DatasetObjRegDto objRegDto) { public void insertDatasetValObj(DatasetObjRegDto objRegDto) {
@@ -250,4 +257,15 @@ public class DatasetCoreService
public Long findDatasetByUidExistsCnt(String uid) { public Long findDatasetByUidExistsCnt(String uid) {
return datasetRepository.findDatasetByUidExistsCnt(uid); return datasetRepository.findDatasetByUidExistsCnt(uid);
} }
/**
* 데이터셋 등록 실패시 Obj 데이터 정리
*
* @param datasetUid 모델 마스터 id
*/
@Transactional
public void deleteAllDatasetObj(Long datasetUid) {
int cnt = datasetObjRepository.deleteAllDatasetObj(datasetUid);
log.info("datasetUid={} 데이터셋 실패 - 전체 삭제 완료. 총 {}건", datasetUid, cnt);
}
} }

View File

@@ -1,3 +1,4 @@
// TODO 미사용시작
package com.kamco.cd.training.postgres.core; package com.kamco.cd.training.postgres.core;
import com.kamco.cd.training.common.exception.NotFoundException; import com.kamco.cd.training.common.exception.NotFoundException;
@@ -70,3 +71,4 @@ public class MapSheetCoreService
} }
} }
} }
// TODO 미사용 끝

View File

@@ -67,10 +67,13 @@ public class ModelTrainDetailCoreService {
return modelDetailRepository.getByModelHyperParamSummary(uuid); return modelDetailRepository.getByModelHyperParamSummary(uuid);
} }
// TODO 미사용시작
public TransferHyperSummary getTransferHyperSummary(UUID uuid) { public TransferHyperSummary getTransferHyperSummary(UUID uuid) {
return modelDetailRepository.getByModelTransferHyperParamSummary(uuid); return modelDetailRepository.getByModelTransferHyperParamSummary(uuid);
} }
// TODO 미사용 끝
public List<MappingDataset> getByModelMappingDataset(UUID uuid) { public List<MappingDataset> getByModelMappingDataset(UUID uuid) {
return modelDetailRepository.getByModelMappingDataset(uuid); return modelDetailRepository.getByModelMappingDataset(uuid);
} }
@@ -80,6 +83,7 @@ public class ModelTrainDetailCoreService {
return entity.toDto(); return entity.toDto();
} }
// TODO 미사용시작
/** /**
* 모델 학습별 config 정보 조회 * 모델 학습별 config 정보 조회
* *
@@ -90,6 +94,7 @@ public class ModelTrainDetailCoreService {
return modelConfigRepository.findModelConfigByModelId(modelId).orElse(null); return modelConfigRepository.findModelConfigByModelId(modelId).orElse(null);
} }
// TODO 미사용 끝
public List<ModelTrainMetrics> getModelTrainMetricResult(UUID uuid) { public List<ModelTrainMetrics> getModelTrainMetricResult(UUID uuid) {
return modelDetailRepository.getModelTrainMetricResult(uuid); return modelDetailRepository.getModelTrainMetricResult(uuid);
} }

View File

@@ -219,6 +219,7 @@ public class ModelTrainMngCoreService {
modelConfigRepository.save(entity); modelConfigRepository.save(entity);
} }
// TODO 미사용시작
/** /**
* 데이터셋 매핑 생성 * 데이터셋 매핑 생성
* *
@@ -235,6 +236,8 @@ public class ModelTrainMngCoreService {
} }
} }
// TODO 미사용 끝
/** /**
* UUID로 모델 조회 * UUID로 모델 조회
* *
@@ -278,6 +281,7 @@ public class ModelTrainMngCoreService {
.orElseThrow(() -> new CustomApiException("NOT_FOUND_DATA", HttpStatus.NOT_FOUND)); .orElseThrow(() -> new CustomApiException("NOT_FOUND_DATA", HttpStatus.NOT_FOUND));
} }
// TODO 미사용시작
public ModelConfigDto.TransferBasic findModelTransferConfigByModelId(UUID uuid) { public ModelConfigDto.TransferBasic findModelTransferConfigByModelId(UUID uuid) {
ModelMasterEntity modelEntity = findByUuid(uuid); ModelMasterEntity modelEntity = findByUuid(uuid);
return modelConfigRepository return modelConfigRepository
@@ -285,6 +289,8 @@ public class ModelTrainMngCoreService {
.orElseThrow(() -> new CustomApiException("NOT_FOUND_DATA", HttpStatus.NOT_FOUND)); .orElseThrow(() -> new CustomApiException("NOT_FOUND_DATA", HttpStatus.NOT_FOUND));
} }
// TODO 미사용 끝
/** /**
* 데이터셋 G1 목록 * 데이터셋 G1 목록
* *
@@ -295,6 +301,7 @@ public class ModelTrainMngCoreService {
return datasetRepository.getDatasetSelectG1List(req); return datasetRepository.getDatasetSelectG1List(req);
} }
// TODO 미사용시작
/** /**
* 전이학습 데이터셋 G1 목록 * 전이학습 데이터셋 G1 목록
* *
@@ -305,6 +312,8 @@ public class ModelTrainMngCoreService {
return datasetRepository.getDatasetTransferSelectG1List(modelId); return datasetRepository.getDatasetTransferSelectG1List(modelId);
} }
// TODO 미사용 끝
/** /**
* 데이터셋 G2, G3 목록 * 데이터셋 G2, G3 목록
* *
@@ -315,6 +324,7 @@ public class ModelTrainMngCoreService {
return datasetRepository.getDatasetSelectG2G3List(req); return datasetRepository.getDatasetSelectG2G3List(req);
} }
// TODO 미사용시작
/** /**
* 전이학습 데이터셋 G2, G3 목록 * 전이학습 데이터셋 G2, G3 목록
* *
@@ -327,6 +337,18 @@ public class ModelTrainMngCoreService {
return datasetRepository.getDatasetTransferSelectG2G3List(modelId, modelNo); return datasetRepository.getDatasetTransferSelectG2G3List(modelId, modelNo);
} }
/**
* 데이터셋 G4 목록
*
* @param req
* @return
*/
public List<SelectDataSet> getDatasetSelectG4List(DatasetReq req) {
return datasetRepository.getDatasetSelectG4List(req);
}
// TODO 미사용 끝
/** /**
* 모델관리 조회 * 모델관리 조회
* *
@@ -341,6 +363,20 @@ public class ModelTrainMngCoreService {
return entity.toDto(); return entity.toDto();
} }
/**
* 모델관리 조회
*
* @param uuid
* @return
*/
public ModelTrainMngDto.Basic findModelByUuid(UUID uuid) {
ModelMasterEntity entity =
modelMngRepository
.findByUuid(uuid)
.orElseThrow(() -> new IllegalArgumentException("Model not found: " + uuid));
return entity.toDto();
}
/** 마스터를 IN_PROGRESS로 전환하고, 현재 실행 jobId를 연결 - UI/중단/상태조회 모두 currentAttemptId를 기준으로 동작 */ /** 마스터를 IN_PROGRESS로 전환하고, 현재 실행 jobId를 연결 - UI/중단/상태조회 모두 currentAttemptId를 기준으로 동작 */
@Transactional @Transactional
public void markInProgress(Long modelId, Long jobId) { public void markInProgress(Long modelId, Long jobId) {

View File

@@ -1,3 +1,4 @@
// TODO 미사용시작
package com.kamco.cd.training.postgres.core; package com.kamco.cd.training.postgres.core;
import com.kamco.cd.training.postgres.entity.SystemMetricsEntity; import com.kamco.cd.training.postgres.entity.SystemMetricsEntity;
@@ -64,3 +65,4 @@ public class SystemMetricsCoreService {
return isAvailable; return isAvailable;
} }
} }
// TODO 미사용 끝

View File

@@ -117,10 +117,12 @@ public class DatasetEntity {
@Column(name = "dataset_path", length = 1000) @Column(name = "dataset_path", length = 1000)
private String datasetPath; private String datasetPath;
// TODO 미사용시작
@Column(name = "class_counts") @Column(name = "class_counts")
@JdbcTypeCode(SqlTypes.JSON) @JdbcTypeCode(SqlTypes.JSON)
private Map<String, Integer> classCounts; private Map<String, Integer> classCounts;
// TODO 미사용 끝
@Size(max = 32) @Size(max = 32)
@Column(name = "uid") @Column(name = "uid")
private String uid; private String uid;

View File

@@ -1,3 +1,4 @@
// TODO 미사용시작
package com.kamco.cd.training.postgres.entity; package com.kamco.cd.training.postgres.entity;
import com.kamco.cd.training.dataset.dto.MapSheetDto; import com.kamco.cd.training.dataset.dto.MapSheetDto;
@@ -103,3 +104,4 @@ public class MapSheetEntity {
return dto; return dto;
} }
} }
// TODO 미사용 끝

View File

@@ -32,10 +32,13 @@ public class ModelDatasetMappEntity {
@Column(name = "dataset_uid", nullable = false) @Column(name = "dataset_uid", nullable = false)
private Long datasetUid; private Long datasetUid;
// TODO 미사용시작
@Size(max = 20) @Size(max = 20)
@Column(name = "dataset_type", length = 20) @Column(name = "dataset_type", length = 20)
private String datasetType; private String datasetType;
// TODO 미사용 끝
@Getter @Getter
@Setter @Setter
@NoArgsConstructor @NoArgsConstructor

View File

@@ -141,6 +141,7 @@ public class ModelMasterEntity {
this.packingState, this.packingState,
this.packingStrtDttm, this.packingStrtDttm,
this.packingEndDttm, this.packingEndDttm,
this.beforeModelId); this.beforeModelId,
this.bestEpoch);
} }
} }

View File

@@ -1,3 +1,4 @@
// TODO 미사용시작
package com.kamco.cd.training.postgres.entity; package com.kamco.cd.training.postgres.entity;
import jakarta.persistence.Column; import jakarta.persistence.Column;
@@ -92,3 +93,4 @@ public class ModelMngEntity {
return this.uuid != null ? this.uuid.toString() : null; return this.uuid != null ? this.uuid.toString() : null;
} }
} }
// TODO 미사용 끝

View File

@@ -1,3 +1,4 @@
// TODO 미사용시작
package com.kamco.cd.training.postgres.entity; package com.kamco.cd.training.postgres.entity;
import jakarta.persistence.Column; import jakarta.persistence.Column;
@@ -53,3 +54,4 @@ public class SystemMetricsEntity {
@Column(name = "memused") @Column(name = "memused")
private Float memused; private Float memused;
} }
// TODO 미사용 끝

View File

@@ -1,3 +1,4 @@
// TODO 미사용시작
package com.kamco.cd.training.postgres.repository; package com.kamco.cd.training.postgres.repository;
import com.kamco.cd.training.postgres.entity.SystemMetricsEntity; import com.kamco.cd.training.postgres.entity.SystemMetricsEntity;
@@ -17,3 +18,4 @@ public interface SystemMetricsRepository extends JpaRepository<SystemMetricsEnti
@Query("SELECT s FROM SystemMetricsEntity s ORDER BY s.timestamp DESC LIMIT 1") @Query("SELECT s FROM SystemMetricsEntity s ORDER BY s.timestamp DESC LIMIT 1")
Optional<SystemMetricsEntity> findLatestMetrics(); Optional<SystemMetricsEntity> findLatestMetrics();
} }
// TODO 미사용 끝

View File

@@ -24,4 +24,12 @@ public interface DatasetObjRepositoryCustom {
void insertDatasetTestObj(DatasetObjRegDto objRegDto); void insertDatasetTestObj(DatasetObjRegDto objRegDto);
void insertDatasetValObj(DatasetObjRegDto objRegDto); void insertDatasetValObj(DatasetObjRegDto objRegDto);
/**
* 데이터셋 등록 실패시 Obj 데이터 정리
*
* @param datasetUid
* @return
*/
int deleteAllDatasetObj(Long datasetUid);
} }

View File

@@ -40,6 +40,7 @@ public class DatasetObjRepositoryImpl implements DatasetObjRepositoryCustom {
private final JPAQueryFactory queryFactory; private final JPAQueryFactory queryFactory;
private final QDatasetEntity dataset = datasetEntity; private final QDatasetEntity dataset = datasetEntity;
private final ObjectMapper objectMapper = new ObjectMapper();
@PersistenceContext EntityManager em; @PersistenceContext EntityManager em;
@@ -55,7 +56,6 @@ public class DatasetObjRepositoryImpl implements DatasetObjRepositoryCustom {
@Override @Override
public void insertDatasetTestObj(DatasetObjRegDto objRegDto) { public void insertDatasetTestObj(DatasetObjRegDto objRegDto) {
ObjectMapper objectMapper = new ObjectMapper();
String json; String json;
Geometry geometry; Geometry geometry;
String geometryJson; String geometryJson;
@@ -99,7 +99,6 @@ public class DatasetObjRepositoryImpl implements DatasetObjRepositoryCustom {
@Override @Override
public void insertDatasetValObj(DatasetObjRegDto objRegDto) { public void insertDatasetValObj(DatasetObjRegDto objRegDto) {
ObjectMapper objectMapper = new ObjectMapper();
String json; String json;
String geometryJson; String geometryJson;
try { try {
@@ -219,7 +218,6 @@ public class DatasetObjRepositoryImpl implements DatasetObjRepositoryCustom {
@Override @Override
public void insertDatasetObj(DatasetObjRegDto objRegDto) { public void insertDatasetObj(DatasetObjRegDto objRegDto) {
ObjectMapper objectMapper = new ObjectMapper();
String json; String json;
String geometryJson; String geometryJson;
try { try {
@@ -276,4 +274,38 @@ public class DatasetObjRepositoryImpl implements DatasetObjRepositoryCustom {
.where(datasetObjEntity.uuid.eq(uuid)) .where(datasetObjEntity.uuid.eq(uuid))
.fetchOne(); .fetchOne();
} }
@Override
public int deleteAllDatasetObj(Long datasetUid) {
int cnt = 0;
cnt =
em.createNativeQuery(
"""
delete from tb_dataset_obj
where dataset_uid = ?
""")
.setParameter(1, datasetUid)
.executeUpdate();
cnt +=
em.createNativeQuery(
"""
delete from tb_dataset_val_obj
where dataset_uid = ?
""")
.setParameter(1, datasetUid)
.executeUpdate();
cnt +=
em.createNativeQuery(
"""
delete from tb_dataset_test_obj
where dataset_uid = ?
""")
.setParameter(1, datasetUid)
.executeUpdate();
em.clear();
return cnt;
}
} }

View File

@@ -6,6 +6,7 @@ import org.springframework.data.jpa.repository.JpaRepository;
public interface DatasetRepository public interface DatasetRepository
extends JpaRepository<DatasetEntity, Long>, DatasetRepositoryCustom { extends JpaRepository<DatasetEntity, Long>, DatasetRepositoryCustom {
// TODO 미사용시작
List<DatasetEntity> findByDeletedOrderByCreatedDttmDesc(Boolean deleted); List<DatasetEntity> findByDeletedOrderByCreatedDttmDesc(Boolean deleted);
// TODO 미사용 끝
} }

View File

@@ -18,12 +18,17 @@ public interface DatasetRepositoryCustom {
List<SelectDataSet> getDatasetSelectG1List(DatasetReq req); List<SelectDataSet> getDatasetSelectG1List(DatasetReq req);
// TODO 미사용시작
public List<SelectTransferDataSet> getDatasetTransferSelectG1List(Long modelId); public List<SelectTransferDataSet> getDatasetTransferSelectG1List(Long modelId);
public List<SelectTransferDataSet> getDatasetTransferSelectG2G3List(Long modelId, String modelNo); public List<SelectTransferDataSet> getDatasetTransferSelectG2G3List(Long modelId, String modelNo);
// TODO 미사용 끝
List<SelectDataSet> getDatasetSelectG2G3List(DatasetReq req); List<SelectDataSet> getDatasetSelectG2G3List(DatasetReq req);
List<SelectDataSet> getDatasetSelectG4List(DatasetReq req);
Long getDatasetMaxStage(int compareYyyy, int targetYyyy); Long getDatasetMaxStage(int compareYyyy, int targetYyyy);
Long insertDatasetMngData(DatasetMngRegDto mngRegDto); Long insertDatasetMngData(DatasetMngRegDto mngRegDto);

View File

@@ -4,6 +4,7 @@ import static com.kamco.cd.training.postgres.entity.QDatasetObjEntity.datasetObj
import static com.kamco.cd.training.postgres.entity.QModelDatasetMappEntity.modelDatasetMappEntity; import static com.kamco.cd.training.postgres.entity.QModelDatasetMappEntity.modelDatasetMappEntity;
import static com.kamco.cd.training.postgres.entity.QModelMasterEntity.modelMasterEntity; import static com.kamco.cd.training.postgres.entity.QModelMasterEntity.modelMasterEntity;
import com.kamco.cd.training.common.enums.DetectionClassification;
import com.kamco.cd.training.common.enums.ModelType; import com.kamco.cd.training.common.enums.ModelType;
import com.kamco.cd.training.dataset.dto.DatasetDto.DatasetMngRegDto; import com.kamco.cd.training.dataset.dto.DatasetDto.DatasetMngRegDto;
import com.kamco.cd.training.dataset.dto.DatasetDto.DatasetReq; import com.kamco.cd.training.dataset.dto.DatasetDto.DatasetReq;
@@ -104,10 +105,6 @@ public class DatasetRepositoryImpl implements DatasetRepositoryCustom {
builder.and(dataset.dataType.eq(req.getDataType())); builder.and(dataset.dataType.eq(req.getDataType()));
} }
if (StringUtils.isNotBlank(req.getDataType()) && !"CURRENT".equals(req.getDataType())) {
builder.and(dataset.dataType.eq(req.getDataType()));
}
if (req.getIds() != null) { if (req.getIds() != null) {
builder.and(dataset.id.in(req.getIds())); builder.and(dataset.id.in(req.getIds()));
} }
@@ -126,12 +123,15 @@ public class DatasetRepositoryImpl implements DatasetRepositoryCustom {
dataset.targetYyyy, dataset.targetYyyy,
dataset.memo, dataset.memo,
new CaseBuilder() new CaseBuilder()
.when(datasetObjEntity.targetClassCd.eq("building")) .when(
datasetObjEntity.targetClassCd.eq(DetectionClassification.BUILDING.getId()))
.then(1) .then(1)
.otherwise(0) .otherwise(0)
.sum(), .sum(),
new CaseBuilder() new CaseBuilder()
.when(datasetObjEntity.targetClassCd.eq("container")) .when(
datasetObjEntity.targetClassCd.eq(
DetectionClassification.CONTAINER.getId()))
.then(1) .then(1)
.otherwise(0) .otherwise(0)
.sum())) .sum()))
@@ -150,6 +150,7 @@ public class DatasetRepositoryImpl implements DatasetRepositoryCustom {
.fetch(); .fetch();
} }
// TODO 미사용시작
@Override @Override
public List<SelectTransferDataSet> getDatasetTransferSelectG1List(Long modelId) { public List<SelectTransferDataSet> getDatasetTransferSelectG1List(Long modelId) {
@@ -247,19 +248,32 @@ public class DatasetRepositoryImpl implements DatasetRepositoryCustom {
.fetch(); .fetch();
} }
// TODO 미사용 끝
@Override @Override
public List<SelectDataSet> getDatasetSelectG2G3List(DatasetReq req) { public List<SelectDataSet> getDatasetSelectG2G3List(DatasetReq req) {
String building = DetectionClassification.BUILDING.getId();
String container = DetectionClassification.CONTAINER.getId();
String waste = DetectionClassification.WASTE.getId();
String solar = DetectionClassification.SOLAR.getId();
BooleanBuilder builder = new BooleanBuilder(); BooleanBuilder builder = new BooleanBuilder();
builder.and(dataset.deleted.isFalse()); builder.and(dataset.deleted.isFalse());
NumberExpression<Long> selectedCnt = null; NumberExpression<Long> selectedCnt = null;
NumberExpression<Long> wasteCnt = NumberExpression<Long> wasteCnt =
datasetObjEntity.targetClassCd.when("waste").then(1L).otherwise(0L).sum(); datasetObjEntity
.targetClassCd
.when(DetectionClassification.WASTE.getId())
.then(1L)
.otherwise(0L)
.sum();
// G1, G2, G4 제외
NumberExpression<Long> elseCnt = NumberExpression<Long> elseCnt =
new CaseBuilder() new CaseBuilder()
.when(datasetObjEntity.targetClassCd.notIn("building", "container", "waste")) .when(datasetObjEntity.targetClassCd.notIn(building, container, waste, solar))
.then(1L) .then(1L)
.otherwise(0L) .otherwise(0L)
.sum(); .sum();
@@ -311,6 +325,7 @@ public class DatasetRepositoryImpl implements DatasetRepositoryCustom {
.fetch(); .fetch();
} }
// TODO 미사용시작
@Override @Override
public List<SelectTransferDataSet> getDatasetTransferSelectG2G3List( public List<SelectTransferDataSet> getDatasetTransferSelectG2G3List(
Long modelId, String modelNo) { Long modelId, String modelNo) {
@@ -421,6 +436,8 @@ public class DatasetRepositoryImpl implements DatasetRepositoryCustom {
.fetch(); .fetch();
} }
// TODO 미사용 끝
@Override @Override
public Long getDatasetMaxStage(int compareYyyy, int targetYyyy) { public Long getDatasetMaxStage(int compareYyyy, int targetYyyy) {
return queryFactory return queryFactory
@@ -476,4 +493,53 @@ public class DatasetRepositoryImpl implements DatasetRepositoryCustom {
.where(dataset.uid.eq(uid), dataset.deleted.isFalse()) .where(dataset.uid.eq(uid), dataset.deleted.isFalse())
.fetchOne(); .fetchOne();
} }
@Override
public List<SelectDataSet> getDatasetSelectG4List(DatasetReq req) {
BooleanBuilder builder = new BooleanBuilder();
builder.and(dataset.deleted.isFalse());
if (StringUtils.isNotBlank(req.getDataType()) && !"CURRENT".equals(req.getDataType())) {
builder.and(dataset.dataType.eq(req.getDataType()));
}
if (req.getIds() != null) {
builder.and(dataset.id.in(req.getIds()));
}
return queryFactory
.select(
Projections.constructor(
SelectDataSet.class,
Expressions.constant(req.getModelNo()),
dataset.id,
dataset.uuid,
dataset.dataType,
dataset.title,
dataset.roundNo,
dataset.compareYyyy,
dataset.targetYyyy,
dataset.memo,
new CaseBuilder()
.when(
datasetObjEntity.targetClassCd.equalsIgnoreCase(
DetectionClassification.SOLAR.getId()))
.then(1)
.otherwise(0)
.sum()))
.from(dataset)
.leftJoin(datasetObjEntity)
.on(dataset.id.eq(datasetObjEntity.datasetUid))
.where(builder)
.groupBy(
dataset.id,
dataset.uuid,
dataset.dataType,
dataset.title,
dataset.roundNo,
dataset.memo)
.orderBy(dataset.createdDttm.desc())
.fetch();
}
} }

View File

@@ -1,3 +1,4 @@
// TODO 미사용시작
package com.kamco.cd.training.postgres.repository.dataset; package com.kamco.cd.training.postgres.repository.dataset;
import com.kamco.cd.training.postgres.entity.MapSheetEntity; import com.kamco.cd.training.postgres.entity.MapSheetEntity;
@@ -11,3 +12,4 @@ public interface MapSheetRepository
long countByDatasetIdAndDeletedFalse(Long datasetId); long countByDatasetIdAndDeletedFalse(Long datasetId);
} }
// TODO 미사용 끝

View File

@@ -1,3 +1,4 @@
// TODO 미사용시작
package com.kamco.cd.training.postgres.repository.dataset; package com.kamco.cd.training.postgres.repository.dataset;
import com.kamco.cd.training.dataset.dto.MapSheetDto; import com.kamco.cd.training.dataset.dto.MapSheetDto;
@@ -7,3 +8,4 @@ import org.springframework.data.domain.Page;
public interface MapSheetRepositoryCustom { public interface MapSheetRepositoryCustom {
Page<MapSheetEntity> findMapSheetList(MapSheetDto.SearchReq searchReq); Page<MapSheetEntity> findMapSheetList(MapSheetDto.SearchReq searchReq);
} }
// TODO 미사용 끝

View File

@@ -1,3 +1,4 @@
// TODO 미사용시작
package com.kamco.cd.training.postgres.repository.dataset; package com.kamco.cd.training.postgres.repository.dataset;
import com.kamco.cd.training.dataset.dto.MapSheetDto; import com.kamco.cd.training.dataset.dto.MapSheetDto;
@@ -52,3 +53,4 @@ public class MapSheetRepositoryImpl implements MapSheetRepositoryCustom {
return new PageImpl<>(content, pageable, total); return new PageImpl<>(content, pageable, total);
} }
} }
// TODO 미사용 끝

View File

@@ -8,5 +8,7 @@ import org.springframework.stereotype.Repository;
@Repository @Repository
public interface HyperParamRepository public interface HyperParamRepository
extends JpaRepository<ModelHyperParamEntity, Long>, HyperParamRepositoryCustom { extends JpaRepository<ModelHyperParamEntity, Long>, HyperParamRepositoryCustom {
// TODO 미사용시작
Optional<ModelHyperParamEntity> findByHyperVer(String hyperVer); Optional<ModelHyperParamEntity> findByHyperVer(String hyperVer);
// TODO 미사용 끝
} }

View File

@@ -11,6 +11,7 @@ import org.springframework.data.domain.Page;
public interface HyperParamRepositoryCustom { public interface HyperParamRepositoryCustom {
// TODO 미사용시작
/** /**
* 마지막 버전 조회 * 마지막 버전 조회
* *
@@ -19,6 +20,8 @@ public interface HyperParamRepositoryCustom {
@Deprecated @Deprecated
Optional<ModelHyperParamEntity> findHyperParamVer(); Optional<ModelHyperParamEntity> findHyperParamVer();
// TODO 미사용 끝
/** /**
* 모델 타입별 마지막 버전 조회 * 모델 타입별 마지막 버전 조회
* *
@@ -27,8 +30,11 @@ public interface HyperParamRepositoryCustom {
*/ */
Optional<ModelHyperParamEntity> findHyperParamVerByModelType(ModelType modelType); Optional<ModelHyperParamEntity> findHyperParamVerByModelType(ModelType modelType);
// TODO 미사용시작
Optional<ModelHyperParamEntity> findHyperParamByHyperVer(String hyperVer); Optional<ModelHyperParamEntity> findHyperParamByHyperVer(String hyperVer);
// TODO 미사용 끝
/** /**
* 하이퍼 파라미터 상세조회 * 하이퍼 파라미터 상세조회
* *

View File

@@ -29,6 +29,7 @@ public class HyperParamRepositoryImpl implements HyperParamRepositoryCustom {
private final JPAQueryFactory queryFactory; private final JPAQueryFactory queryFactory;
// TODO 미사용시작
@Override @Override
public Optional<ModelHyperParamEntity> findHyperParamVer() { public Optional<ModelHyperParamEntity> findHyperParamVer() {
@@ -42,6 +43,8 @@ public class HyperParamRepositoryImpl implements HyperParamRepositoryCustom {
.fetchOne()); .fetchOne());
} }
// TODO 미사용 끝
@Override @Override
public Optional<ModelHyperParamEntity> findHyperParamVerByModelType(ModelType modelType) { public Optional<ModelHyperParamEntity> findHyperParamVerByModelType(ModelType modelType) {
@@ -59,6 +62,7 @@ public class HyperParamRepositoryImpl implements HyperParamRepositoryCustom {
.fetchOne()); .fetchOne());
} }
// TODO 미사용시작
@Override @Override
public Optional<ModelHyperParamEntity> findHyperParamByHyperVer(String hyperVer) { public Optional<ModelHyperParamEntity> findHyperParamByHyperVer(String hyperVer) {
@@ -75,6 +79,8 @@ public class HyperParamRepositoryImpl implements HyperParamRepositoryCustom {
.fetchOne()); .fetchOne());
} }
// TODO 미사용 끝
@Override @Override
public Optional<ModelHyperParamEntity> findHyperParamByUuid(UUID uuid) { public Optional<ModelHyperParamEntity> findHyperParamByUuid(UUID uuid) {
return Optional.ofNullable( return Optional.ofNullable(

View File

@@ -7,14 +7,18 @@ import java.util.UUID;
import org.springframework.data.domain.Page; import org.springframework.data.domain.Page;
public interface MembersRepositoryCustom { public interface MembersRepositoryCustom {
// TODO 미사용시작
boolean existsByUserId(String userId); boolean existsByUserId(String userId);
// TODO 미사용 끝
boolean existsByEmployeeNo(String employeeNo); boolean existsByEmployeeNo(String employeeNo);
Optional<MemberEntity> findByEmployeeNo(String employeeNo); Optional<MemberEntity> findByEmployeeNo(String employeeNo);
// TODO 미사용시작
Optional<MemberEntity> findByUserId(String userId); Optional<MemberEntity> findByUserId(String userId);
// TODO 미사용 끝
Optional<MemberEntity> findByUUID(UUID uuid); Optional<MemberEntity> findByUUID(UUID uuid);
Page<MemberEntity> findByMembers(MembersDto.SearchReq searchReq); Page<MemberEntity> findByMembers(MembersDto.SearchReq searchReq);

View File

@@ -27,6 +27,7 @@ public class MembersRepositoryImpl extends QuerydslRepositorySupport
this.queryFactory = queryFactory; this.queryFactory = queryFactory;
} }
// TODO 미사용시작
/** /**
* 사용자 ID 조회 * 사용자 ID 조회
* *
@@ -43,6 +44,8 @@ public class MembersRepositoryImpl extends QuerydslRepositorySupport
!= null; != null;
} }
// TODO 미사용 끝
/** /**
* 사용자 사번 조회 * 사용자 사번 조회
* *
@@ -59,6 +62,7 @@ public class MembersRepositoryImpl extends QuerydslRepositorySupport
!= null; != null;
} }
// TODO 미사용시작
/** /**
* 사용자 조회 user id * 사용자 조회 user id
* *
@@ -71,6 +75,8 @@ public class MembersRepositoryImpl extends QuerydslRepositorySupport
queryFactory.selectFrom(memberEntity).where(memberEntity.userId.eq(userId)).fetchOne()); queryFactory.selectFrom(memberEntity).where(memberEntity.userId.eq(userId)).fetchOne());
} }
// TODO 미사용 끝
/** /**
* 사용자 조회 employeed no * 사용자 조회 employeed no
* *

View File

@@ -6,5 +6,7 @@ import java.util.Optional;
public interface ModelConfigRepositoryCustom { public interface ModelConfigRepositoryCustom {
Optional<ModelConfigDto.Basic> findModelConfigByModelId(Long modelId); Optional<ModelConfigDto.Basic> findModelConfigByModelId(Long modelId);
// TODO 미사용시작
Optional<ModelConfigDto.TransferBasic> findModelTransferConfigByModelId(Long modelId); Optional<ModelConfigDto.TransferBasic> findModelTransferConfigByModelId(Long modelId);
// TODO 미사용 끝
} }

View File

@@ -39,6 +39,7 @@ public class ModelConfigRepositoryImpl implements ModelConfigRepositoryCustom {
.fetchOne()); .fetchOne());
} }
// TODO 미사용시작
@Override @Override
public Optional<TransferBasic> findModelTransferConfigByModelId(Long modelId) { public Optional<TransferBasic> findModelTransferConfigByModelId(Long modelId) {
QModelConfigEntity beforeConfig = new QModelConfigEntity("beforeConfig"); QModelConfigEntity beforeConfig = new QModelConfigEntity("beforeConfig");
@@ -78,4 +79,5 @@ public class ModelConfigRepositoryImpl implements ModelConfigRepositoryCustom {
.where(modelMasterEntity.id.eq(modelId)) .where(modelMasterEntity.id.eq(modelId))
.fetchOne()); .fetchOne());
} }
// TODO 미사용 끝
} }

View File

@@ -23,8 +23,11 @@ public interface ModelDetailRepositoryCustom {
HyperSummary getByModelHyperParamSummary(UUID uuid); HyperSummary getByModelHyperParamSummary(UUID uuid);
// TODO 미사용시작
TransferHyperSummary getByModelTransferHyperParamSummary(UUID uuid); TransferHyperSummary getByModelTransferHyperParamSummary(UUID uuid);
// TODO 미사용 끝
List<MappingDataset> getByModelMappingDataset(UUID uuid); List<MappingDataset> getByModelMappingDataset(UUID uuid);
ModelMasterEntity findByModelByUUID(UUID uuid); ModelMasterEntity findByModelByUUID(UUID uuid);

View File

@@ -20,8 +20,11 @@ public interface ModelMngRepositoryCustom {
Optional<ModelMasterEntity> findByUuid(UUID uuid); Optional<ModelMasterEntity> findByUuid(UUID uuid);
// TODO 미사용시작
Optional<ModelMasterEntity> findFirstByStatusCdAndDelYn(String statusCd, Boolean delYn); Optional<ModelMasterEntity> findFirstByStatusCdAndDelYn(String statusCd, Boolean delYn);
// TODO 미사용 끝
TrainRunRequest findTrainRunRequest(Long modelId); TrainRunRequest findTrainRunRequest(Long modelId);
Long findModelStep1InProgressCnt(); Long findModelStep1InProgressCnt();

View File

@@ -133,11 +133,14 @@ public class ModelMngRepositoryImpl implements ModelMngRepositoryCustom {
.fetchOne()); .fetchOne());
} }
// TODO 미사용시작
@Override @Override
public Optional<ModelMasterEntity> findFirstByStatusCdAndDelYn(String statusCd, Boolean delYn) { public Optional<ModelMasterEntity> findFirstByStatusCdAndDelYn(String statusCd, Boolean delYn) {
return Optional.empty(); return Optional.empty();
} }
// TODO 미사용 끝
@Override @Override
public TrainRunRequest findTrainRunRequest(Long modelId) { public TrainRunRequest findTrainRunRequest(Long modelId) {
return queryFactory return queryFactory

View File

@@ -23,11 +23,11 @@ import org.springframework.stereotype.Service;
public class DataSetCountersService { public class DataSetCountersService {
private final ModelTrainMngCoreService modelTrainMngCoreService; private final ModelTrainMngCoreService modelTrainMngCoreService;
@Value("${train.docker.requestDir}") @Value("${train.docker.request_dir}")
private String requestDir; private String requestDir;
@Value("${train.docker.basePath}") @Value("${train.docker.symbolic_link_dir}")
private String trainBaseDir; private String symbolicDir;
public String getCount(Long modelId) { public String getCount(Long modelId) {
ModelTrainMngDto.Basic basic = modelTrainMngCoreService.findModelById(modelId); ModelTrainMngDto.Basic basic = modelTrainMngCoreService.findModelById(modelId);
@@ -45,7 +45,7 @@ public class DataSetCountersService {
} }
// tmp // tmp
Path tmpPath = Path.of(trainBaseDir, "tmp", basic.getRequestPath()); Path tmpPath = Path.of(symbolicDir, basic.getRequestPath());
// 차이나는거 // 차이나는거
diffMergedRequestsVsTmp(uids, tmpPath); diffMergedRequestsVsTmp(uids, tmpPath);

View File

@@ -31,27 +31,26 @@ public class DockerTrainService {
private String image; private String image;
// 학습 요청 데이터가 위치한 호스트 디렉토리 // 학습 요청 데이터가 위치한 호스트 디렉토리
@Value("${train.docker.requestDir}") @Value("${train.docker.request_dir}")
private String requestDir; private String requestDir;
// 학습 결과가 저장될 호스트 디렉토리 // 학습 결과가 저장될 호스트 디렉토리
@Value("${train.docker.responseDir}") @Value("${train.docker.response_dir}")
private String responseDir; private String responseDir;
// 컨테이너 이름 prefix
@Value("${train.docker.containerPrefix}")
private String containerPrefix;
// 공유메모리 사이즈 설정 (대용량 학습시 필요) // 공유메모리 사이즈 설정 (대용량 학습시 필요)
@Value("${train.docker.shmSize:16g}") @Value("${train.docker.shm_size:16g}")
private String shmSize; private String shmSize;
// data 경로 request,response 상위 폴더 // data 경로 request,response 상위 폴더
@Value("${train.docker.basePath}") @Value("${train.docker.base_path}")
private String basePath; private String basePath;
@Value("${train.docker.symbolic_link_dir}")
private String symbolicDir;
// IPC host 사용 여부 // IPC host 사용 여부
@Value("${train.docker.ipcHost:true}") @Value("${train.docker.ipc_host:true}")
private boolean ipcHost; private boolean ipcHost;
@Value("${spring.profiles.active}") @Value("${spring.profiles.active}")
@@ -263,9 +262,9 @@ public class DockerTrainService {
c.add("-v"); c.add("-v");
c.add(basePath + ":" + basePath); // 심볼릭 링크와 연결되는 실제 파일 경로도 마운트를 해줘야 함 c.add(basePath + ":" + basePath); // 심볼릭 링크와 연결되는 실제 파일 경로도 마운트를 해줘야 함
c.add("-v"); c.add("-v");
c.add(basePath + "/tmp:/data"); c.add(symbolicDir + ":/data"); // 요청할경로
c.add("-v"); c.add("-v");
c.add(responseDir + ":/checkpoints"); c.add(responseDir + ":/checkpoints"); // 저장될경로
// 표준입력 유지 (-it 대신 -i만 사용) // 표준입력 유지 (-it 대신 -i만 사용)
c.add("-i"); c.add("-i");

View File

@@ -50,7 +50,7 @@ public class JobRecoveryOnStartupService {
* *
* <p>컨테이너가 --rm 으로 삭제된 경우에도 이 경로에 val.csv / *.pth 등이 남아있으면 정상 종료 여부를 "파일 기반"으로 판정합니다. * <p>컨테이너가 --rm 으로 삭제된 경우에도 이 경로에 val.csv / *.pth 등이 남아있으면 정상 종료 여부를 "파일 기반"으로 판정합니다.
*/ */
@Value("${train.docker.responseDir}") @Value("${train.docker.response_dir}")
private String responseDir; private String responseDir;
/** /**
@@ -384,7 +384,20 @@ public class JobRecoveryOnStartupService {
return new OutputResult(false, "total-epoch-missing"); return new OutputResult(false, "total-epoch-missing");
} }
log.info("[RECOVERY] totalEpoch={}. jobId={}", totalEpoch, job.getId()); Integer valInterval = extractValInterval(job).orElse(null);
if (valInterval == null || valInterval <= 0) {
log.warn(
"[RECOVERY] valInterval missing or invalid. jobId={}, valInterval={}",
job.getId(),
valInterval);
return new OutputResult(false, "val-interval-missing");
}
log.info(
"[RECOVERY] totalEpoch={}. valInterval={}. jobId={}",
totalEpoch,
valInterval,
job.getId());
// 3) val.csv 존재 확인 // 3) val.csv 존재 확인
Path valCsv = outDir.resolve("val.csv"); Path valCsv = outDir.resolve("val.csv");
@@ -396,14 +409,17 @@ public class JobRecoveryOnStartupService {
// 4) val.csv 라인 수 확인 // 4) val.csv 라인 수 확인
long lines = countNonHeaderLines(valCsv); long lines = countNonHeaderLines(valCsv);
// expected = 실제 val 실행 횟수
int expectedLines = totalEpoch / valInterval;
log.info( log.info(
"[RECOVERY] val.csv lines counted. jobId={}, lines={}, expected={}", "[RECOVERY] val.csv lines counted. jobId={}, lines={}, expected={}",
job.getId(), job.getId(),
lines, lines,
totalEpoch); expectedLines);
// 5) 완료 판정 // 5) 완료 판정
if (lines == totalEpoch) { if (lines >= expectedLines) {
log.info("[RECOVERY] outputs look COMPLETE. jobId={}", job.getId()); log.info("[RECOVERY] outputs look COMPLETE. jobId={}", job.getId());
return new OutputResult(true, "ok"); return new OutputResult(true, "ok");
} }
@@ -412,7 +428,7 @@ public class JobRecoveryOnStartupService {
"[RECOVERY] val.csv line mismatch. jobId={}, lines={}, expected={}", "[RECOVERY] val.csv line mismatch. jobId={}, lines={}, expected={}",
job.getId(), job.getId(),
lines, lines,
totalEpoch); expectedLines);
return new OutputResult( return new OutputResult(
false, "val.csv-lines-mismatch lines=" + lines + " expected=" + totalEpoch); false, "val.csv-lines-mismatch lines=" + lines + " expected=" + totalEpoch);
@@ -530,4 +546,19 @@ public class JobRecoveryOnStartupService {
return reason; return reason;
} }
} }
/** paramsJson에서 valInterval 추출 */
private Optional<Integer> extractValInterval(ModelTrainJobDto job) {
Map<String, Object> params = job.getParamsJson();
if (params == null) return Optional.empty();
Object v = params.get("valInterval");
if (v == null) return Optional.empty();
try {
return Optional.of(Integer.parseInt(String.valueOf(v)));
} catch (Exception ignore) {
return Optional.empty();
}
}
} }

View File

@@ -42,7 +42,7 @@ public class ModelTestMetricsJobService {
private String profile; private String profile;
// 학습 결과가 저장될 호스트 디렉토리 // 학습 결과가 저장될 호스트 디렉토리
@Value("${train.docker.responseDir}") @Value("${train.docker.response_dir}")
private String responseDir; private String responseDir;
@Value("${file.pt-path}") @Value("${file.pt-path}")

View File

@@ -33,7 +33,7 @@ public class ModelTrainMetricsJobService {
private String profile; private String profile;
// 학습 결과가 저장될 호스트 디렉토리 // 학습 결과가 저장될 호스트 디렉토리
@Value("${train.docker.responseDir}") @Value("${train.docker.response_dir}")
private String responseDir; private String responseDir;
/** 결과 csv 파일 정보 등록 */ /** 결과 csv 파일 정보 등록 */
@@ -84,18 +84,21 @@ public class ModelTrainMetricsJobService {
for (CSVRecord record : parser) { for (CSVRecord record : parser) {
int epoch = Integer.parseInt(record.get("Epoch")); int epoch = Integer.parseInt(record.get("Epoch"));
float aAcc = Float.parseFloat(record.get("aAcc"));
float mFscore = Float.parseFloat(record.get("mFscore")); Float aAcc = parseFloatSafe(record.get("aAcc"));
float mPrecision = Float.parseFloat(record.get("mPrecision")); Float mFscore = parseFloatSafe(record.get("mFscore"));
float mRecall = Float.parseFloat(record.get("mRecall")); Float mPrecision = parseFloatSafe(record.get("mPrecision"));
float mIoU = Float.parseFloat(record.get("mIoU")); Float mRecall = parseFloatSafe(record.get("mRecall"));
float mAcc = Float.parseFloat(record.get("mAcc")); Float mIoU = parseFloatSafe(record.get("mIoU"));
float changed_fscore = Float.parseFloat(record.get("changed_fscore")); Float mAcc = parseFloatSafe(record.get("mAcc"));
float changed_precision = Float.parseFloat(record.get("changed_precision"));
float changed_recall = Float.parseFloat(record.get("changed_recall")); Float changed_fscore = parseFloatSafe(record.get("changed_fscore"));
float unchanged_fscore = Float.parseFloat(record.get("unchanged_fscore")); Float changed_precision = parseFloatSafe(record.get("changed_precision"));
float unchanged_precision = Float.parseFloat(record.get("unchanged_precision")); Float changed_recall = parseFloatSafe(record.get("changed_recall"));
float unchanged_recall = Float.parseFloat(record.get("unchanged_recall"));
Float unchanged_fscore = parseFloatSafe(record.get("unchanged_fscore"));
Float unchanged_precision = parseFloatSafe(record.get("unchanged_precision"));
Float unchanged_recall = parseFloatSafe(record.get("unchanged_recall"));
batchArgs.add( batchArgs.add(
new Object[] { new Object[] {
@@ -153,4 +156,23 @@ public class ModelTrainMetricsJobService {
modelInfo.getModelId(), "step1"); modelInfo.getModelId(), "step1");
} }
} }
private Float parseFloatSafe(String value) {
try {
if (value == null) return null;
value = value.trim();
if (value.isEmpty()) return null;
if (value.equalsIgnoreCase("nan")) return null;
float f = Float.parseFloat(value);
return Float.isNaN(f) ? null : f;
} catch (Exception e) {
return null;
}
}
} }

View File

@@ -14,11 +14,11 @@ import org.springframework.stereotype.Service;
@RequiredArgsConstructor @RequiredArgsConstructor
public class TmpDatasetService { public class TmpDatasetService {
@Value("${train.docker.requestDir}") @Value("${train.docker.request_dir}")
private String requestDir; private String requestDir;
@Value("${train.docker.basePath}") @Value("${train.docker.symbolic_link_dir}")
private String trainBaseDir; private String symbolicDir;
/** /**
* train, val, test 폴더별로 link * train, val, test 폴더별로 link
@@ -36,7 +36,7 @@ public class TmpDatasetService {
throw new IOException("links is empty"); throw new IOException("links is empty");
} }
Path tmp = Path.of(trainBaseDir, "tmp", uid); Path tmp = Path.of(symbolicDir, uid);
long linksMade = 0; long linksMade = 0;
@@ -98,6 +98,7 @@ public class TmpDatasetService {
return 1; return 1;
} }
// TODO 미사용시작
/** /**
* request 전체 폴더 link * request 전체 폴더 link
* *
@@ -114,7 +115,7 @@ public class TmpDatasetService {
log.info("requestDir(raw)={}", requestDir); log.info("requestDir(raw)={}", requestDir);
Path BASE = toPath(requestDir); Path BASE = toPath(requestDir);
Path tmp = Path.of(trainBaseDir, "tmp", uid); Path tmp = Path.of(symbolicDir, uid);
log.info("BASE={}", BASE); log.info("BASE={}", BASE);
log.info("BASE exists? {}", Files.isDirectory(BASE)); log.info("BASE exists? {}", Files.isDirectory(BASE));
@@ -210,4 +211,5 @@ public class TmpDatasetService {
} }
return Paths.get(p).toAbsolutePath().normalize(); return Paths.get(p).toAbsolutePath().normalize();
} }
// TODO 미사용 끝
} }

View File

@@ -40,7 +40,7 @@ public class TrainJobService {
private final DataSetCountersService dataSetCounters; private final DataSetCountersService dataSetCounters;
// 학습 결과가 저장될 호스트 디렉토리 // 학습 결과가 저장될 호스트 디렉토리
@Value("${train.docker.responseDir}") @Value("${train.docker.response_dir}")
private String responseDir; private String responseDir;
public Long getModelIdByUuid(UUID uuid) { public Long getModelIdByUuid(UUID uuid) {

View File

@@ -132,7 +132,9 @@ public class TrainJobWorker {
String failMsg = result.getStatus() + "\n" + result.getLogs(); String failMsg = result.getStatus() + "\n" + result.getLogs();
log.info("training fail exitCode={} Msg ={}", result.getExitCode(), failMsg); log.info("training fail exitCode={} Msg ={}", result.getExitCode(), failMsg);
if (result.getExitCode() == -1 || result.getExitCode() == 143) { if (result.getExitCode() == -1
|| result.getExitCode() == 143
|| result.getExitCode() == 137) {
// 실패 처리 // 실패 처리
modelTrainJobCoreService.markPaused( modelTrainJobCoreService.markPaused(
jobId, result.getExitCode(), result.getStatus() + "\n" + result.getLogs()); jobId, result.getExitCode(), result.getStatus() + "\n" + result.getLogs());

View File

@@ -83,6 +83,7 @@ public class UploadDto {
private UUID uuid; private UUID uuid;
} }
// TODO 미사용시작
@Schema(name = "UploadCompleteReq", description = "업로드 완료 요청") @Schema(name = "UploadCompleteReq", description = "업로드 완료 요청")
@Getter @Getter
@Setter @Setter
@@ -126,12 +127,15 @@ public class UploadDto {
@Schema(description = "상태", example = "UPLOADING") @Schema(description = "상태", example = "UPLOADING")
private String status; private String status;
// TODO 미사용시작
@Schema(description = "총 청크 수", example = "100") @Schema(description = "총 청크 수", example = "100")
private Integer totalChunks; private Integer totalChunks;
@Schema(description = "업로드된 청크 수", example = "50") @Schema(description = "업로드된 청크 수", example = "50")
private Integer uploadedChunks; private Integer uploadedChunks;
// TODO 미사용 끝
@Schema(description = "진행률 (%)", example = "50.0") @Schema(description = "진행률 (%)", example = "50.0")
private Double progress; private Double progress;
@@ -139,6 +143,8 @@ public class UploadDto {
private String errorMessage; private String errorMessage;
} }
// TODO 미사용 끝
@Schema(name = "UploadAddReq", description = "업로드 요청") @Schema(name = "UploadAddReq", description = "업로드 요청")
@Getter @Getter
@Setter @Setter

View File

@@ -26,27 +26,21 @@ public class UploadService {
private final UploadSessionCoreService uploadSessionCoreService; private final UploadSessionCoreService uploadSessionCoreService;
@Value("${file.sync-root-dir}")
private String syncRootDir;
@Value("${file.sync-tmp-dir}")
private String syncTmpDir;
@Value("${file.sync-file-extention}")
private String syncFileExtention;
@Value("${file.dataset-dir}") @Value("${file.dataset-dir}")
private String datasetDir; private String datasetDir;
@Value("${file.dataset-tmp-dir}") @Value("${file.dataset-tmp-dir}")
private String datasetTmpDir; private String datasetTmpDir;
// TODO 미사용시작
@Transactional @Transactional
public DmlReturn initUpload(UploadDto.InitReq initReq) { public DmlReturn initUpload(UploadDto.InitReq initReq) {
return new DmlReturn("success", "UPLOAD CHUNK INIT"); return new DmlReturn("success", "UPLOAD CHUNK INIT");
} }
// TODO 미사용 끝
@Transactional @Transactional
public UploadDto.UploadRes uploadChunk(UploadDto.UploadAddReq upAddReqDto, MultipartFile file) { public UploadDto.UploadRes uploadChunk(UploadDto.UploadAddReq upAddReqDto, MultipartFile file) {

View File

@@ -1,72 +1,49 @@
spring: spring:
config: config:
activate: activate:
on-profile: dev on-profile: dev
jpa: jpa:
show-sql: true show-sql: true
hibernate: properties:
ddl-auto: validate hibernate:
properties: use_sql_comments: true # ⚠️ 선택 - SQL에 주석 추가 (디버깅용)
hibernate: format_sql: true # ⚠️ 선택 - SQL 포맷팅 (가독성)
default_batch_fetch_size: 100 # ✅ 성능 - N+1 쿼리 방지
order_updates: true # ✅ 성능 - 업데이트 순서 정렬로 데드락 방지 datasource:
use_sql_comments: true # ⚠️ 선택 - SQL에 주석 추가 (디버깅용) url: jdbc:postgresql://192.168.2.127:15432/kamco_training_db
format_sql: true # ⚠️ 선택 - SQL 포맷팅 (가독성) username: kamco_training_user
password: kamco_training_user_2025_!@#
datasource: hikari:
url: jdbc:postgresql://192.168.2.127:15432/kamco_training_db minimum-idle: 10
username: kamco_training_user maximum-pool-size: 20
password: kamco_training_user_2025_!@#
hikari: jwt:
minimum-idle: 10 secret: "kamco_token_dev_dfc6446d-68fc-4eba-a2ff-c80a14a0bf3a"
maximum-pool-size: 20 access-token-validity-in-ms: 86400000 # 1일
connection-timeout: 60000 # 60초 연결 타임아웃 refresh-token-validity-in-ms: 604800000 # 7일
idle-timeout: 300000 # 5분 유휴 타임아웃
max-lifetime: 1800000 # 30분 최대 수명 token:
leak-detection-threshold: 60000 # 연결 누수 감지 refresh-cookie-name: kamco-dev # 개발용 쿠키 이름
refresh-cookie-secure: false # 로컬 http 테스트면 false
transaction:
default-timeout: 300 # 5분 트랜잭션 타임아웃 swagger:
local-port: 8080
jwt: file:
secret: "kamco_token_dev_dfc6446d-68fc-4eba-a2ff-c80a14a0bf3a" dataset-dir: /home/kcomu/data/request/
access-token-validity-in-ms: 86400000 # 1일 dataset-tmp-dir: ${file.dataset-dir}tmp/
refresh-token-validity-in-ms: 604800000 # 7일
pt-path: /home/kcomu/data/response/v6-cls-checkpoints/
token: pt-FileName: yolov8_6th-6m.pt
refresh-cookie-name: kamco-dev # 개발용 쿠키 이름
refresh-cookie-secure: false # 로컬 http 테스트면 false train:
docker:
springdoc: image: kamco-cd-train:latest
swagger-ui: base_path: /home/kcomu/data
persist-authorization: true # 스웨거 새로고침해도 토큰 유지, 로컬스토리지에 저장 request_dir: ${train.docker.base_path}/request
response_dir: ${train.docker.base_path}/response
member: symbolic_link_dir: ${train.docker.base_path}/tmp
init_password: kamco1234! container_prefix: kamco-cd-train
shm_size: 16g
swagger: ipc_host: true
local-port: 8080
file:
sync-root-dir: /app/original-images/
sync-tmp-dir: ${file.sync-root-dir}tmp/
sync-file-extention: tfw,tif
dataset-dir: /home/kcomu/data/request/
dataset-tmp-dir: ${file.dataset-dir}tmp/
pt-path: /home/kcomu/data/response/v6-cls-checkpoints/
pt-FileName: yolov8_6th-6m.pt
train:
docker:
image: kamco-cd-train:latest
requestDir: /home/kcomu/data/request
responseDir: /home/kcomu/data/response
basePath: /home/kcomu/data
containerPrefix: kamco-cd-train
shmSize: 16g
ipcHost: true

View File

@@ -1,72 +1,43 @@
spring: spring:
config: config:
activate: activate:
on-profile: prod on-profile: prod
jpa: datasource:
show-sql: false # 운영 환경에서는 성능을 위해 비활성화 url: jdbc:postgresql://kamco-cd-train-db:5432/kamco_training_db
hibernate: username: kamco_training_user
ddl-auto: validate password: kamco_training_user_2025_!@#
properties: hikari:
hibernate: minimum-idle: 10
default_batch_fetch_size: 100 # N+1 쿼리 방지 maximum-pool-size: 20
order_updates: true # 업데이트 순서 정렬로 데드락 방지
jwt:
datasource: # ⚠️ 운영 환경에서는 반드시 별도의 강력한 시크릿 키를 사용하세요
url: jdbc:postgresql://kamco-cd-train-db:5432/kamco_training_db secret: "kamco_token_prod_CHANGE_THIS_TO_SECURE_SECRET_KEY"
username: kamco_training_user access-token-validity-in-ms: 86400000 # 1일
password: kamco_training_user_2025_!@# refresh-token-validity-in-ms: 604800000 # 7일
hikari:
minimum-idle: 10 token:
maximum-pool-size: 20 refresh-cookie-name: kamco
connection-timeout: 60000 # 60초 연결 타임아웃 refresh-cookie-secure: true # HTTPS 환경에서 필수
idle-timeout: 300000 # 5분 유휴 타임아웃
max-lifetime: 1800000 # 30분 최대 수명 swagger:
leak-detection-threshold: 60000 # 연결 누수 감지 local-port: 9080
transaction: file:
default-timeout: 300 # 5분 트랜잭션 타임아웃 dataset-dir: /data/training/request/
dataset-tmp-dir: ${file.dataset-dir}tmp/
jwt:
# ⚠️ 운영 환경에서는 반드시 별도의 강력한 시크릿 키를 사용하세요 pt-path: /data/training/response/v6-cls-checkpoints/
secret: "kamco_token_prod_CHANGE_THIS_TO_SECURE_SECRET_KEY" pt-FileName: yolov8_6th-6m.pt
access-token-validity-in-ms: 86400000 # 1일
refresh-token-validity-in-ms: 604800000 # 7일 train:
docker:
token: image: kamco-cd-train:latest
refresh-cookie-name: kamco base_path: /data/training
refresh-cookie-secure: true # HTTPS 환경에서 필수 request_dir: ${train.docker.base_path}/request
response_dir: ${train.docker.base_path}/response
springdoc: symbolic_link_dir: ${train.docker.base_path}/tmp
swagger-ui: container_prefix: kamco-cd-train
persist-authorization: true # 스웨거 새로고침해도 토큰 유지, 로컬스토리지에 저장 shm_size: 16g
ipc_host: true
member:
init_password: kamco1234!
swagger:
local-port: 9080
file:
sync-root-dir: /app/original-images/
sync-tmp-dir: ${file.sync-root-dir}tmp/
sync-file-extention: tfw,tif
dataset-dir: /home/kcomu/data/request/
dataset-tmp-dir: ${file.dataset-dir}tmp/
pt-path: /home/kcomu/data/response/v6-cls-checkpoints/
pt-FileName: yolov8_6th-6m.pt
train:
docker:
image: kamco-cd-train:latest
requestDir: /home/kcomu/data/request
responseDir: /home/kcomu/data/response
basePath: /home/kcomu/data
containerPrefix: kamco-cd-train
shmSize: 16g
ipcHost: true

View File

@@ -1,71 +1,80 @@
server: server:
port: 8080 port: 8080
spring: spring:
application: application:
name: kamco-training-api name: kamco-training-api
profiles: profiles:
active: dev # 사용할 프로파일 지정 (ex. dev, prod, test) active: dev # 사용할 프로파일 지정 (ex. dev, prod, test)
datasource: datasource:
driver-class-name: org.postgresql.Driver driver-class-name: org.postgresql.Driver
hikari: hikari:
minimum-idle: 2 connection-timeout: 60000 # 60초 연결 타임아웃
maximum-pool-size: 2 idle-timeout: 300000 # 5분 유휴 타임아웃
connection-timeout: 20000 max-lifetime: 1800000 # 30분 최대 수명
idle-timeout: 300000 leak-detection-threshold: 60000 # 연결 누수 감지
max-lifetime: 1800000 # minimum-idle, maximum-pool-size 는 프로파일별 설정
leak-detection-threshold: 60000
jpa:
jpa: hibernate:
hibernate: ddl-auto: validate
ddl-auto: update # 스키마 자동 관리 활성화 properties:
properties: hibernate:
hibernate: jakarta:
javax: persistence:
persistence: validation:
validation: mode: none
mode: none jdbc:
jdbc: batch_size: 50
batch_size: 50 default_batch_fetch_size: 100
default_batch_fetch_size: 100 order_updates: true
show-sql: false show-sql: false
servlet: servlet:
multipart: multipart:
enabled: true enabled: true
max-file-size: 10GB max-file-size: 10GB
max-request-size: 10GB max-request-size: 10GB
logging:
level: transaction:
org: default-timeout: 300 # 5분 트랜잭션 타임아웃
springframework:
web: INFO logging:
security: INFO level:
root: INFO org:
springframework:
web: INFO
security: INFO
# actuator root: INFO
management:
health: springdoc:
readinessstate: swagger-ui:
enabled: true persist-authorization: true # 스웨거 새로고침해도 토큰 유지
livenessstate:
enabled: true member:
diskspace: init_password: kamco1234!
enabled: true
endpoint: # actuator
health: management:
probes: health:
enabled: true readinessstate:
show-details: when-authorized enabled: true
endpoints: livenessstate:
jmx: enabled: true
exposure: diskspace:
exclude: "*" enabled: true
web: endpoint:
base-path: /monitor health:
exposure: probes:
include: enabled: true
- "health" show-details: when-authorized
endpoints:
jmx:
exposure:
exclude: "*"
web:
base-path: /monitor
exposure:
include:
- "health"

View File

@@ -1,77 +1,74 @@
server: server:
port: 8080 port: 8080
spring: spring:
application: application:
name: kamco-training-api name: kamco-training-api
datasource: datasource:
driver-class-name: org.postgresql.Driver driver-class-name: org.postgresql.Driver
url: jdbc:postgresql://localhost:15432/kamco_training_db url: jdbc:postgresql://localhost:15432/kamco_training_db
username: kamco_cds username: kamco_cds
password: kamco_cds_Q!W@E#R$ password: kamco_cds_Q!W@E#R$
hikari: hikari:
minimum-idle: 2 minimum-idle: 2
maximum-pool-size: 2 maximum-pool-size: 2
connection-timeout: 20000 connection-timeout: 20000
idle-timeout: 300000 idle-timeout: 300000
max-lifetime: 1800000 max-lifetime: 1800000
leak-detection-threshold: 60000 leak-detection-threshold: 60000
jpa: jpa:
hibernate: hibernate:
ddl-auto: none # 테스트 환경에서는 DDL 자동 생성/수정 비활성화 ddl-auto: none # 테스트 환경에서는 DDL 자동 생성/수정 비활성화
properties: properties:
hibernate: hibernate:
hbm2ddl: jakarta:
auto: none persistence:
javax: validation:
persistence: mode: none
validation: jdbc:
mode: none batch_size: 50
jdbc: default_batch_fetch_size: 100
batch_size: 50 show-sql: false
default_batch_fetch_size: 100
show-sql: false logging:
level:
logging: org:
level: springframework:
org: web: DEBUG
springframework: security: DEBUG
web: DEBUG root: INFO
security: DEBUG
root: INFO management:
health:
management: readinessstate:
health: enabled: true
readinessstate: livenessstate:
enabled: true enabled: true
livenessstate: endpoint:
enabled: true health:
endpoint: probes:
health: enabled: true
probes: show-details: always
enabled: true endpoints:
show-details: always jmx:
endpoints: exposure:
jmx: exclude: "*"
exposure: web:
exclude: "*" base-path: /monitor
web: exposure:
base-path: /monitor include:
exposure: - "health"
include:
- "health" jwt:
secret: "test_secret_key_for_testing_purposes_only"
jwt: access-token-validity-in-ms: 86400000
secret: "test_secret_key_for_testing_purposes_only" refresh-token-validity-in-ms: 604800000
access-token-validity-in-ms: 86400000
refresh-token-validity-in-ms: 604800000 token:
refresh-cookie-name: kamco-test
token: refresh-cookie-secure: false
refresh-cookie-name: kamco-test
refresh-cookie-secure: false member:
init_password: test1234!
member:
init_password: test1234!