23 Commits

Author SHA1 Message Date
96dd76101b 분류데이터셋 tif 바이너리화 API 추가 2026-07-20 17:07:13 +09:00
9b1bd16d1b 분류데이터셋 업로드 chunk API 추가 2026-07-20 16:45:52 +09:00
a94e8e94a0 분류데이터셋 업로드,상세,목록 API 2026-07-20 15:58:26 +09:00
10319ce0a6 커맨드 수정 2026-07-07 17:01:11 +09:00
fcc5d7a437 test 실행에 user 권한 추가 2026-07-07 15:39:03 +09:00
0ab0fdc43e 전이학습 등록 시 uuid 추가 2026-06-23 16:13:53 +09:00
ad750f0f06 학습 상세 API에 beforeModelUuid 추가 2026-06-23 15:25:03 +09:00
5fc93b64b6 전이학습 API 호출 파라미터 적용 2026-06-23 14:25:16 +09:00
cd14e6e3aa percent 수정 2026-06-04 13:43:10 +09:00
0447dd80ed 수정 2026-06-04 12:47:12 +09:00
5f4640ea60 학습 진행 현황 - 진행율, 진행중인 모델 API 추가 2026-06-04 12:38:23 +09:00
b85f920f40 학습 소요시간 로직 수정 2026-05-19 14:55:49 +09:00
e4851b1153 test 실행 도커 명령어 수정 2026-05-19 14:33:36 +09:00
b73aef5cf8 class 명칭 가져오는 부분 영어버전 추가 2026-05-15 14:59:39 +09:00
72c8f6a047 제작 업로드 시 dataset_path 안 맞는 현상 수정 2026-05-15 10:45:44 +09:00
c7a49ea4ea headerUtil 확인 2026-05-15 10:36:05 +09:00
fbef92af55 모델관리 API 영문버전 로직 수정 2026-05-12 14:51:18 +09:00
154db0ac27 영어버전도 나오게 추가 2026-05-12 14:27:09 +09:00
9835170cd7 납품데이터 등록 시 totalSize 추가 2026-05-08 10:40:06 +09:00
fd51f21ba6 납품 데이터 저장 시 uid 수정, 파일 업로드 경로 수정 2026-05-08 09:58:52 +09:00
776622e0a2 트랜잭션 5분 걸려있던 내용 주석처리 2026-05-07 15:29:24 +09:00
dabeeo
4d2d7a9ad1 check 2026-04-20 17:45:33 +09:00
dabeeo
532fbdbee4 hello 2026-04-20 17:35:26 +09:00
37 changed files with 2820 additions and 58 deletions

View File

@@ -1,7 +1,10 @@
services:
kamco-changedetection-api:
image: kamco-train-app:latest
container_name: kamco-train-api
build:
context: .
dockerfile: Dockerfile-dev
image: kamco-cd-training-api:${IMAGE_TAG:-latest}
container_name: kamco-cd-training-api
deploy:
resources:
reservations:
@@ -9,13 +12,13 @@ services:
- driver: nvidia
count: all
capabilities: [gpu]
ports:
- "7200:8080"
environment:
- SPRING_PROFILES_ACTIVE=dev
- TZ=Asia/Seoul
- cors.allowed-origins=*
- cors.allowed-origins[0]=*
volumes:
- /data/training:/data/training
- /backup/data/training:/backup/data/training
- /var/run/docker.sock:/var/run/docker.sock
networks:
- kamco-cds

View File

@@ -37,7 +37,8 @@ public class GpuDmonReader {
int avg =
deque.isEmpty()
? 0
: (int) Math.round(deque.stream().mapToInt(Integer::intValue).average().orElse(0));
: (int)
Math.round(deque.stream().mapToInt(Integer::intValue).average().orElse(0));
result.put(gpu, avg);
}
});
@@ -98,8 +99,7 @@ public class GpuDmonReader {
Process process = pb.start();
// 프로세스 실행 후 stdout 읽기
try (BufferedReader br =
new BufferedReader(new InputStreamReader(process.getInputStream()))) {
try (BufferedReader br = new BufferedReader(new InputStreamReader(process.getInputStream()))) {
String line;

View File

@@ -1,7 +1,12 @@
package com.kamco.cd.training.common.utils;
import jakarta.servlet.http.HttpServletRequest;
import lombok.extern.slf4j.Slf4j;
import org.springframework.web.context.request.RequestAttributes;
import org.springframework.web.context.request.RequestContextHolder;
import org.springframework.web.context.request.ServletRequestAttributes;
@Slf4j
public final class HeaderUtil {
private HeaderUtil() {}
@@ -20,4 +25,20 @@ public final class HeaderUtil {
public static String getRequired(HttpServletRequest request, String headerName) {
return get(request, headerName);
}
public static boolean isEnglishRequest() {
RequestAttributes attrs = RequestContextHolder.getRequestAttributes();
if (!(attrs instanceof ServletRequestAttributes servletAttrs)) {
return false;
}
String acceptLanguage = servletAttrs.getRequest().getHeader("Accept-Language");
if (acceptLanguage == null || acceptLanguage.isBlank()) {
return false;
}
return acceptLanguage.toLowerCase().startsWith("en");
}
}

View File

@@ -1,5 +1,6 @@
package com.kamco.cd.training.common.utils.enums;
import com.kamco.cd.training.common.utils.HeaderUtil;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
@@ -31,10 +32,11 @@ public class Enums {
// enum -> CodeDto list
public static List<CodeDto> toList(Class<? extends Enum<?>> enumClass) {
Object[] enums = enumClass.getEnumConstants();
boolean english = HeaderUtil.isEnglishRequest();
return Arrays.stream(enums)
.map(e -> (EnumType) e)
.map(e -> new CodeDto(e.getId(), e.getText()))
.map(e -> new CodeDto(e.getId(), english ? e.getId() : e.getText()))
.toList();
}

View File

@@ -293,6 +293,8 @@ public class DatasetApiController {
DatasetService.validateTrainValTestDirs(req.getFilePath());
// 파일 개수 검증
DatasetService.validateDirFileCount(req.getFilePath());
// 폴더명(uid)으로 등록한 건이 있는지 체크
datasetService.validateExistsUidChk(req.getFilePath());
datasetAsyncService.insertDeliveriesDatasetAsync(req);
return ApiResponseDto.createOK("ok");

View File

@@ -0,0 +1,292 @@
package com.kamco.cd.training.dataset;
import com.kamco.cd.training.config.api.ApiResponseDto;
import com.kamco.cd.training.dataset.dto.DatasetClsObjDto;
import com.kamco.cd.training.dataset.dto.DatasetDto;
import com.kamco.cd.training.dataset.dto.DatasetDto.AddDeliveriesReq;
import com.kamco.cd.training.dataset.dto.DatasetObjDto.DatasetClass;
import com.kamco.cd.training.dataset.dto.DatasetObjDto.DatasetStorage;
import com.kamco.cd.training.dataset.service.DatasetAsyncService;
import com.kamco.cd.training.dataset.service.DatasetClsService;
import com.kamco.cd.training.dataset.service.DatasetService;
import com.kamco.cd.training.model.dto.FileDto.FoldersDto;
import com.kamco.cd.training.model.dto.FileDto.SrchFoldersDto;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.media.Content;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.responses.ApiResponse;
import io.swagger.v3.oas.annotations.responses.ApiResponses;
import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.validation.Valid;
import java.io.IOException;
import java.nio.file.FileStore;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import java.util.UUID;
import lombok.RequiredArgsConstructor;
import org.springframework.core.io.Resource;
import org.springframework.data.domain.Page;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.*;
@Tag(name = "분류모델 학습데이터 관리", description = "어드민 홈 > 학습데이터관리 > 분류데이터셋")
@RestController
@RequestMapping("/api/datasets/cls")
@RequiredArgsConstructor
public class DatasetClsApiController {
private final DatasetClsService datasetClsService;
private final DatasetAsyncService datasetAsyncService;
@Operation(summary = "분류 학습데이터 관리 목록 조회", description = "분류 학습데이터 목록을 조회합니다.")
@ApiResponses(
value = {
@ApiResponse(
responseCode = "200",
description = "조회 성공",
content =
@Content(
mediaType = "application/json",
schema = @Schema(implementation = Page.class))),
@ApiResponse(responseCode = "400", description = "잘못된 검색 조건", content = @Content),
@ApiResponse(responseCode = "500", description = "서버 오류", content = @Content)
})
@GetMapping
public ApiResponseDto<Page<DatasetDto.Basic>> searchDatasets(
@Parameter(
description = "구분",
example = "",
schema = @Schema(allowableValues = {"DELIVER", "PRODUCTION"}))
@RequestParam(required = false)
String dataType,
@Parameter(description = "제목", example = "") @RequestParam(required = false) String title,
@Parameter(description = "페이지 번호 (0부터 시작)", example = "0") @RequestParam(defaultValue = "0")
int page,
@Parameter(description = "페이지 크기", example = "20") @RequestParam(defaultValue = "20")
int size) {
DatasetDto.SearchReq searchReq = new DatasetDto.SearchReq();
searchReq.setTitle(title);
searchReq.setDataType(dataType);
searchReq.setPage(page);
searchReq.setSize(size);
return ApiResponseDto.ok(datasetClsService.searchDatasets(searchReq));
}
@Operation(summary = "학습데이터관리 상세 조회", description = "학습데이터관리 상세 정보를 조회합니다.")
@ApiResponses(
value = {
@ApiResponse(
responseCode = "200",
description = "조회 성공",
content =
@Content(
mediaType = "application/json",
schema = @Schema(implementation = DatasetDto.Basic.class))),
@ApiResponse(responseCode = "404", description = "데이터셋을 찾을 수 없음", content = @Content),
@ApiResponse(responseCode = "500", description = "서버 오류", content = @Content)
})
@GetMapping("/{uuid}")
public ApiResponseDto<DatasetDto.Basic> getDatasetDetail(@PathVariable UUID uuid) {
return ApiResponseDto.ok(datasetClsService.getDatasetDetail(uuid));
}
@Operation(summary = "학습데이터 수정", description = "학습데이터 제목, 메모 수정")
@ApiResponses(
value = {
@ApiResponse(
responseCode = "200",
description = "수정 성공",
content =
@Content(
mediaType = "application/json",
schema = @Schema(implementation = Long.class))),
@ApiResponse(responseCode = "400", description = "잘못된 요청 데이터", content = @Content),
@ApiResponse(responseCode = "404", description = "데이터셋을 찾을 수 없음", content = @Content),
@ApiResponse(responseCode = "500", description = "서버 오류", content = @Content)
})
@PutMapping("/{uuid}")
public ApiResponseDto<UUID> updateDataset(
@PathVariable UUID uuid, @RequestBody DatasetDto.UpdateReq updateReq) {
datasetClsService.updateDataset(uuid, updateReq);
return ApiResponseDto.ok(uuid);
}
@Operation(summary = "학습데이터 삭제", description = "학습데이터를 삭제합니다.(납품 데이터는 삭제 불가)")
@ApiResponses(
value = {
@ApiResponse(responseCode = "201", description = "삭제 성공", content = @Content),
@ApiResponse(responseCode = "400", description = "잘못된 요청", content = @Content),
@ApiResponse(responseCode = "404", description = "데이터셋을 찾을 수 없음", content = @Content),
@ApiResponse(responseCode = "500", description = "서버 오류", content = @Content)
})
@DeleteMapping("/{uuid}")
public ApiResponseDto<UUID> deleteDatasets(@PathVariable UUID uuid) {
datasetClsService.deleteDatasets(uuid);
return ApiResponseDto.ok(uuid);
}
@Operation(summary = "분류 학습데이터 관리 목록 조회", description = "분류 학습데이터 목록을 조회합니다.")
@ApiResponses(
value = {
@ApiResponse(
responseCode = "200",
description = "조회 성공",
content =
@Content(
mediaType = "application/json",
schema = @Schema(implementation = Page.class))),
@ApiResponse(responseCode = "400", description = "잘못된 검색 조건", content = @Content),
@ApiResponse(responseCode = "500", description = "서버 오류", content = @Content)
})
@GetMapping("/obj-list")
public ApiResponseDto<Page<DatasetClsObjDto.Basic>> searchDatasetObjectList(
@Parameter(description = "회차 uuid", example = "e9a6774b-4f81-4402-b080-51d27fac1f01")
@RequestParam(required = true)
UUID uuid,
@Parameter(description = "년도", example = "2023") @RequestParam(required = false) Integer yyyy,
@Parameter(description = "분류", example = "container") @RequestParam(required = false)
String classCd,
@Parameter(description = "도엽번호", example = "36713060") @RequestParam(required = false)
String mapSheetNum,
@RequestParam(defaultValue = "0") int page,
@RequestParam(defaultValue = "20") int size) {
DatasetClsObjDto.ClsSearchReq searchReq = new DatasetClsObjDto.ClsSearchReq();
searchReq.setUuid(uuid);
searchReq.setYyyy(yyyy);
searchReq.setClassCd(classCd);
searchReq.setMapSheetNum(mapSheetNum);
searchReq.setPage(page);
searchReq.setSize(size);
return ApiResponseDto.ok(datasetClsService.searchDatasetClsObjectList(searchReq));
}
@Operation(summary = "학습데이터 관리 obj 삭제", description = "학습데이터 관리 obj 삭제 API")
@ApiResponses(
value = {
@ApiResponse(
responseCode = "200",
description = "삭제 성공",
content =
@Content(
mediaType = "application/json",
schema = @Schema(implementation = Page.class))),
@ApiResponse(responseCode = "400", description = "잘못된 검색 조건", content = @Content),
@ApiResponse(responseCode = "500", description = "서버 오류", content = @Content)
})
@DeleteMapping("/obj/{uuid}")
public ApiResponseDto<UUID> deleteDatasetObjByUuid(@PathVariable UUID uuid) {
return ApiResponseDto.ok(datasetClsService.deleteDatasetObjByUuid(uuid));
}
@Operation(summary = "학습데이터 결과 class 조회", description = "학습데이터 결과 class 조회 API")
@ApiResponses(
value = {
@ApiResponse(
responseCode = "200",
description = "조회 성공",
content =
@Content(
mediaType = "application/json",
schema = @Schema(implementation = Page.class))),
@ApiResponse(responseCode = "400", description = "잘못된 검색 조건", content = @Content),
@ApiResponse(responseCode = "500", description = "서버 오류", content = @Content)
})
@GetMapping("/class/{uuid}")
public ApiResponseDto<List<DatasetClass>> getDatasetObjByUuid(
@Parameter(description = "dataset uuid", example = "e1416f32-769f-495c-a883-3ebfacef4bac")
@PathVariable
UUID uuid,
@Parameter(description = "compare, target", example = "compare") @RequestParam String type) {
return ApiResponseDto.ok(datasetClsService.getDatasetObjByUuid(uuid, type));
}
@Operation(summary = "남은 저장공간 조회", description = "남은 저장공간 조회 API")
@ApiResponses(
value = {
@ApiResponse(
responseCode = "200",
description = "조회 성공",
content =
@Content(
mediaType = "application/json",
schema = @Schema(implementation = Page.class))),
@ApiResponse(responseCode = "404", description = "저장 공간 조회 오류", content = @Content),
@ApiResponse(responseCode = "500", description = "서버 오류", content = @Content)
})
@GetMapping("/usable-bytes")
public ApiResponseDto<DatasetStorage> getUsableBytes() throws IOException {
FileStore store = Files.getFileStore(Path.of("."));
long usable = store.getUsableSpace();
DatasetStorage storage = new DatasetStorage();
storage.setUsableBytes(String.valueOf(usable));
// datasetClsService.getUsableBytes();
return ApiResponseDto.ok(storage);
}
@Operation(summary = "학습데이터 zip파일 등록", description = "학습데이터 zip파일 등록 합니다.")
@PostMapping
public ApiResponseDto<ApiResponseDto.ResponseObj> insertDataset(
@RequestBody @Valid DatasetDto.AddReq addReq) {
return ApiResponseDto.okObject(datasetClsService.insertDataset(addReq));
}
@Operation(summary = "분류데이터셋 객체별 파일 Path 조회", description = "파일 Path 조회")
@GetMapping("/files")
public ResponseEntity<Resource> getFile(@RequestParam UUID uuid) throws Exception {
String path = datasetClsService.getFilePathByUUIDPathTif(uuid);
return datasetClsService.getFilePathByFile(path);
}
@Operation(summary = "납품 폴더 조회", description = "납품 폴더 조회 API")
@ApiResponses(
value = {
@ApiResponse(
responseCode = "200",
description = "조회 성공",
content =
@Content(
mediaType = "application/json",
schema = @Schema(implementation = FoldersDto.class))),
@ApiResponse(responseCode = "404", description = "조회 오류", content = @Content),
@ApiResponse(responseCode = "500", description = "서버 오류", content = @Content)
})
@PostMapping("/folder-list")
public ApiResponseDto<FoldersDto> getDir(@RequestBody SrchFoldersDto srchDto) throws IOException {
return ApiResponseDto.createOK(datasetClsService.getFolderAll(srchDto));
}
@Operation(summary = "납품 학습데이터셋 등록", description = "납품 학습데이터셋 등록 API")
@ApiResponses(
value = {
@ApiResponse(
responseCode = "200",
description = "등록 성공",
content =
@Content(
mediaType = "application/json",
schema = @Schema(implementation = String.class))),
@ApiResponse(responseCode = "404", description = "조회 오류", content = @Content),
@ApiResponse(responseCode = "500", description = "서버 오류", content = @Content)
})
@PostMapping("/deliveries")
public ApiResponseDto<String> insertDeliveriesDataset(@RequestBody AddDeliveriesReq req) {
// 폴더 구조 검증
DatasetService.validateTrainValTestDirs(req.getFilePath());
// 파일 개수 검증
DatasetService.validateDirFileCount(req.getFilePath());
// 폴더명(uid)으로 등록한 건이 있는지 체크
datasetClsService.validateExistsUidChk(req.getFilePath());
datasetAsyncService.insertDeliveriesDatasetAsync(req);
return ApiResponseDto.createOK("ok");
}
}

View File

@@ -0,0 +1,160 @@
package com.kamco.cd.training.dataset.dto;
import com.fasterxml.jackson.databind.JsonNode;
import com.kamco.cd.training.common.enums.DetectionClassification;
import com.kamco.cd.training.common.utils.HeaderUtil;
import com.kamco.cd.training.common.utils.enums.CodeExpose;
import com.kamco.cd.training.common.utils.enums.EnumType;
import com.kamco.cd.training.common.utils.interfaces.JsonFormatDttm;
import io.swagger.v3.oas.annotations.media.Schema;
import java.time.ZonedDateTime;
import java.util.UUID;
import lombok.*;
import lombok.extern.slf4j.Slf4j;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Pageable;
import org.springframework.data.domain.Sort;
@Slf4j
public class DatasetClsObjDto {
@Schema(name = "DatasetClsObj Basic", description = "분류 데이터셋 객체 Obj 기본 정보")
@Getter
@Setter
@NoArgsConstructor
public static class Basic {
private Long objId;
private UUID uuid;
private Long datasetUid;
private String type;
private Integer yyyy;
private String classCd;
private String mapSheetNum;
private String tifImgPath;
private String fileName;
@JsonFormatDttm private ZonedDateTime createdDttm;
private Long createdUid;
private Boolean deleted;
public Basic(
Long objId,
UUID uuid,
Long datasetUid,
String type,
Integer yyyy,
String classCd,
String mapSheetNum,
String tifImgPath,
String fileName,
ZonedDateTime createdDttm,
Long createdUid,
Boolean deleted) {
this.objId = objId;
this.uuid = uuid;
this.datasetUid = datasetUid;
this.type = type;
this.yyyy = yyyy;
this.classCd = classCd;
this.mapSheetNum = mapSheetNum;
this.tifImgPath = tifImgPath;
this.fileName = fileName;
this.createdDttm = createdDttm;
this.createdUid = createdUid;
this.deleted = deleted;
}
}
@Schema(name = "ClsSearchReq", description = "분류 데이터셋 상세 도엽목록 조회 요청")
@Getter
@Setter
@NoArgsConstructor
@AllArgsConstructor
public static class ClsSearchReq {
@Schema(description = "회차 uuid", example = "e9a6774b-4f81-4402-b080-51d27fac1f01")
private UUID uuid;
@Schema(description = "년도", example = "2021")
private Integer yyyy;
@Schema(description = "분류", example = "waste")
private String classCd;
@Schema(description = "도엽번호", example = "36713060")
private String mapSheetNum;
@Schema(description = "페이지 번호 (0부터 시작)", example = "0")
private int page = 0;
@Schema(description = "페이지 크기", example = "20")
private int size = 20;
public Pageable toPageable() {
return PageRequest.of(page, size, Sort.by(Sort.Direction.DESC, "createdDttm"));
}
}
@Getter
@Setter
@AllArgsConstructor
@NoArgsConstructor
public static class DatasetClass {
private String classCd;
public String getClassName() {
return HeaderUtil.isEnglishRequest()
? DetectionClassification.fromString(classCd).getId()
: DetectionClassification.fromString(classCd).getDesc();
// fromString 메서드를 사용하여 안전하게 변환 (미정의 값은 ETC로 처리)
// return DetectionClassification.valueOf(classCd.toUpperCase()).getDesc();
}
}
@Getter
@Setter
public static class DatasetStorage {
private String usableBytes;
}
@Getter
@Setter
@NoArgsConstructor
@AllArgsConstructor
@Builder
public static class DatasetObjRegDto {
private Long datasetUid;
private Integer compareYyyy;
private String compareClassCd;
private Integer targetYyyy;
private String targetClassCd;
private String comparePath;
private String targetPath;
private String labelPath;
private String geojsonPath;
private String mapSheetNum;
private JsonNode geojson;
private String fileName;
}
@CodeExpose
@Getter
@AllArgsConstructor
public enum FolderType implements EnumType {
TRAIN("train"),
VAL("val"),
TEST("test");
private String desc;
@Override
public String getId() {
return name();
}
@Override
public String getText() {
return desc;
}
}
}

View File

@@ -4,6 +4,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore;
import com.kamco.cd.training.common.enums.LearnDataRegister;
import com.kamco.cd.training.common.enums.LearnDataType;
import com.kamco.cd.training.common.enums.ModelType;
import com.kamco.cd.training.common.utils.HeaderUtil;
import com.kamco.cd.training.common.utils.enums.Enums;
import com.kamco.cd.training.common.utils.interfaces.JsonFormatDttm;
import io.swagger.v3.oas.annotations.media.Schema;
@@ -46,6 +47,7 @@ public class DatasetDto {
private String statusCd;
private Boolean deleted;
private String dataType;
private String cdClsType;
public Basic(
Long id,
@@ -59,7 +61,8 @@ public class DatasetDto {
ZonedDateTime createdDttm,
String status,
Boolean deleted,
String dataType) {
String dataType,
String cdClsType) {
this.id = id;
this.uuid = uuid;
this.title = title;
@@ -73,6 +76,7 @@ public class DatasetDto {
this.statusCd = status;
this.deleted = deleted;
this.dataType = dataType;
this.cdClsType = cdClsType;
}
public String getTotalSize(Long totalSize) {
@@ -90,7 +94,8 @@ public class DatasetDto {
public String getStatus(String status) {
LearnDataRegister type = Enums.fromId(LearnDataRegister.class, status);
return type == null ? null : type.getText();
boolean english = HeaderUtil.isEnglishRequest();
return type == null ? null : (english ? type.getId() : type.getText());
}
public String getYear() {
@@ -99,7 +104,8 @@ public class DatasetDto {
public String getDataTypeName() {
LearnDataType type = Enums.fromId(LearnDataType.class, this.dataType);
return type == null ? null : type.getText();
boolean english = HeaderUtil.isEnglishRequest();
return type == null ? null : (english ? type.getId() : type.getText());
}
}
@@ -315,7 +321,7 @@ public class DatasetDto {
public String getDataTypeName(String groupTitleCd) {
LearnDataType type = Enums.fromId(LearnDataType.class, groupTitleCd);
return type == null ? null : type.getText();
return type == null ? null : (HeaderUtil.isEnglishRequest() ? type.getId() : type.getText());
}
public String getYear() {
@@ -539,6 +545,7 @@ public class DatasetDto {
private Long totalSize;
private Long totalObjectCount;
private String datasetPath;
private String cdClsType;
}
@Getter

View File

@@ -5,6 +5,7 @@ import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.kamco.cd.training.common.enums.DetectionClassification;
import com.kamco.cd.training.common.utils.HeaderUtil;
import com.kamco.cd.training.common.utils.interfaces.JsonFormatDttm;
import io.swagger.v3.oas.annotations.media.Schema;
import java.time.ZonedDateTime;
@@ -131,7 +132,10 @@ public class DatasetObjDto {
private String classCd;
public String getClassName() {
return DetectionClassification.fromString(classCd).getDesc();
return HeaderUtil.isEnglishRequest()
? DetectionClassification.fromString(classCd).getId()
: DetectionClassification.fromString(classCd).getDesc();
// fromString 메서드를 사용하여 안전하게 변환 (미정의 값은 ETC로 처리)
// return DetectionClassification.valueOf(classCd.toUpperCase()).getDesc();
}
@@ -162,4 +166,19 @@ public class DatasetObjDto {
private JsonNode geojson;
private String fileName;
}
@Getter
@Setter
@NoArgsConstructor
@AllArgsConstructor
@Builder
public static class DatasetClsObjRegDto {
private Long datasetUid;
private String type;
private Integer yyyy;
private String classCd;
private String mapSheetNum;
private String tifImgPath;
private String fileName;
}
}

View File

@@ -4,7 +4,11 @@ import com.kamco.cd.training.common.enums.LearnDataRegister;
import com.kamco.cd.training.dataset.dto.DatasetDto.AddDeliveriesReq;
import com.kamco.cd.training.dataset.dto.DatasetDto.DatasetMngRegDto;
import com.kamco.cd.training.postgres.core.DatasetCoreService;
import java.util.UUID;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.stream.Stream;
import lombok.RequiredArgsConstructor;
import lombok.extern.log4j.Log4j2;
import org.springframework.scheduling.annotation.Async;
@@ -45,7 +49,11 @@ public class DatasetAsyncService {
try {
// ===== 1. UID 생성 =====
String uid = UUID.randomUUID().toString().replace("-", "").toUpperCase();
// String uid = UUID.randomUUID().toString().replace("-", "").toUpperCase();
// 26-05-08: UID를 생성하지 않고 folder name 을 uid 로 저장하기 -> req.getFilePath()
Path selectedPath = Paths.get(req.getFilePath());
String uid = selectedPath.getFileName().toString();
log.info("{} 생성된 UID: {}", LOG_PREFIX, uid);
// ===== 2. 마스터 데이터 생성 =====
@@ -71,6 +79,7 @@ public class DatasetAsyncService {
datasetMngRegDto.setTitle(title);
datasetMngRegDto.setMemo(req.getMemo());
datasetMngRegDto.setDatasetPath(req.getFilePath());
datasetMngRegDto.setTotalSize(getDirectorySize(req.getFilePath())); // 선택한 폴더 안의 모든 파일 용량 합계
// 마스터 저장
datasetUid = datasetCoreService.insertDatasetMngData(datasetMngRegDto);
@@ -121,4 +130,24 @@ public class DatasetAsyncService {
}
}
}
private Long getDirectorySize(String filePath) {
Path selectedPath = Paths.get(filePath);
try (Stream<Path> paths = Files.walk(selectedPath)) {
return paths
.filter(Files::isRegularFile)
.mapToLong(
path -> {
try {
return Files.size(path);
} catch (IOException e) {
return 0L;
}
})
.sum();
} catch (IOException e) {
return 0L;
}
}
}

View File

@@ -0,0 +1,825 @@
package com.kamco.cd.training.dataset.service;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.kamco.cd.training.common.enums.LearnDataRegister;
import com.kamco.cd.training.common.enums.LearnDataType;
import com.kamco.cd.training.common.exception.CustomApiException;
import com.kamco.cd.training.common.service.FormatStorage;
import com.kamco.cd.training.common.utils.FIleChecker;
import com.kamco.cd.training.config.api.ApiResponseDto.ApiResponseCode;
import com.kamco.cd.training.config.api.ApiResponseDto.ResponseObj;
import com.kamco.cd.training.dataset.dto.DatasetClsObjDto;
import com.kamco.cd.training.dataset.dto.DatasetDto;
import com.kamco.cd.training.dataset.dto.DatasetDto.AddDeliveriesReq;
import com.kamco.cd.training.dataset.dto.DatasetDto.AddReq;
import com.kamco.cd.training.dataset.dto.DatasetDto.DatasetMngRegDto;
import com.kamco.cd.training.dataset.dto.DatasetObjDto;
import com.kamco.cd.training.dataset.dto.DatasetObjDto.DatasetClass;
import com.kamco.cd.training.dataset.dto.DatasetObjDto.DatasetObjRegDto;
import com.kamco.cd.training.dataset.dto.DatasetObjDto.DatasetStorage;
import com.kamco.cd.training.model.dto.FileDto.FoldersDto;
import com.kamco.cd.training.model.dto.FileDto.SrchFoldersDto;
import com.kamco.cd.training.postgres.core.DatasetClsCoreService;
import jakarta.validation.Valid;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.core.io.InputStreamResource;
import org.springframework.core.io.Resource;
import org.springframework.data.domain.Page;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
@Slf4j
@Service
@RequiredArgsConstructor
public class DatasetClsService {
private final DatasetClsCoreService datasetClsCoreService;
private final DatasetBatchService datasetBatchService;
private static final List<String> LABEL_DIRS = List.of("label-json", "label", "input1", "input2");
private static final List<String> REQUIRED_DIRS = Arrays.asList("train", "val", "test");
private static final List<String> CHECK_DIRS = List.of("label", "input1", "input2");
/**
* 데이터셋 목록 조회
*
* @param searchReq 검색 조건
* @return 데이터셋 목록
*/
public Page<DatasetDto.Basic> searchDatasets(DatasetDto.SearchReq searchReq) {
log.info("데이터셋 목록 조회 - 조건: {}", searchReq);
return datasetClsCoreService.findDatasetList(searchReq);
}
/**
* 데이터셋 상세 조회
*
* @param id 상세 조회할 목록 Id
* @return 데이터셋 상세 정보
*/
public DatasetDto.Basic getDatasetDetail(UUID id) {
return datasetClsCoreService.getOneByUuid(id);
}
// TODO 미사용시작
/**
* 데이터셋 등록
*
* @param registerReq 등록 요청
* @return 등록된 데이터셋 ID
*/
@Transactional
public Long registerDataset(DatasetDto.RegisterReq registerReq) {
log.info("데이터셋 등록 - 요청: {}", registerReq);
DatasetDto.Basic saved = datasetClsCoreService.save(registerReq);
log.info("데이터셋 등록 완료 - ID: {}", saved.getId());
return saved.getId();
}
// TODO 미사용 끝
/**
* 데이터셋 수정
*
* @param updateReq 수정 요청
* @return 수정된 데이터셋 ID
*/
@Transactional
public void updateDataset(UUID uuid, DatasetDto.UpdateReq updateReq) {
datasetClsCoreService.update(uuid, updateReq);
}
/**
* 데이터셋 삭제 (다건)
*
* @param uuid 삭제 요청
*/
@Transactional
public void deleteDatasets(UUID uuid) {
datasetClsCoreService.deleteDatasets(uuid);
}
/**
* 데이터셋 통계 요약
*
* @param summaryReq 요약 요청
* @return 통계 요약
*/
public DatasetDto.Summary getDatasetSummary(DatasetDto.SummaryReq summaryReq) {
log.info("데이터셋 통계 요약 - 요청: {}", summaryReq);
return datasetClsCoreService.getDatasetSummary(summaryReq);
}
public Page<DatasetClsObjDto.Basic> searchDatasetClsObjectList(
DatasetClsObjDto.ClsSearchReq searchReq) {
return datasetClsCoreService.searchDatasetClsObjectList(searchReq);
}
public UUID deleteDatasetObjByUuid(UUID uuid) {
return datasetClsCoreService.deleteDatasetObjByUuid(uuid);
}
/**
* 데이터셋 object class 조회
*
* @param uuid dataset uuid
* @param type compare, target
*/
public List<DatasetClass> getDatasetObjByUuid(UUID uuid, String type) {
return datasetClsCoreService.findDatasetObjClassByUuid(uuid, type);
}
/**
* 사용 가능 공간 조회
*
* @return
*/
public DatasetStorage getUsableBytes() {
// 현재 실행 위치가 속한 디스크 기준
try {
FormatStorage.DiskUsage usage = FormatStorage.getDiskUsage(Path.of("."));
log.debug("경로 : {}", usage.path());
log.debug("총 저장공간 : {}", usage.totalText());
log.debug("남은 저장공간 : {}", usage.usableText());
log.debug("사용률 : {}", usage.usedPercent());
DatasetStorage datasetStorage = new DatasetStorage();
datasetStorage.setUsableBytes(usage.usableText());
return datasetStorage;
} catch (Exception e) {
throw new CustomApiException("NOT_FOUND", HttpStatus.NOT_FOUND);
}
}
/**
* 분류모델 업로드
*
* @param addReq
* @return
*/
@Transactional
public ResponseObj insertDataset(@Valid AddReq addReq) {
Long datasetUid = null; // master id 값, 등록하면서 가져올 예정
try {
// 압축 해제
FIleChecker.unzip(addReq.getFileName(), addReq.getFilePath());
// 압축 해제한 폴더 하위에 train,val,test 폴더 모두 존재하는지 확인
validateClsTrainValTestDirs(addReq.getFilePath() + addReq.getFileName().replace(".zip", ""));
// 해제한 폴더 읽어서 데이터 저장
datasetUid =
this.processAndInsertDataset(
addReq.getFilePath() + addReq.getFileName().replace(".zip", ""), addReq);
log.info("######testttest#### datasetUid: {}", datasetUid);
} catch (IOException e) {
log.error(e.getMessage());
return new ResponseObj(ApiResponseCode.INTERNAL_SERVER_ERROR, e.getMessage());
}
datasetClsCoreService.updateDatasetUploadStatus(datasetUid, LearnDataRegister.COMPLETED);
return new ResponseObj(ApiResponseCode.OK, "업로드 성공하였습니다.");
}
private List<Map<String, Object>> getUnzipDatasetFiles(String unzipRootPath, String subDir) {
Path root = Paths.get(unzipRootPath).resolve(subDir);
Map<String, Map<String, Object>> grouped = new HashMap<>();
for (String dirName : LABEL_DIRS) {
Path dir = root.resolve(dirName);
if (!Files.isDirectory(dir)) {
throw new CustomApiException(
ApiResponseCode.NOT_FOUND_DATA.getId(),
HttpStatus.CONFLICT,
"폴더가 존재하지 않습니다. 업로드 된 파일을 확인하세요. : " + dir);
}
try (Stream<Path> stream = Files.list(dir)) {
stream
.filter(Files::isRegularFile)
.forEach(
path -> {
String fileName = path.getFileName().toString();
String baseName = getBaseName(fileName);
// baseName 기준 Map 생성
Map<String, Object> data =
grouped.computeIfAbsent(baseName, k -> new HashMap<>());
// 공통 메타
data.put("baseName", baseName);
// 폴더별 처리
if ("label-json".equals(dirName)) {
// json 파일이면 파싱
try {
data.put("label-json", readJson(path));
} catch (Exception e) {
log.error("파일 JSON 읽기 실패. skip. file={}", path, e);
return; // skip
}
data.put("geojson_path", path.toAbsolutePath().toString());
} else {
data.put(dirName, path.toAbsolutePath().toString());
}
});
} catch (IOException e) {
throw new RuntimeException(e);
}
}
return new ArrayList<>(grouped.values());
}
@Transactional
public Long insertTrainTestData(
Map<String, Object> map, AddReq addReq, int idx, Long datasetUid, String subDir) {
ObjectMapper mapper = new ObjectMapper();
String comparePath = (String) map.get("input1");
String targetPath = (String) map.get("input2");
String labelPath = (String) map.get("label");
String geojsonPath = (String) map.get("geojson_path");
Object labelJson = map.get("label-json");
JsonNode json;
if (labelJson instanceof JsonNode jn) {
json = jn;
} else {
try {
json = mapper.readTree(labelJson.toString());
} catch (IOException e) {
throw new RuntimeException("label_json parse error", e);
}
}
String fileName = Paths.get(comparePath).getFileName().toString();
String[] fileNameStr = fileName.split("_");
String compareYyyy = fileNameStr[1];
String targetYyyy = fileNameStr[2];
String mapSheetNum = fileNameStr[3];
if (idx == 0 && subDir.equals("train")) {
String title = compareYyyy + "-" + targetYyyy; // 제목 : 비교년도-기준년도
String dataType = LearnDataType.PRODUCTION.getId(); // 만들어 넣는 건 다 제작
Long stage =
datasetClsCoreService.getDatasetMaxStage(
Integer.parseInt(compareYyyy), Integer.parseInt(targetYyyy))
+ 1;
String uid = addReq.getFileName().replace(".zip", "");
DatasetMngRegDto mngRegDto =
DatasetMngRegDto.builder()
.uid(uid)
.dataType(dataType)
.compareYyyy(Integer.parseInt(compareYyyy))
.targetYyyy(Integer.parseInt(targetYyyy))
.title(title)
.memo(addReq.getMemo())
.roundNo(stage)
.totalSize(addReq.getFileSize())
.datasetPath(addReq.getFilePath() + uid)
.build();
datasetUid = datasetClsCoreService.insertDatasetMngData(mngRegDto); // tb_dataset 에 insert
}
// datasetUid 로 obj 도 등록하기
// Json 갯수만큼 for문 돌려서 insert 해야 함, features에 빈값
if (json != null && json.path("features") != null && !json.path("features").isEmpty()) {
for (JsonNode feature : json.path("features")) {
JsonNode prop = feature.path("properties");
String compareClassCd = prop.path("before").asText(null);
String targetClassCd = prop.path("after").asText(null);
// 한 개씩 자른 geojson을 FeatureCollection 으로 만들어서 넣기
ObjectNode root = mapper.createObjectNode();
root.put("type", "FeatureCollection");
ArrayNode features = mapper.createArrayNode();
features.add(feature);
root.set("features", features);
DatasetObjRegDto objRegDto =
DatasetObjRegDto.builder()
.datasetUid(datasetUid)
.compareYyyy(Integer.parseInt(compareYyyy))
.compareClassCd(compareClassCd)
.targetYyyy(Integer.parseInt(targetYyyy))
.targetClassCd(targetClassCd)
.comparePath(comparePath)
.targetPath(targetPath)
.labelPath(labelPath)
.mapSheetNum(mapSheetNum)
.geojson(root)
.geojsonPath(geojsonPath)
.fileName(fileName)
.build();
if (subDir.equals("train")) {
datasetClsCoreService.insertDatasetObj(objRegDto);
} else if (subDir.equals("val")) {
datasetClsCoreService.insertDatasetValObj(objRegDto);
} else {
datasetClsCoreService.insertDatasetTestObj(objRegDto);
}
}
}
return datasetUid;
}
/**
* CLS zip file DB화
*
* @param unzipRootPath
* @param addReq
* @return
*/
@Transactional
public Long processAndInsertDataset(String unzipRootPath, AddReq addReq) {
Path datasetRoot = Paths.get(unzipRootPath).resolve("dataset");
log.info("datasetRoot: {}", datasetRoot);
Long datasetUid = null;
boolean isMasterInserted = false; // Master(Mng) 테이블에 최초 1회만 등록하기 위한 플래그
// 1. 순회할 상위 폴더 목록
for (String subDir : REQUIRED_DIRS) {
Path subDirPath = datasetRoot.resolve(subDir);
if (!Files.exists(subDirPath) || !Files.isDirectory(subDirPath)) {
continue; // 해당 폴더가 없으면 다음으로 패스
}
try (Stream<Path> classDirs = Files.list(subDirPath)) {
// 2. 클래스명 폴더 목록 추출 (예: building, container, water 등)
List<Path> classDirList = classDirs.filter(Files::isDirectory).toList();
for (Path classDir : classDirList) {
// DB에 넣을 classCd (폴더명) 추출
String classCd = classDir.getFileName().toString();
try (Stream<Path> files = Files.list(classDir)) {
// 3. 해당 클래스 폴더 안의 실제 파일(tif) 목록 추출
List<Path> filePaths =
files
.filter(Files::isRegularFile)
.sorted(Comparator.comparing(Path::getFileName)) // 정렬 로직 추가
.toList();
if (filePaths.isEmpty()) continue; // 폴더가 비어있으면 에러 방지를 위해 패스
// ★ 맨 앞의 첫 번째 파일명에서 compareYyyy 추출 및 targetYyyy 계산
String firstFileName = filePaths.getFirst().getFileName().toString();
String compareYyyy = firstFileName.split("_")[0]; // 예: "2021"
String targetYyyy = String.valueOf(Integer.parseInt(compareYyyy) + 1); // 예: "2022"
for (Path file : filePaths) {
String fileName = file.getFileName().toString();
String absolutePath = file.toAbsolutePath().toString();
String yyyy = fileName.split("_")[0];
// 4. 파일명 파싱 (확장자 제거 후 Split)
// ※ 주의: 첨부하신 이미지의 파일명(2021_33607077_idx93)을 기준으로 인덱스를 재조정했습니다.
String nameWithoutExt = fileName.substring(0, fileName.lastIndexOf('.'));
String[] fileNameStr = nameWithoutExt.split("_");
String mapSheetNum = fileNameStr.length > 1 ? fileNameStr[1] : ""; // 33607077
// 5. Master(Mng) 데이터 등록 (전체 순회 중 최초 1회만 실행)
if (!isMasterInserted) {
String title = compareYyyy + "-" + targetYyyy;
Long stage =
datasetClsCoreService.getDatasetMaxStage(
Integer.parseInt(compareYyyy), Integer.parseInt(targetYyyy))
+ 1;
String uid = addReq.getFileName().replace(".zip", "");
DatasetMngRegDto mngRegDto =
DatasetMngRegDto.builder()
.uid(uid)
.dataType(LearnDataType.PRODUCTION.getId())
.compareYyyy(Integer.parseInt(compareYyyy))
.targetYyyy(Integer.parseInt(targetYyyy))
.title(title)
.memo(addReq.getMemo())
.roundNo(stage)
.totalSize(addReq.getFileSize())
.datasetPath(addReq.getFilePath() + uid)
.cdClsType("CLS")
.build();
datasetUid = datasetClsCoreService.insertDatasetMngData(mngRegDto);
isMasterInserted = true;
}
// 6. Obj(상세) 데이터 등록
DatasetObjDto.DatasetClsObjRegDto objRegDto =
DatasetObjDto.DatasetClsObjRegDto.builder()
.datasetUid(datasetUid)
.type(subDir)
.yyyy(Integer.parseInt(yyyy))
.classCd(classCd)
.mapSheetNum(mapSheetNum)
.tifImgPath(absolutePath) // 실제 파일의 절대 경로
.fileName(fileName)
.build();
// 7. 폴더 위치(train/val/test)에 맞는 테이블에 Insert
datasetClsCoreService.insertDatasetClsObj(objRegDto);
}
}
}
} catch (IOException e) {
throw new RuntimeException("디렉토리 읽기 중 오류가 발생했습니다: " + subDirPath, e);
}
}
log.info("$$$$$$$$$$datasetUid: {}", datasetUid);
return datasetUid;
}
/**
* Cls File 목록
*
* @param unzipRootPath
* @param subDir
* @return
*/
private List<String> getUnzipDatasetClsFiles(String unzipRootPath, String subDir) {
Path root = Paths.get(unzipRootPath).resolve("dataset");
Map<String, Map<String, Object>> grouped = new HashMap<>();
Path dir = root.resolve(subDir);
if (!Files.isDirectory(dir)) {
throw new CustomApiException(
ApiResponseCode.NOT_FOUND_DATA.getId(),
HttpStatus.CONFLICT,
"폴더가 존재하지 않습니다. 업로드 된 파일을 확인하세요. : " + dir);
}
try (Stream<Path> stream = Files.list(dir)) {
return stream
.filter(Files::isRegularFile) // 파일만 필터링 (하위 폴더 제외)
.map(path -> path.toAbsolutePath().toString()) // 절대 경로 문자열로 변환
.toList(); // List로 수집 (Java 16 미만인 경우 .collect(Collectors.toList()) 사용)
} catch (IOException e) {
throw new RuntimeException(e);
}
}
private String getBaseName(String fileName) {
int idx = fileName.lastIndexOf('.');
return (idx > 0) ? fileName.substring(0, idx) : fileName;
}
private JsonNode readJson(Path jsonPath) {
try {
ObjectMapper mapper = new ObjectMapper();
return mapper.readTree(jsonPath.toFile());
} catch (IOException e) {
throw new RuntimeException("JSON 읽기 실패: " + jsonPath, e);
}
}
public String getFilePathByUUIDPathType(UUID uuid, String pathType) {
return datasetClsCoreService.getFilePathByUUIDPathType(uuid, pathType);
}
private String escape(String path) {
// 쉘 커맨드에서 안전하게 사용할 수 있도록 문자열을 작은따옴표로 감싸면서, 내부의 작은따옴표를 이스케이프 처리
return "'" + path.replace("'", "'\"'\"'") + "'";
}
private static String normalizeLinuxPath(String path) {
return path.replace("\\", "/");
}
public ResponseEntity<Resource> getFilePathByFile(String remoteFilePath) {
try {
Path path = Paths.get(remoteFilePath);
InputStream inputStream = Files.newInputStream(path);
InputStreamResource resource =
new InputStreamResource(inputStream) {
@Override
public long contentLength() {
return -1; // 알 수 없으면 -1
}
};
String fileName = Paths.get(remoteFilePath.replace("\\", "/")).getFileName().toString();
return ResponseEntity.ok()
.header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + fileName + "\"")
.contentType(MediaType.APPLICATION_OCTET_STREAM)
.body(resource);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
/** unzipRootDir: 압축 해제된 폴더 경로 (ex: /data/xxx/myzipname) */
public static void validateTrainValTestDirs(String unzipRootDir) {
Path root = Paths.get(unzipRootDir);
// 루트 폴더 자체 존재 확인
if (!Files.exists(root) || !Files.isDirectory(root)) {
throw new CustomApiException(
ApiResponseCode.NOT_FOUND_DATA.getId(),
HttpStatus.CONFLICT,
"압축 해제 폴더가 존재하지 않습니다: " + unzipRootDir);
}
// 필요한 폴더들 존재/디렉토리 여부 확인
List<String> missing =
REQUIRED_DIRS.stream()
.filter(d -> !Files.isDirectory(root.resolve(d)))
.collect(Collectors.toList());
if (!missing.isEmpty()) {
throw new CustomApiException(
ApiResponseCode.NOT_FOUND_DATA.getId(),
HttpStatus.CONFLICT,
"데이터 폴더 구조가 올바르지 않습니다. 누락된 폴더: "
+ String.join(", ", missing)
+ " (필수: train, val, test)");
}
}
/** unzipRootDir: 압축 해제된 폴더 경로 (ex: /zip파일폴더/dataset/train,val,test) */
public static void validateClsTrainValTestDirs(String unzipRootDir) {
Path root = Paths.get(unzipRootDir).resolve("dataset");
// 루트 폴더 자체 존재 확인
if (!Files.exists(root) || !Files.isDirectory(root)) {
throw new CustomApiException(
ApiResponseCode.NOT_FOUND_DATA.getId(),
HttpStatus.CONFLICT,
"압축 해제 폴더가 존재하지 않습니다: " + root.toAbsolutePath().toString());
}
// 필요한 폴더들 존재/디렉토리 여부 확인
List<String> missing =
REQUIRED_DIRS.stream()
.filter(d -> !Files.isDirectory(root.resolve(d)))
.collect(Collectors.toList());
if (!missing.isEmpty()) {
throw new CustomApiException(
ApiResponseCode.NOT_FOUND_DATA.getId(),
HttpStatus.CONFLICT,
"데이터 폴더 구조가 올바르지 않습니다. 누락된 폴더: "
+ String.join(", ", missing)
+ " (필수: train, val, test)");
}
}
public static void validateDirFileCount(String unzipRootDir) {
Path root = Paths.get(unzipRootDir);
for (String split : REQUIRED_DIRS) {
Path splitPath = root.resolve(split);
Map<String, Long> fileCountMap = new HashMap<>();
for (String subDir : CHECK_DIRS) { // input1, input2, label 폴더만 수행하기
Path subDirPath = splitPath.resolve(subDir);
if (!Files.isDirectory(subDirPath)) {
throw new CustomApiException(
ApiResponseCode.NOT_FOUND_DATA.getId(),
HttpStatus.CONFLICT,
split + " 폴더 하위에 " + subDir + " 폴더가 존재하지 않습니다.");
}
long count;
try (Stream<Path> files = Files.list(subDirPath)) {
count = files.filter(Files::isRegularFile).count();
log.info("dir: " + subDirPath + ", count: " + count);
} catch (IOException e) {
throw new CustomApiException(
ApiResponseCode.NOT_FOUND_DATA.getId(),
HttpStatus.CONFLICT,
split + "/" + subDir + " 파일 개수 확인 중 오류 발생");
}
fileCountMap.put(subDir, count);
}
// 모든 폴더 파일 개수가 동일한지 확인
Set<Long> uniqueCounts = new HashSet<>(fileCountMap.values());
if (uniqueCounts.size() != 1) {
throw new CustomApiException(
ApiResponseCode.NOT_FOUND_DATA.getId(),
HttpStatus.CONFLICT,
split + " 데이터 파일 개수가 일치하지 않습니다. " + fileCountMap.toString());
}
}
}
/**
* 폴더 조회
*
* @param srchDto 폴더 경로
* @return 폴더 리스트
* @throws IOException
*/
public FoldersDto getFolderAll(SrchFoldersDto srchDto) throws IOException {
File dir = new File(srchDto.getDirPath() == null ? "/" : srchDto.getDirPath());
// 존재 + 디렉토리 체크
if (!dir.exists() || !dir.isDirectory()) {
throw new CustomApiException("BAD_REQUEST", HttpStatus.BAD_REQUEST, "잘못된 경로입니다.");
}
// 권한 없을때
if (!dir.canRead()) {
throw new CustomApiException(
ApiResponseCode.FORBIDDEN.getId(), HttpStatus.FORBIDDEN, "디렉토리에 접근할 권한이 없습니다.");
}
String canonicalPath = dir.getCanonicalPath();
File[] files = dir.listFiles();
if (files == null) {
return new FoldersDto(canonicalPath, 0, 0, Collections.emptyList());
}
List<FIleChecker.Folder> folders = new ArrayList<>();
int folderTotCnt = 0;
int folderErrTotCnt = 0;
for (File f : files) {
// 숨김 제외
if (f.isHidden()) continue;
if (f.isDirectory()) {
// 폴더 개수 증가
folderTotCnt++;
// 폴더 유효성 여부 (기본 true, 이후 검증 로직으로 변경 가능)
boolean isValid = true;
// 유효하지 않은 폴더 카운트 증가
if (!isValid) folderErrTotCnt++;
// 현재 폴더 이름 (ex: train, images 등)
String folderNm = f.getName();
// 부모 경로 (ex: /data/datasets)
String parentPath = f.getParent();
// 부모 폴더 이름 (ex: datasets)
String parentFolderNm = new File(parentPath).getName();
// 전체 절대 경로 (ex: /data/datasets/train)
String fullPath = f.getAbsolutePath();
// 폴더 깊이 (경로 기준 depth)
// ex: /a/b/c → depth = 3
int depth = f.toPath().getNameCount();
// 하위 폴더 개수
long childCnt = FIleChecker.getChildFolderCount(f);
// 마지막 수정 시간 (문자열 포맷)
String lastModified = FIleChecker.getLastModified(f);
// Folder DTO 생성 및 리스트에 추가
folders.add(
new FIleChecker.Folder(
folderNm, // 폴더명
parentFolderNm, // 부모 폴더명
parentPath, // 부모 경로
fullPath, // 전체 경로
depth, // 깊이
childCnt, // 하위 폴더 개수
lastModified, // 수정일시
isValid // 유효성 여부
));
}
}
// 폴더 정렬
folders.sort(
Comparator.comparing(FIleChecker.Folder::getFolderNm, String.CASE_INSENSITIVE_ORDER));
return new FoldersDto(canonicalPath, folderTotCnt, folderErrTotCnt, folders);
}
/**
* 납품 데이터 등록
*
* @param req 폴더경로, 메모
* @return 성공/실패 여부0
*/
public void insertDeliveriesDataset(AddDeliveriesReq req, Long datasetUid) {
long startTime = System.currentTimeMillis();
// 처리
processType(req.getFilePath(), datasetUid, "train");
processType(req.getFilePath(), datasetUid, "val");
processType(req.getFilePath(), datasetUid, "test");
log.info("========== 전체 완료. 총 소요시간: {} ms ==========", System.currentTimeMillis() - startTime);
}
/**
* 납품 데이터 등록 처리
*
* @param path
* @param datasetUid
* @param type
*/
private void processType(String path, Long datasetUid, String type) {
long start = System.currentTimeMillis();
log.info("[납품 데이터 등록 처리][{}] 시작", type.toUpperCase());
List<Map<String, Object>> list = getUnzipDatasetFiles(path, type);
int batchSize = 1000;
int total = list.size();
int processed = 0;
for (int i = 0; i < total; i += batchSize) {
List<Map<String, Object>> batch = list.subList(i, Math.min(i + batchSize, total));
try {
log.info("[납품 데이터 등록 처리][{}] batch 시작: {} ~ {}", type, i, i + batch.size());
datasetBatchService.saveBatch(batch, datasetUid, type);
processed += batch.size();
} catch (Exception e) {
log.error("batch 실패 row 데이터: {}", batch);
log.error(
"[납품 데이터 등록 처리][{}] batch 실패. range: {} ~ {}, datasetUid={}",
type,
i,
i + batch.size(),
datasetUid,
e);
}
}
log.info(
"[납품 데이터 등록 처리][{}] 완료. 총 {}건, 소요시간: {} ms",
type,
total,
System.currentTimeMillis() - start);
}
public void validateExistsUidChk(String filePath) {
Path selectedPath = Paths.get(filePath);
String uid = selectedPath.getFileName().toString();
// 같은 uid 로 등록한 파일이 있는지 확인
Long existsCnt = datasetClsCoreService.findDatasetByUidExistsCnt(uid);
if (existsCnt > 0) {
throw new CustomApiException(
ApiResponseCode.DUPLICATE_DATA.getId(),
HttpStatus.CONFLICT,
"이미 등록된 회차 데이터 파일입니다. 확인 부탁드립니다.");
}
}
public String getFilePathByUUIDPathTif(UUID uuid) {
return datasetClsCoreService.getFilePathByUUIDPathTif(uuid);
}
}

View File

@@ -286,7 +286,7 @@ public class DatasetService {
.memo(addReq.getMemo())
.roundNo(stage)
.totalSize(addReq.getFileSize())
.datasetPath(addReq.getFilePath())
.datasetPath(addReq.getFilePath() + uid)
.build();
datasetUid = datasetCoreService.insertDatasetMngData(mngRegDto); // tb_dataset 에 insert
@@ -676,4 +676,18 @@ public class DatasetService {
total,
System.currentTimeMillis() - start);
}
public void validateExistsUidChk(String filePath) {
Path selectedPath = Paths.get(filePath);
String uid = selectedPath.getFileName().toString();
// 같은 uid 로 등록한 파일이 있는지 확인
Long existsCnt = datasetCoreService.findDatasetByUidExistsCnt(uid);
if (existsCnt > 0) {
throw new CustomApiException(
ApiResponseCode.DUPLICATE_DATA.getId(),
HttpStatus.CONFLICT,
"이미 등록된 회차 데이터 파일입니다. 확인 부탁드립니다.");
}
}
}

View File

@@ -277,9 +277,7 @@ public class FileManagerDto {
@Schema(description = "디스크 사용률 (%)", example = "50.5")
private Double usagePercentage;
@JsonFormatDttm
private ZonedDateTime lastModifiedDate;
@JsonFormatDttm private ZonedDateTime lastModifiedDate;
public ZonedDateTime getLastModifiedDate() {
return ZonedDateTime.now();

View File

@@ -627,8 +627,7 @@ public class FileManagerService {
duPb.redirectErrorStream(true);
Process duProcess = duPb.start();
try (java.io.BufferedReader br =
new java.io.BufferedReader(
new java.io.InputStreamReader(duProcess.getInputStream()))) {
new java.io.BufferedReader(new java.io.InputStreamReader(duProcess.getInputStream()))) {
String line = br.readLine();
if (line != null) {
// du -sb 출력 형식: "12345678\t/path/to/dir"

View File

@@ -236,4 +236,41 @@ public class ModelTrainMngApiController {
public ApiResponseDto<MonitorDto> getSystem() throws IOException {
return ApiResponseDto.ok(systemMonitorService.get());
}
@Operation(summary = "모델학습 1단계/2단계 실행중인 것 id 정보", description = "모델학습 1단계/2단계 실행중인 것 id 정보")
@ApiResponses(
value = {
@ApiResponse(
responseCode = "200",
description = "검색 성공",
content =
@Content(
mediaType = "application/json",
schema = @Schema(implementation = Long.class))),
@ApiResponse(responseCode = "400", description = "잘못된 검색 조건", content = @Content),
@ApiResponse(responseCode = "500", description = "서버 오류", content = @Content)
})
@GetMapping("/ing-model")
public ApiResponseDto<ModelTrainMngDto.InProgressModel> findInprogressModel() {
return ApiResponseDto.ok(modelTrainMngService.findInprogressModel());
}
@Operation(summary = "모델학습 진행율 퍼센트", description = "모델학습 진행율 퍼센트")
@ApiResponses(
value = {
@ApiResponse(
responseCode = "200",
description = "검색 성공",
content =
@Content(
mediaType = "application/json",
schema = @Schema(implementation = Long.class))),
@ApiResponse(responseCode = "400", description = "잘못된 검색 조건", content = @Content),
@ApiResponse(responseCode = "500", description = "서버 오류", content = @Content)
})
@GetMapping("/progress-percent/{uuid}")
public ApiResponseDto<ModelTrainMngDto.ProgressPercent> findTrainProgressPercent(
@PathVariable UUID uuid) {
return ApiResponseDto.ok(modelTrainMngService.findTrainProgressPercent(uuid));
}
}

View File

@@ -4,6 +4,7 @@ import com.fasterxml.jackson.annotation.JsonInclude;
import com.kamco.cd.training.common.enums.LearnDataType;
import com.kamco.cd.training.common.enums.TrainStatusType;
import com.kamco.cd.training.common.enums.TrainType;
import com.kamco.cd.training.common.utils.HeaderUtil;
import com.kamco.cd.training.common.utils.enums.Enums;
import com.kamco.cd.training.common.utils.interfaces.JsonFormatDttm;
import com.kamco.cd.training.dataset.dto.DatasetDto.SelectTransferDataSet;
@@ -32,6 +33,8 @@ public class ModelTrainDetailDto {
private String modelNo;
private String modelVer;
@JsonFormatDttm private ZonedDateTime step1StrtDttm;
@JsonFormatDttm private ZonedDateTime step1EndDttm;
@JsonFormatDttm private ZonedDateTime step2StrtDttm;
@JsonFormatDttm private ZonedDateTime step2EndDttm;
private String statusCd;
private String trainType;
@@ -40,7 +43,9 @@ public class ModelTrainDetailDto {
public String getStatusName() {
if (this.statusCd == null || this.statusCd.isBlank()) return null;
try {
return TrainStatusType.valueOf(this.statusCd).getText(); // 또는 getName()
return HeaderUtil.isEnglishRequest()
? TrainStatusType.valueOf(this.statusCd).getId()
: TrainStatusType.valueOf(this.statusCd).getText(); // 또는 getName()
} catch (IllegalArgumentException e) {
return this.statusCd; // 매핑 못하면 코드 그대로 반환(원하면 null 처리)
}
@@ -49,18 +54,16 @@ public class ModelTrainDetailDto {
public String getTrainTypeName() {
if (this.trainType == null || this.trainType.isBlank()) return null;
try {
return TrainType.valueOf(this.trainType).getText(); // 또는 getName()
return HeaderUtil.isEnglishRequest()
? TrainType.valueOf(this.trainType).getId()
: TrainType.valueOf(this.trainType).getText(); // 또는 getName()
} catch (IllegalArgumentException e) {
return this.trainType; // 매핑 못하면 코드 그대로 반환(원하면 null 처리)
}
}
private String formatDuration(ZonedDateTime start, ZonedDateTime end) {
if (end == null) {
end = ZonedDateTime.now();
}
if (start == null) {
if (start == null || end == null) {
return null;
}
@@ -70,11 +73,42 @@ public class ModelTrainDetailDto {
long minutes = (totalSeconds % 3600) / 60;
long seconds = totalSeconds % 60;
if (HeaderUtil.isEnglishRequest()) {
return String.format("%dh %dm %ds", hours, minutes, seconds);
} else {
return String.format("%d시간 %d분 %d초", hours, minutes, seconds);
}
}
public String getStepAllDuration() {
return formatDuration(this.step1StrtDttm, this.step2EndDttm);
if (this.step2EndDttm != null) {
// step1 + step2 실제 소요시간 합산
long step1Seconds = 0;
long step2Seconds = 0;
if (this.step1StrtDttm != null && this.step1EndDttm != null) {
step1Seconds =
Math.abs(Duration.between(this.step1StrtDttm, this.step1EndDttm).getSeconds());
}
if (this.step2StrtDttm != null && this.step2EndDttm != null) {
step2Seconds =
Math.abs(Duration.between(this.step2StrtDttm, this.step2EndDttm).getSeconds());
}
long totalSeconds = step1Seconds + step2Seconds;
long hours = totalSeconds / 3600;
long minutes = (totalSeconds % 3600) / 60;
long seconds = totalSeconds % 60;
if (HeaderUtil.isEnglishRequest()) {
return String.format("%dh %dm %ds", hours, minutes, seconds);
} else {
return String.format("%d시간 %d분 %d초", hours, minutes, seconds);
}
} else {
// step2 없으면 step1만
return formatDuration(this.step1StrtDttm, this.step1EndDttm);
}
}
}
@@ -174,7 +208,7 @@ public class ModelTrainDetailDto {
public String getDataTypeName(String groupTitleCd) {
LearnDataType type = Enums.fromId(LearnDataType.class, groupTitleCd);
return type == null ? null : type.getText();
return type == null ? null : (HeaderUtil.isEnglishRequest() ? type.getId() : type.getText());
}
}

View File

@@ -3,6 +3,7 @@ package com.kamco.cd.training.model.dto;
import com.kamco.cd.training.common.dto.HyperParam;
import com.kamco.cd.training.common.enums.TrainStatusType;
import com.kamco.cd.training.common.enums.TrainType;
import com.kamco.cd.training.common.utils.HeaderUtil;
import com.kamco.cd.training.common.utils.interfaces.JsonFormatDttm;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.constraints.NotNull;
@@ -44,16 +45,20 @@ public class ModelTrainMngDto {
private String requestPath;
private String packingState;
private ZonedDateTime packingStrtDttm;
private ZonedDateTime packingEndDttm;
@JsonFormatDttm private ZonedDateTime packingStrtDttm;
@JsonFormatDttm private ZonedDateTime packingEndDttm;
private Long beforeModelId;
private Integer bestEpoch;
private UUID beforeModelUuid;
public String getStatusName() {
if (this.statusCd == null || this.statusCd.isBlank()) return null;
try {
return TrainStatusType.valueOf(this.statusCd).getText(); // 또는 getName()
return (HeaderUtil.isEnglishRequest()
? TrainStatusType.valueOf(this.statusCd).getId()
: TrainStatusType.valueOf(this.statusCd).getText()); // 또는 getName()
} catch (IllegalArgumentException e) {
return this.statusCd; // 매핑 못하면 코드 그대로 반환(원하면 null 처리)
}
@@ -62,7 +67,9 @@ public class ModelTrainMngDto {
public String getStep1StatusName() {
if (this.step1Status == null || this.step1Status.isBlank()) return null;
try {
return TrainStatusType.valueOf(this.step1Status).getText(); // 또는 getName()
return (HeaderUtil.isEnglishRequest()
? TrainStatusType.valueOf(this.step1Status).getId()
: TrainStatusType.valueOf(this.step1Status).getText()); // 또는 getName()
} catch (IllegalArgumentException e) {
return this.step1Status; // 매핑 못하면 코드 그대로 반환(원하면 null 처리)
}
@@ -71,7 +78,9 @@ public class ModelTrainMngDto {
public String getStep2StatusName() {
if (this.step2Status == null || this.step2Status.isBlank()) return null;
try {
return TrainStatusType.valueOf(this.step2Status).getText(); // 또는 getName()
return (HeaderUtil.isEnglishRequest()
? TrainStatusType.valueOf(this.step2Status).getId()
: TrainStatusType.valueOf(this.step2Status).getText()); // 또는 getName()
} catch (IllegalArgumentException e) {
return this.step2Status; // 매핑 못하면 코드 그대로 반환(원하면 null 처리)
}
@@ -80,7 +89,9 @@ public class ModelTrainMngDto {
public String getTrainTypeName() {
if (this.trainType == null || this.trainType.isBlank()) return null;
try {
return TrainType.valueOf(this.trainType).getText(); // 또는 getName()
return (HeaderUtil.isEnglishRequest()
? TrainType.valueOf(this.trainType).getId()
: TrainType.valueOf(this.trainType).getText()); // 또는 getName()
} catch (IllegalArgumentException e) {
return this.trainType; // 매핑 못하면 코드 그대로 반환(원하면 null 처리)
}
@@ -97,8 +108,12 @@ public class ModelTrainMngDto {
long minutes = (totalSeconds % 3600) / 60;
long seconds = totalSeconds % 60;
if (HeaderUtil.isEnglishRequest()) {
return String.format("%dh %dm %ds", hours, minutes, seconds);
} else {
return String.format("%d시간 %d분 %d초", hours, minutes, seconds);
}
}
public String getStep1Duration() {
return formatDuration(this.step1StrtDttm, this.step1EndDttm);
@@ -260,7 +275,9 @@ public class ModelTrainMngDto {
public String getStatusName() {
if (this.statusCd == null || this.statusCd.isBlank()) return null;
try {
return TrainStatusType.valueOf(this.statusCd).getText(); // 또는 getName()
return HeaderUtil.isEnglishRequest()
? TrainStatusType.valueOf(this.statusCd).getId()
: TrainStatusType.valueOf(this.statusCd).getText(); // 또는 getName()
} catch (IllegalArgumentException e) {
return this.statusCd; // 매핑 못하면 코드 그대로 반환(원하면 null 처리)
}
@@ -269,7 +286,9 @@ public class ModelTrainMngDto {
public String getStep1StatusName() {
if (this.step1Status == null || this.step1Status.isBlank()) return null;
try {
return TrainStatusType.valueOf(this.step1Status).getText(); // 또는 getName()
return HeaderUtil.isEnglishRequest()
? TrainStatusType.valueOf(this.step1Status).getId()
: TrainStatusType.valueOf(this.step1Status).getText(); // 또는 getName()
} catch (IllegalArgumentException e) {
return this.step1Status; // 매핑 못하면 코드 그대로 반환(원하면 null 처리)
}
@@ -278,7 +297,9 @@ public class ModelTrainMngDto {
public String getStep2StatusName() {
if (this.step2Status == null || this.step2Status.isBlank()) return null;
try {
return TrainStatusType.valueOf(this.step2Status).getText(); // 또는 getName()
return HeaderUtil.isEnglishRequest()
? TrainStatusType.valueOf(this.step2Status).getId()
: TrainStatusType.valueOf(this.step2Status).getText(); // 또는 getName()
} catch (IllegalArgumentException e) {
return this.step2Status; // 매핑 못하면 코드 그대로 반환(원하면 null 처리)
}
@@ -287,7 +308,9 @@ public class ModelTrainMngDto {
public String getTrainTypeName() {
if (this.trainType == null || this.trainType.isBlank()) return null;
try {
return TrainType.valueOf(this.trainType).getText(); // 또는 getName()
return HeaderUtil.isEnglishRequest()
? TrainType.valueOf(this.trainType).getId()
: TrainType.valueOf(this.trainType).getText(); // 또는 getName()
} catch (IllegalArgumentException e) {
return this.trainType; // 매핑 못하면 코드 그대로 반환(원하면 null 처리)
}
@@ -304,8 +327,12 @@ public class ModelTrainMngDto {
long minutes = (totalSeconds % 3600) / 60;
long seconds = totalSeconds % 60;
if (HeaderUtil.isEnglishRequest()) {
return String.format("%dh %dm %ds", hours, minutes, seconds);
} else {
return String.format("%d시간 %d분 %d초", hours, minutes, seconds);
}
}
public String getStep1Duration() {
return formatDuration(this.step1StrtDttm, this.step1EndDttm);
@@ -353,4 +380,26 @@ public class ModelTrainMngDto {
// 삭제 될 파일
private List<String> deleteTargets;
}
@Getter
@Builder
@AllArgsConstructor
public static class InProgressModel {
private String modelNo;
private UUID uuid;
}
@Getter
@Builder
@AllArgsConstructor
public static class ProgressPercent {
private Long modelId;
private String jobType;
private String statusCd;
private Integer totalEpoch;
private Integer currentEpoch;
private Double percent;
}
}

View File

@@ -350,4 +350,12 @@ public class ModelTrainMngService {
public Long findModelStep1InProgressCnt() {
return modelTrainMngCoreService.findModelStep1InProgressCnt();
}
public ModelTrainMngDto.InProgressModel findInprogressModel() {
return modelTrainMngCoreService.findInprogressModel();
}
public ModelTrainMngDto.ProgressPercent findTrainProgressPercent(UUID uuid) {
return modelTrainMngCoreService.findTrainProgressPercent(uuid);
}
}

View File

@@ -0,0 +1,282 @@
package com.kamco.cd.training.postgres.core;
import com.kamco.cd.training.common.enums.LearnDataRegister;
import com.kamco.cd.training.common.enums.LearnDataType;
import com.kamco.cd.training.common.exception.NotFoundException;
import com.kamco.cd.training.common.service.BaseCoreService;
import com.kamco.cd.training.dataset.dto.DatasetClsObjDto;
import com.kamco.cd.training.dataset.dto.DatasetDto;
import com.kamco.cd.training.dataset.dto.DatasetDto.DatasetMngRegDto;
import com.kamco.cd.training.dataset.dto.DatasetObjDto;
import com.kamco.cd.training.dataset.dto.DatasetObjDto.DatasetClass;
import com.kamco.cd.training.dataset.dto.DatasetObjDto.DatasetObjRegDto;
import com.kamco.cd.training.postgres.entity.DatasetClsObjEntity;
import com.kamco.cd.training.postgres.entity.DatasetEntity;
import com.kamco.cd.training.postgres.entity.DatasetObjEntity;
import com.kamco.cd.training.postgres.repository.dataset.DatasetClsRepository;
import com.kamco.cd.training.postgres.repository.dataset.DatasetObjRepository;
import jakarta.transaction.Transactional;
import java.time.ZonedDateTime;
import java.util.List;
import java.util.UUID;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.data.domain.Page;
import org.springframework.stereotype.Service;
@Service
@RequiredArgsConstructor
@Slf4j
public class DatasetClsCoreService
implements BaseCoreService<DatasetDto.Basic, Long, DatasetDto.SearchReq> {
private final DatasetClsRepository datasetClsRepository;
private final DatasetObjRepository datasetObjRepository;
/**
* 학습 데이터 삭제
*
* @param id 데이터셋 ID
*/
@Override
public void remove(Long id) {
DatasetEntity entity =
datasetClsRepository
.findById(id)
.orElseThrow(() -> new NotFoundException("데이터셋을 찾을 수 없습니다. ID: " + id));
entity.setDeleted(true);
datasetClsRepository.save(entity);
}
public void remove(UUID id) {
DatasetEntity entity =
datasetClsRepository
.findByUuid(id)
.orElseThrow(() -> new NotFoundException("데이터셋을 찾을 수 없습니다. ID: " + id));
entity.setDeleted(true);
datasetClsRepository.save(entity);
}
@Override
public DatasetDto.Basic getOneById(Long id) {
DatasetEntity entity =
datasetClsRepository
.findById(id)
.orElseThrow(() -> new NotFoundException("데이터셋을 찾을 수 없습니다. ID: " + id));
if (entity.getDeleted()) {
throw new NotFoundException("삭제된 데이터셋입니다. ID: " + id);
}
return entity.toDto();
}
public DatasetDto.Basic getOneByUuid(UUID id) {
DatasetEntity entity =
datasetClsRepository
.findByUuid(id)
.orElseThrow(() -> new NotFoundException("데이터셋을 찾을 수 없습니다. uuid: " + id));
return entity.toDto();
}
@Override
public Page<DatasetDto.Basic> search(DatasetDto.SearchReq searchReq) {
Page<DatasetEntity> entityPage = datasetClsRepository.findDatasetList(searchReq);
return entityPage.map(DatasetEntity::toDto);
}
/**
* 학습데이터 조회
*
* @param searchReq 검색 조건
* @return 페이징 처리된 데이터셋 목록
*/
public Page<DatasetDto.Basic> findDatasetList(DatasetDto.SearchReq searchReq) {
return search(searchReq);
}
// TODO 미사용시작
/**
* 학습데이터 등록
*
* @param registerReq 등록 요청 데이터
* @return 등록된 데이터셋 정보
*/
public DatasetDto.Basic save(DatasetDto.RegisterReq registerReq) {
// 먼저 id1 필드를 임시값(0)으로 설정하여 저장
DatasetEntity entity = new DatasetEntity();
entity.setTitle(registerReq.getTitle());
// entity.setYear(registerReq.getYear());
entity.setDataType(LearnDataType.PRODUCTION.getId());
entity.setCompareYyyy(registerReq.getCompareYear());
entity.setTargetYyyy(registerReq.getTargetYyyy());
entity.setRoundNo(registerReq.getRoundNo() != null ? registerReq.getRoundNo() : 1L);
entity.setMemo(registerReq.getMemo());
entity.setStatus(LearnDataRegister.READY.getId());
entity.setDataType("CREATE");
entity.setTotalItems(0L);
entity.setTotalSize(0L);
entity.setItemCount(0L);
entity.setDeleted(false);
entity.setCreatedDttm(ZonedDateTime.now());
// entity.setId1(0L); // 임시값
DatasetEntity savedEntity = datasetClsRepository.save(entity);
// 저장 후 id1을 dataset_uid와 동일하게 업데이트
// savedEntity.setId1(savedEntity.getId());
// savedEntity = datasetClsRepository.save(savedEntity);
return savedEntity.toDto();
}
// TODO 미사용 끝
/**
* 학습 데이터 수정
*
* @param updateReq 수정 요청 데이터
* @return 수정된 데이터셋 정보
*/
public void update(UUID uuid, DatasetDto.UpdateReq updateReq) {
DatasetEntity entity =
datasetClsRepository
.findByUuid(uuid)
.orElseThrow(() -> new NotFoundException("데이터셋을 찾을 수 없습니다. uuid: " + uuid));
if (StringUtils.isNotBlank(updateReq.getTitle())) {
entity.setTitle(updateReq.getTitle());
}
if (StringUtils.isNotBlank(updateReq.getMemo())) {
entity.setMemo(updateReq.getMemo());
}
datasetClsRepository.save(entity);
}
/**
* 학습데이터 삭제
*
* @param uuid 삭제 요청 (데이터셋 ID 목록)
*/
public void deleteDatasets(UUID uuid) {
remove(uuid);
}
public DatasetDto.Summary getDatasetSummary(DatasetDto.SummaryReq summaryReq) {
long totalMapSheets = 0;
long totalFileSize = 0;
for (Long datasetId : summaryReq.getDatasetIds()) {
DatasetEntity entity =
datasetClsRepository
.findById(datasetId)
.orElseThrow(() -> new NotFoundException("데이터셋을 찾을 수 없습니다. ID: " + datasetId));
if (!entity.getDeleted()) {
totalMapSheets += entity.getTotalItems() != null ? entity.getTotalItems() : 0;
totalFileSize += entity.getTotalSize() != null ? entity.getTotalSize() : 0;
}
}
double averageMapSheets =
!summaryReq.getDatasetIds().isEmpty()
? (double) totalMapSheets / summaryReq.getDatasetIds().size()
: 0;
return new DatasetDto.Summary(
summaryReq.getDatasetIds().size(), totalMapSheets, totalFileSize, averageMapSheets);
}
public Page<DatasetClsObjDto.Basic> searchDatasetClsObjectList(
DatasetClsObjDto.ClsSearchReq searchReq) {
Page<DatasetClsObjEntity> entityPage =
datasetClsRepository.searchDatasetClsObjectList(searchReq);
return entityPage.map(DatasetClsObjEntity::toDto);
}
public UUID deleteDatasetObjByUuid(UUID uuid) {
DatasetObjEntity entity =
datasetObjRepository
.findByUuid(uuid)
.orElseThrow(() -> new NotFoundException("데이터셋 obj를 찾을 수 없습니다. ID: " + uuid));
entity.setDeleted(true);
datasetObjRepository.save(entity);
return entity.getUuid();
}
/**
* 데이터셋 object class 조회
*
* @param uuid dataset uuid
* @param type compare, target
* @return
*/
public List<DatasetClass> findDatasetObjClassByUuid(UUID uuid, String type) {
return datasetObjRepository.findDatasetObjClassByUuid(uuid, type);
}
public Long getDatasetMaxStage(int compareYyyy, int targetYyyy) {
return datasetClsRepository.getDatasetMaxStage(compareYyyy, targetYyyy);
}
@Transactional
public Long insertDatasetMngData(DatasetMngRegDto mngRegDto) {
return datasetClsRepository.insertDatasetMngData(mngRegDto);
}
public void insertDatasetObj(DatasetObjRegDto objRegDto) {
datasetObjRepository.insertDatasetObj(objRegDto);
}
public String getFilePathByUUIDPathType(UUID uuid, String pathType) {
return datasetObjRepository.getFilePathByUUIDPathType(uuid, pathType);
}
public void insertDatasetTestObj(DatasetObjRegDto objRegDto) {
datasetObjRepository.insertDatasetTestObj(objRegDto);
}
/**
* 학습데이터셋 마스터 상태 변경
*
* @param datasetUid 학습데이터셋 마스터 id
* @param register 상태
*/
@Transactional
public void updateDatasetUploadStatus(Long datasetUid, LearnDataRegister register) {
DatasetEntity entity =
datasetClsRepository
.findById(datasetUid)
.orElseThrow(() -> new NotFoundException("데이터셋을 찾을 수 없습니다. ID: " + datasetUid));
entity.setStatus(register.getId());
}
public void insertDatasetValObj(DatasetObjRegDto objRegDto) {
datasetObjRepository.insertDatasetValObj(objRegDto);
}
public Long findDatasetByUidExistsCnt(String uid) {
return datasetClsRepository.findDatasetByUidExistsCnt(uid);
}
/**
* 데이터셋 등록 실패시 Obj 데이터 정리
*
* @param datasetUid 모델 마스터 id
*/
@Transactional
public void deleteAllDatasetObj(Long datasetUid) {
int cnt = datasetObjRepository.deleteAllDatasetObj(datasetUid);
log.info("datasetUid={} 데이터셋 실패 - 전체 삭제 완료. 총 {}건", datasetUid, cnt);
}
public void insertDatasetClsObj(DatasetObjDto.DatasetClsObjRegDto objRegDto) {
datasetClsRepository.insertDatasetClsObj(objRegDto);
}
public String getFilePathByUUIDPathTif(UUID uuid) {
return datasetClsRepository.getFilePathByUUIDPathType(uuid);
}
}

View File

@@ -120,6 +120,15 @@ public class ModelTrainMngCoreService {
entity.setTrainType(addReq.getTrainType()); // 일반, 전이
entity.setBeforeModelId(addReq.getBeforeModelId());
// 전이학습일 때 beforeModelUuid 추가하기
if (addReq.getBeforeModelId() != null) {
ModelMasterEntity beforeModel =
modelMngRepository
.findById(addReq.getBeforeModelId())
.orElseThrow(() -> new CustomApiException("NOT_FOUND_DATA", HttpStatus.NOT_FOUND));
entity.setBeforeModelUuid(beforeModel.getUuid());
}
entity.setStatusCd(TrainStatusType.READY.getId());
entity.setStep1State(TrainStatusType.READY.getId());
@@ -713,4 +722,17 @@ public class ModelTrainMngCoreService {
entity.setTmpFileStatus("FAIL");
entity.setTmpFileErrMessage(message);
}
public ModelTrainMngDto.InProgressModel findInprogressModel() {
return modelMngRepository.findInprogressModel();
}
public ModelTrainMngDto.ProgressPercent findTrainProgressPercent(UUID uuid) {
ModelMasterEntity entity =
modelMngRepository
.findByUuid(uuid)
.orElseThrow(() -> new CustomApiException("NOT_FOUND_DATA", HttpStatus.NOT_FOUND));
return modelMngRepository.findTrainProgressPercent(entity.getId());
}
}

View File

@@ -0,0 +1,77 @@
package com.kamco.cd.training.postgres.entity;
import com.kamco.cd.training.dataset.dto.DatasetClsObjDto.Basic;
import jakarta.persistence.*;
import jakarta.validation.constraints.NotNull;
import jakarta.validation.constraints.Size;
import java.time.ZonedDateTime;
import java.util.UUID;
import lombok.Getter;
import lombok.Setter;
import org.hibernate.annotations.ColumnDefault;
@Getter
@Setter
@Entity
@Table(name = "tb_dataset_cls_obj")
public class DatasetClsObjEntity {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
@Column(name = "obj_id", nullable = false)
private Long objId;
@Column(name = "uuid")
private UUID uuid;
@NotNull
@Column(name = "dataset_uid", nullable = false)
private Long datasetUid;
@Column(name = "type")
private String type;
@Column(name = "yyyy")
private Integer yyyy;
@Size(max = 255)
@Column(name = "class_cd")
private String classCd;
@Size(max = 255)
@Column(name = "map_sheet_num")
private String mapSheetNum;
@Column(name = "tif_img_path")
private String tifImgPath;
@Column(name = "file_name")
private String fileName;
@ColumnDefault("now()")
@Column(name = "created_dttm")
private ZonedDateTime createdDttm;
@Column(name = "created_uid")
private Long createdUid;
@ColumnDefault("false")
@Column(name = "deleted")
private Boolean deleted;
public Basic toDto() {
return new Basic(
this.objId,
this.uuid,
this.datasetUid,
this.type,
this.yyyy,
this.classCd,
this.mapSheetNum,
this.tifImgPath,
this.fileName,
this.createdDttm,
this.createdUid,
this.deleted);
}
}

View File

@@ -127,6 +127,9 @@ public class DatasetEntity {
@Column(name = "uid")
private String uid;
@Column(name = "cd_cls_type")
private String cdClsType;
public DatasetDto.Basic toDto() {
return new DatasetDto.Basic(
this.id,
@@ -140,6 +143,7 @@ public class DatasetEntity {
this.createdDttm,
this.status,
this.deleted,
this.dataType);
this.dataType,
this.cdClsType);
}
}

View File

@@ -136,6 +136,9 @@ public class ModelMasterEntity {
@Column(name = "tmp_file_err_message", columnDefinition = "TEXT")
private String tmpFileErrMessage;
@Column(name = "before_model_uuid")
private UUID beforeModelUuid;
public ModelTrainMngDto.Basic toDto() {
return new ModelTrainMngDto.Basic(
this.id,
@@ -157,6 +160,7 @@ public class ModelMasterEntity {
this.packingStrtDttm,
this.packingEndDttm,
this.beforeModelId,
this.bestEpoch);
this.bestEpoch,
this.beforeModelUuid);
}
}

View File

@@ -0,0 +1,7 @@
package com.kamco.cd.training.postgres.repository.dataset;
import com.kamco.cd.training.postgres.entity.DatasetEntity;
import org.springframework.data.jpa.repository.JpaRepository;
public interface DatasetClsRepository
extends JpaRepository<DatasetEntity, Long>, DatasetClsRepositoryCustom {}

View File

@@ -0,0 +1,48 @@
package com.kamco.cd.training.postgres.repository.dataset;
import com.kamco.cd.training.dataset.dto.DatasetClsObjDto;
import com.kamco.cd.training.dataset.dto.DatasetDto;
import com.kamco.cd.training.dataset.dto.DatasetDto.DatasetMngRegDto;
import com.kamco.cd.training.dataset.dto.DatasetDto.DatasetReq;
import com.kamco.cd.training.dataset.dto.DatasetDto.SelectDataSet;
import com.kamco.cd.training.dataset.dto.DatasetDto.SelectTransferDataSet;
import com.kamco.cd.training.dataset.dto.DatasetObjDto;
import com.kamco.cd.training.postgres.entity.DatasetClsObjEntity;
import com.kamco.cd.training.postgres.entity.DatasetEntity;
import java.util.List;
import java.util.Optional;
import java.util.UUID;
import org.springframework.data.domain.Page;
public interface DatasetClsRepositoryCustom {
Page<DatasetEntity> findDatasetList(DatasetDto.SearchReq searchReq);
Optional<DatasetEntity> findByUuid(UUID id);
List<SelectDataSet> getDatasetSelectG1List(DatasetReq req);
// TODO 미사용시작
public List<SelectTransferDataSet> getDatasetTransferSelectG1List(Long modelId);
public List<SelectTransferDataSet> getDatasetTransferSelectG2G3List(Long modelId, String modelNo);
// TODO 미사용 끝
List<SelectDataSet> getDatasetSelectG2G3List(DatasetReq req);
List<SelectDataSet> getDatasetSelectG4List(DatasetReq req);
Long getDatasetMaxStage(int compareYyyy, int targetYyyy);
Long insertDatasetMngData(DatasetMngRegDto mngRegDto);
List<String> findDatasetUid(List<Long> datasetIds);
Long findDatasetByUidExistsCnt(String uid);
Page<DatasetClsObjEntity> searchDatasetClsObjectList(DatasetClsObjDto.ClsSearchReq searchReq);
void insertDatasetClsObj(DatasetObjDto.DatasetClsObjRegDto objRegDto);
String getFilePathByUUIDPathType(UUID uuid);
}

View File

@@ -0,0 +1,668 @@
package com.kamco.cd.training.postgres.repository.dataset;
import static com.kamco.cd.training.postgres.entity.QDatasetClsObjEntity.datasetClsObjEntity;
import static com.kamco.cd.training.postgres.entity.QDatasetEntity.datasetEntity;
import static com.kamco.cd.training.postgres.entity.QDatasetObjEntity.datasetObjEntity;
import static com.kamco.cd.training.postgres.entity.QModelDatasetMappEntity.modelDatasetMappEntity;
import static com.kamco.cd.training.postgres.entity.QModelMasterEntity.modelMasterEntity;
import com.kamco.cd.training.common.enums.DetectionClassification;
import com.kamco.cd.training.common.enums.ModelType;
import com.kamco.cd.training.dataset.dto.DatasetClsObjDto;
import com.kamco.cd.training.dataset.dto.DatasetDto.*;
import com.kamco.cd.training.dataset.dto.DatasetObjDto;
import com.kamco.cd.training.postgres.entity.*;
import com.querydsl.core.BooleanBuilder;
import com.querydsl.core.types.Projections;
import com.querydsl.core.types.dsl.CaseBuilder;
import com.querydsl.core.types.dsl.Expressions;
import com.querydsl.core.types.dsl.NumberExpression;
import com.querydsl.jpa.impl.JPAQueryFactory;
import jakarta.persistence.EntityManager;
import jakarta.persistence.EntityNotFoundException;
import jakarta.persistence.PersistenceContext;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.UUID;
import lombok.RequiredArgsConstructor;
import org.apache.commons.lang3.StringUtils;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageImpl;
import org.springframework.data.domain.Pageable;
import org.springframework.stereotype.Repository;
@Repository
@RequiredArgsConstructor
public class DatasetClsRepositoryImpl implements DatasetClsRepositoryCustom {
private final JPAQueryFactory queryFactory;
private final QDatasetEntity dataset = QDatasetEntity.datasetEntity;
@PersistenceContext private final EntityManager em;
/**
* 데이터셋 목록 조회
*
* @param searchReq 검색 조건
* @return 페이징 처리된 데이터셋 Entity 목록
*/
@Override
public Page<DatasetEntity> findDatasetList(SearchReq searchReq) {
Pageable pageable = searchReq.toPageable();
BooleanBuilder builder = new BooleanBuilder();
// 제목
if (StringUtils.isNotBlank(searchReq.getTitle())) {
String contains = "%" + searchReq.getTitle() + "%";
builder.and(dataset.title.likeIgnoreCase(contains));
}
// 구분
if (StringUtils.isNotBlank(searchReq.getDataType())) {
builder.and(dataset.dataType.eq(searchReq.getDataType()));
}
// Entity 직접 조회 (Projections 사용 지양)
List<DatasetEntity> content =
queryFactory
.selectFrom(dataset)
.where(builder.and(dataset.deleted.isFalse()).and(dataset.cdClsType.eq("CLS")))
.offset(pageable.getOffset())
.limit(pageable.getPageSize())
.orderBy(dataset.createdDttm.desc())
.fetch();
// Count 쿼리 별도 실행 (null safe handling)
long total =
Optional.ofNullable(
queryFactory
.select(dataset.count())
.from(dataset)
.where(builder.and(dataset.deleted.isFalse()).and(dataset.cdClsType.eq("CLS")))
.fetchOne())
.orElse(0L);
return new PageImpl<>(content, pageable, total);
}
@Override
public Optional<DatasetEntity> findByUuid(UUID id) {
return Optional.ofNullable(
queryFactory
.select(dataset)
.from(dataset)
.where(dataset.uuid.eq(id), dataset.deleted.isFalse())
.fetchOne());
}
@Override
public List<SelectDataSet> getDatasetSelectG1List(DatasetReq req) {
BooleanBuilder builder = new BooleanBuilder();
builder.and(dataset.deleted.isFalse());
if (StringUtils.isNotBlank(req.getDataType()) && !"CURRENT".equals(req.getDataType())) {
builder.and(dataset.dataType.eq(req.getDataType()));
}
if (req.getIds() != null) {
builder.and(dataset.id.in(req.getIds()));
}
return queryFactory
.select(
Projections.constructor(
SelectDataSet.class,
Expressions.constant(req.getModelNo()),
dataset.id,
dataset.uuid,
dataset.dataType,
dataset.title,
dataset.roundNo,
dataset.compareYyyy,
dataset.targetYyyy,
dataset.memo,
new CaseBuilder()
.when(
datasetObjEntity.targetClassCd.eq(DetectionClassification.BUILDING.getId()))
.then(1L)
.otherwise(0L)
.sum(),
new CaseBuilder()
.when(
datasetObjEntity.targetClassCd.eq(
DetectionClassification.CONTAINER.getId()))
.then(1L)
.otherwise(0L)
.sum()))
.from(dataset)
.leftJoin(datasetObjEntity)
.on(dataset.id.eq(datasetObjEntity.datasetUid))
.where(builder) // datasetObjEntity.targetClassCd.in("building", "container").and(
.groupBy(
dataset.id,
dataset.uuid,
dataset.dataType,
dataset.title,
dataset.roundNo,
dataset.memo)
.orderBy(dataset.createdDttm.desc())
.fetch();
}
// TODO 미사용시작
@Override
public List<SelectTransferDataSet> getDatasetTransferSelectG1List(Long modelId) {
QModelMasterEntity beforeMaster = new QModelMasterEntity("beforeMaster");
QModelDatasetMappEntity beforeMapp = new QModelDatasetMappEntity("beforeMapp");
QDatasetEntity beforeDataset = new QDatasetEntity("beforeDataset");
QDatasetObjEntity beforeObj = new QDatasetObjEntity("beforeObj");
return queryFactory
.select(
Projections.constructor(
SelectTransferDataSet.class,
// ===== 현재 =====
modelMasterEntity.modelNo,
dataset.id,
dataset.uuid,
dataset.dataType,
dataset.title,
dataset.roundNo,
dataset.compareYyyy,
dataset.targetYyyy,
dataset.memo,
new CaseBuilder()
.when(datasetObjEntity.targetClassCd.eq("building"))
.then(1)
.otherwise(0)
.sum(),
new CaseBuilder()
.when(datasetObjEntity.targetClassCd.eq("container"))
.then(1)
.otherwise(0)
.sum(),
// ===== before (join으로) =====
beforeMaster.modelNo,
beforeDataset.id,
beforeDataset.uuid,
beforeDataset.dataType,
beforeDataset.title,
beforeDataset.roundNo,
beforeDataset.compareYyyy,
beforeDataset.targetYyyy,
beforeDataset.memo,
new CaseBuilder()
.when(beforeObj.targetClassCd.eq("building"))
.then(1)
.otherwise(0)
.sum(),
new CaseBuilder()
.when(beforeObj.targetClassCd.eq("container"))
.then(1)
.otherwise(0)
.sum()))
.from(modelMasterEntity)
// ===== 현재 dataset join =====
.leftJoin(modelDatasetMappEntity)
.on(modelDatasetMappEntity.modelUid.eq(modelMasterEntity.id))
.leftJoin(dataset)
.on(modelDatasetMappEntity.datasetUid.eq(dataset.id))
.leftJoin(datasetObjEntity)
.on(dataset.id.eq(datasetObjEntity.datasetUid))
// ===== before 모델 join =====
.leftJoin(beforeMaster)
.on(beforeMaster.id.eq(modelMasterEntity.beforeModelId))
.leftJoin(beforeMapp)
.on(beforeMapp.modelUid.eq(beforeMaster.id))
.leftJoin(beforeDataset)
.on(beforeMapp.datasetUid.eq(beforeDataset.id))
.leftJoin(beforeObj)
.on(beforeDataset.id.eq(beforeObj.datasetUid))
.where(modelMasterEntity.id.eq(modelId))
.groupBy(
modelMasterEntity.modelNo,
dataset.id,
dataset.uuid,
dataset.dataType,
dataset.title,
dataset.roundNo,
dataset.compareYyyy,
dataset.targetYyyy,
dataset.memo,
beforeMaster.modelNo,
beforeDataset.id,
beforeDataset.uuid,
beforeDataset.dataType,
beforeDataset.title,
beforeDataset.roundNo,
beforeDataset.compareYyyy,
beforeDataset.targetYyyy,
beforeDataset.memo)
.orderBy(dataset.createdDttm.desc())
.fetch();
}
// TODO 미사용 끝
@Override
public List<SelectDataSet> getDatasetSelectG2G3List(DatasetReq req) {
String building = DetectionClassification.BUILDING.getId();
String container = DetectionClassification.CONTAINER.getId();
String waste = DetectionClassification.WASTE.getId();
String solar = DetectionClassification.SOLAR.getId();
BooleanBuilder builder = new BooleanBuilder();
builder.and(dataset.deleted.isFalse());
NumberExpression<Long> selectedCnt = null;
// G2
NumberExpression<Long> wasteCnt =
datasetObjEntity
.targetClassCd
.when(DetectionClassification.WASTE.getId())
.then(1L)
.otherwise(0L)
.sum();
// G3 (G1, G2, G4 제외)
NumberExpression<Long> elseCnt =
new CaseBuilder()
.when(datasetObjEntity.targetClassCd.notIn(building, container, waste, solar))
.then(1L)
.otherwise(0L)
.sum();
if (req.getModelNo().equals(ModelType.G2.getId())) {
selectedCnt = wasteCnt;
} else {
selectedCnt = elseCnt;
}
if (StringUtils.isNotBlank(req.getDataType())) {
if (!"CURRENT".equals(req.getDataType())) {
builder.and(dataset.dataType.eq(req.getDataType()));
}
}
if (req.getIds() != null) {
builder.and(dataset.id.in(req.getIds()));
}
return queryFactory
.select(
Projections.constructor(
SelectDataSet.class,
Expressions.constant(req.getModelNo()),
dataset.id,
dataset.uuid,
dataset.dataType,
dataset.title,
dataset.roundNo,
dataset.compareYyyy,
dataset.targetYyyy,
dataset.memo,
selectedCnt.as("cnt")))
.from(dataset)
.leftJoin(datasetObjEntity)
.on(dataset.id.eq(datasetObjEntity.datasetUid))
.where(builder)
.groupBy(
dataset.id,
dataset.uuid,
dataset.dataType,
dataset.title,
dataset.roundNo,
dataset.memo)
.orderBy(dataset.createdDttm.desc())
.fetch();
}
// TODO 미사용시작
@Override
public List<SelectTransferDataSet> getDatasetTransferSelectG2G3List(
Long modelId, String modelNo) {
// before join용
QModelMasterEntity beforeMaster = new QModelMasterEntity("beforeMaster");
QModelDatasetMappEntity beforeMapp = new QModelDatasetMappEntity("beforeMapp");
QDatasetEntity beforeDataset = new QDatasetEntity("beforeDataset");
QDatasetObjEntity beforeObj = new QDatasetObjEntity("beforeObj");
BooleanBuilder builder = new BooleanBuilder();
NumberExpression<Long> wasteCnt =
datasetObjEntity.targetClassCd.when("waste").then(1L).otherwise(0L).sum();
NumberExpression<Long> elseCnt =
new CaseBuilder()
.when(datasetObjEntity.targetClassCd.notIn("building", "container", "waste"))
.then(1L)
.otherwise(0L)
.sum();
NumberExpression<Long> selectedCnt = ModelType.G2.getId().equals(modelNo) ? wasteCnt : elseCnt;
// before도 동일 로직으로 cnt 계산
NumberExpression<Long> beforeWasteCnt =
beforeObj.targetClassCd.when("waste").then(1L).otherwise(0L).sum();
NumberExpression<Long> beforeElseCnt =
new CaseBuilder()
.when(beforeObj.targetClassCd.notIn("building", "container", "waste"))
.then(1L)
.otherwise(0L)
.sum();
NumberExpression<Long> beforeSelectedCnt =
ModelType.G2.getId().equals(modelNo) ? beforeWasteCnt : beforeElseCnt;
return queryFactory
.select(
Projections.constructor(
SelectTransferDataSet.class,
// ===== 현재 =====
modelMasterEntity.modelNo, // modelNo 파라미터 사용 (req.getModelNo() 제거)
dataset.id,
dataset.uuid,
dataset.dataType,
dataset.title,
dataset.roundNo,
dataset.compareYyyy,
dataset.targetYyyy,
dataset.memo,
selectedCnt, // classCount 자리에 들어가는 cnt (Long)
// ===== before =====
beforeMaster.modelNo,
beforeDataset.id,
beforeDataset.uuid,
beforeDataset.dataType,
beforeDataset.title,
beforeDataset.roundNo,
beforeDataset.compareYyyy,
beforeDataset.targetYyyy,
beforeDataset.memo,
beforeSelectedCnt))
.from(modelMasterEntity)
// ===== 현재 dataset =====
.leftJoin(modelDatasetMappEntity)
.on(modelDatasetMappEntity.modelUid.eq(modelMasterEntity.id))
.leftJoin(dataset)
.on(modelDatasetMappEntity.datasetUid.eq(dataset.id))
.leftJoin(datasetObjEntity)
.on(dataset.id.eq(datasetObjEntity.datasetUid))
// ===== before dataset =====
.leftJoin(beforeMaster)
.on(beforeMaster.id.eq(modelMasterEntity.beforeModelId))
.leftJoin(beforeMapp)
.on(beforeMapp.modelUid.eq(beforeMaster.id))
.leftJoin(beforeDataset)
.on(beforeMapp.datasetUid.eq(beforeDataset.id))
.leftJoin(beforeObj)
.on(beforeDataset.id.eq(beforeObj.datasetUid))
.where(modelMasterEntity.id.eq(modelId).and(builder))
// sum() 때문에 groupBy 필요
.groupBy(
dataset.id,
dataset.uuid,
dataset.dataType,
dataset.title,
dataset.roundNo,
dataset.compareYyyy,
dataset.targetYyyy,
dataset.memo,
beforeMaster.modelNo,
beforeDataset.id,
beforeDataset.uuid,
beforeDataset.dataType,
beforeDataset.title,
beforeDataset.roundNo,
beforeDataset.compareYyyy,
beforeDataset.targetYyyy,
beforeDataset.memo)
.orderBy(dataset.createdDttm.desc())
.fetch();
}
// TODO 미사용 끝
@Override
public Long getDatasetMaxStage(int compareYyyy, int targetYyyy) {
return queryFactory
.select(dataset.roundNo.max().coalesce(0L))
.from(dataset)
.where(dataset.compareYyyy.eq(compareYyyy), dataset.targetYyyy.eq(targetYyyy))
.fetchOne();
}
@Override
public Long insertDatasetMngData(DatasetMngRegDto mngRegDto) {
UUID uuid = UUID.randomUUID();
// 1. 기본 생성자로 빈 엔티티 객체 생성
// DatasetEntity datasetEntity = new DatasetEntity();
//
// // 2. Setter를 이용해 데이터 채우기
// datasetEntity.setUid(mngRegDto.getUid());
// datasetEntity.setDataType(mngRegDto.getDataType());
// datasetEntity.setCompareYyyy(mngRegDto.getCompareYyyy());
// datasetEntity.setTargetYyyy(mngRegDto.getTargetYyyy());
// datasetEntity.setRoundNo(mngRegDto.getRoundNo());
// datasetEntity.setTotalSize(mngRegDto.getTotalSize());
// datasetEntity.setTitle(mngRegDto.getTitle());
// datasetEntity.setMemo(mngRegDto.getMemo());
// datasetEntity.setDatasetPath(mngRegDto.getDatasetPath());
// datasetEntity.setCdClsType(mngRegDto.getCdClsType());
//
// datasetEntity.setUuid(UUID.randomUUID());
//
//
// // 2. JPA 영속성 컨텍스트에 저장 (이 시점에 DB에 들어가거나 ID가 생성됨)
// em.persist(datasetEntity);
queryFactory
.insert(dataset)
.columns(
dataset.uid,
dataset.uuid,
dataset.dataType,
dataset.compareYyyy,
dataset.targetYyyy,
dataset.roundNo,
dataset.totalSize,
dataset.title,
dataset.memo,
dataset.datasetPath,
dataset.cdClsType)
.values(
mngRegDto.getUid(),
uuid,
mngRegDto.getDataType(),
mngRegDto.getCompareYyyy(),
mngRegDto.getTargetYyyy(),
mngRegDto.getRoundNo(),
mngRegDto.getTotalSize(),
mngRegDto.getTitle(),
mngRegDto.getMemo(),
mngRegDto.getDatasetPath(),
mngRegDto.getCdClsType())
.execute();
// 3. uid 조회 필요 없이, 생성된 Key를 안전하게 바로 리턴
// return datasetEntity.getId();
return queryFactory.select(dataset.id).from(dataset).where(dataset.uuid.eq(uuid)).fetchOne();
}
@Override
public List<String> findDatasetUid(List<Long> datasetIds) {
return queryFactory.select(dataset.uid).from(dataset).where(dataset.id.in(datasetIds)).fetch();
}
@Override
public Long findDatasetByUidExistsCnt(String uid) {
return queryFactory
.select(dataset.id.count())
.from(dataset)
.where(dataset.uid.eq(uid), dataset.deleted.isFalse())
.fetchOne();
}
@Override
public List<SelectDataSet> getDatasetSelectG4List(DatasetReq req) {
BooleanBuilder builder = new BooleanBuilder();
builder.and(dataset.deleted.isFalse());
if (StringUtils.isNotBlank(req.getDataType()) && !"CURRENT".equals(req.getDataType())) {
builder.and(dataset.dataType.eq(req.getDataType()));
}
if (req.getIds() != null) {
builder.and(dataset.id.in(req.getIds()));
}
return queryFactory
.select(
Projections.constructor(
SelectDataSet.class,
Expressions.constant(req.getModelNo()),
dataset.id,
dataset.uuid,
dataset.dataType,
dataset.title,
dataset.roundNo,
dataset.compareYyyy,
dataset.targetYyyy,
dataset.memo,
new CaseBuilder()
.when(datasetObjEntity.targetClassCd.eq(DetectionClassification.SOLAR.getId()))
.then(1L)
.otherwise(0L)
.sum()))
.from(dataset)
.leftJoin(datasetObjEntity)
.on(dataset.id.eq(datasetObjEntity.datasetUid))
.where(builder)
.groupBy(
dataset.id,
dataset.uuid,
dataset.dataType,
dataset.title,
dataset.roundNo,
dataset.memo)
.orderBy(dataset.createdDttm.desc())
.fetch();
}
@Override
public Page<DatasetClsObjEntity> searchDatasetClsObjectList(
DatasetClsObjDto.ClsSearchReq searchReq) {
{
Pageable pageable = searchReq.toPageable();
BooleanBuilder builder = new BooleanBuilder();
if (StringUtils.isNotBlank(searchReq.getClassCd())) {
builder.and(datasetClsObjEntity.classCd.eq(searchReq.getClassCd()));
}
if (searchReq.getYyyy() != null) {
builder.and(datasetClsObjEntity.yyyy.eq(searchReq.getYyyy()));
}
if (StringUtils.isNotBlank(searchReq.getMapSheetNum())) {
builder.and(datasetClsObjEntity.mapSheetNum.eq(searchReq.getMapSheetNum()));
}
DatasetEntity entity =
queryFactory
.selectFrom(datasetEntity)
.where(datasetEntity.uuid.eq(searchReq.getUuid()))
.fetchOne();
if (Objects.isNull(entity)) {
throw new EntityNotFoundException(
"DatasetEntity not found for uuid: " + searchReq.getUuid());
}
List<DatasetClsObjEntity> content =
queryFactory
.selectFrom(datasetClsObjEntity)
.where(
datasetClsObjEntity
.deleted
.isFalse()
.and(datasetClsObjEntity.datasetUid.eq(entity.getId()))
.and(builder)
.and(
datasetClsObjEntity.type.eq(DatasetClsObjDto.FolderType.TRAIN.getText())))
.offset(pageable.getOffset())
.limit(pageable.getPageSize())
.fetch();
// Count 쿼리 별도 실행 (null safe handling)
long total =
Optional.ofNullable(
queryFactory
.select(datasetClsObjEntity.count())
.from(datasetClsObjEntity)
.where(
datasetClsObjEntity
.deleted
.isFalse()
.and(datasetClsObjEntity.datasetUid.eq(entity.getId()))
.and(builder)
.and(
datasetClsObjEntity.type.eq(
DatasetClsObjDto.FolderType.TRAIN.getText())))
.fetchOne())
.orElse(0L);
return new PageImpl<>(content, pageable, total);
}
}
@Override
public void insertDatasetClsObj(DatasetObjDto.DatasetClsObjRegDto objRegDto) {
try {
em.createNativeQuery(
"""
insert into tb_dataset_cls_obj
(uuid, dataset_uid, type, yyyy, class_cd,
map_sheet_num, tif_img_path, file_name)
values
(?, ?, ?, ?, ?, ?, ?, ?)
""")
.setParameter(1, UUID.randomUUID())
.setParameter(2, objRegDto.getDatasetUid())
.setParameter(3, objRegDto.getType())
.setParameter(4, objRegDto.getYyyy())
.setParameter(5, objRegDto.getClassCd())
.setParameter(6, objRegDto.getMapSheetNum())
.setParameter(7, objRegDto.getTifImgPath())
.setParameter(8, objRegDto.getFileName())
.executeUpdate();
} catch (Exception e) {
throw new RuntimeException(e);
}
}
@Override
public String getFilePathByUUIDPathType(UUID uuid) {
return queryFactory
.select(datasetClsObjEntity.tifImgPath)
.from(datasetClsObjEntity)
.where(datasetClsObjEntity.uuid.eq(uuid))
.fetchOne();
}
}

View File

@@ -65,7 +65,7 @@ public class DatasetRepositoryImpl implements DatasetRepositoryCustom {
List<DatasetEntity> content =
queryFactory
.selectFrom(dataset)
.where(builder.and(dataset.deleted.isFalse()))
.where(builder.and(dataset.deleted.isFalse()).and(dataset.cdClsType.isNull()))
.offset(pageable.getOffset())
.limit(pageable.getPageSize())
.orderBy(dataset.createdDttm.desc())
@@ -77,7 +77,7 @@ public class DatasetRepositoryImpl implements DatasetRepositoryCustom {
queryFactory
.select(dataset.count())
.from(dataset)
.where(builder.and(dataset.deleted.isFalse()))
.where(builder.and(dataset.deleted.isFalse()).and(dataset.cdClsType.isNull()))
.fetchOne())
.orElse(0L);

View File

@@ -80,6 +80,8 @@ public class ModelDetailRepositoryImpl implements ModelDetailRepositoryCustom {
modelMasterEntity.modelNo,
modelMasterEntity.modelVer,
modelMasterEntity.step1StrtDttm,
modelMasterEntity.step1EndDttm,
modelMasterEntity.step2StrtDttm,
modelMasterEntity.step2EndDttm,
modelMasterEntity.statusCd,
modelMasterEntity.trainType,

View File

@@ -47,4 +47,13 @@ public interface ModelMngRepositoryCustom {
* @return 모델 목록
*/
List<ModelMasterEntity> findByHyperParamId(Long hyperParamId);
/**
* 학습 진행중인 모델 type, uuid 조회
*
* @return
*/
ModelTrainMngDto.InProgressModel findInprogressModel();
ModelTrainMngDto.ProgressPercent findTrainProgressPercent(Long id);
}

View File

@@ -4,6 +4,7 @@ import static com.kamco.cd.training.postgres.entity.QMemberEntity.memberEntity;
import static com.kamco.cd.training.postgres.entity.QModelConfigEntity.modelConfigEntity;
import static com.kamco.cd.training.postgres.entity.QModelHyperParamEntity.modelHyperParamEntity;
import static com.kamco.cd.training.postgres.entity.QModelMasterEntity.modelMasterEntity;
import static com.kamco.cd.training.postgres.entity.QModelTrainJobEntity.modelTrainJobEntity;
import com.kamco.cd.training.common.enums.TrainStatusType;
import com.kamco.cd.training.model.dto.ModelTrainMngDto;
@@ -14,7 +15,9 @@ import com.kamco.cd.training.train.dto.TrainRunRequest;
import com.querydsl.core.BooleanBuilder;
import com.querydsl.core.types.Expression;
import com.querydsl.core.types.Projections;
import com.querydsl.core.types.dsl.CaseBuilder;
import com.querydsl.core.types.dsl.Expressions;
import com.querydsl.core.types.dsl.NumberExpression;
import com.querydsl.jpa.impl.JPAQueryFactory;
import java.util.List;
import java.util.Optional;
@@ -143,6 +146,8 @@ public class ModelMngRepositoryImpl implements ModelMngRepositoryCustom {
@Override
public TrainRunRequest findTrainRunRequest(Long modelId) {
QModelMasterEntity beforeModel = new QModelMasterEntity("beforeModel");
return queryFactory
.select(
Projections.constructor(
@@ -190,12 +195,17 @@ public class ModelMngRepositoryImpl implements ModelMngRepositoryCustom {
Expressions.nullExpression(Integer.class),
Expressions.nullExpression(String.class),
modelHyperParamEntity.uuid,
modelMasterEntity.reqTmpYn))
modelMasterEntity.reqTmpYn,
modelMasterEntity.trainType,
beforeModel.uuid,
beforeModel.bestEpoch))
.from(modelMasterEntity)
.leftJoin(modelHyperParamEntity)
.on(modelHyperParamEntity.id.eq(modelMasterEntity.hyperParamId))
.leftJoin(modelConfigEntity)
.on(modelConfigEntity.model.id.eq(modelMasterEntity.id))
.leftJoin(beforeModel)
.on(modelMasterEntity.beforeModelId.eq(beforeModel.id))
.where(modelMasterEntity.id.eq(modelId))
.fetchOne();
}
@@ -231,4 +241,58 @@ public class ModelMngRepositoryImpl implements ModelMngRepositoryCustom {
.orderBy(modelMasterEntity.createdDttm.desc())
.fetch();
}
@Override
public ModelTrainMngDto.InProgressModel findInprogressModel() {
return queryFactory
.select(
Projections.constructor(
ModelTrainMngDto.InProgressModel.class,
modelMasterEntity.modelNo,
modelMasterEntity.uuid))
.from(modelMasterEntity)
.where(
modelMasterEntity
.step1State
.eq(TrainStatusType.IN_PROGRESS.getId())
.or(modelMasterEntity.step2State.eq(TrainStatusType.IN_PROGRESS.getId())))
.fetchOne();
}
@Override
public ModelTrainMngDto.ProgressPercent findTrainProgressPercent(Long id) {
NumberExpression<Integer> currentEpoch =
new CaseBuilder()
.when(
modelTrainJobEntity
.jobType
.eq("TEST")
.and(modelTrainJobEntity.statusCd.eq("SUCCESS")))
.then(1)
.otherwise(modelTrainJobEntity.currentEpoch.coalesce(0));
NumberExpression<Integer> totalEpoch = modelTrainJobEntity.totalEpoch.coalesce(1);
// per 계산
NumberExpression<Double> per =
currentEpoch
.castToNum(Double.class)
.divide(totalEpoch.castToNum(Double.class))
.multiply(100);
return queryFactory
.select(
Projections.constructor(
ModelTrainMngDto.ProgressPercent.class,
modelTrainJobEntity.modelId,
modelTrainJobEntity.jobType,
modelTrainJobEntity.statusCd,
totalEpoch.as("totalEpoch"),
currentEpoch.as("currentEpoch"),
per.as("per")))
.from(modelTrainJobEntity)
.where(modelTrainJobEntity.modelId.eq(id))
.orderBy(modelTrainJobEntity.attemptNo.desc())
.fetchFirst();
}
}

View File

@@ -86,6 +86,12 @@ public class TrainRunRequest {
private Boolean reqTmpYn; // tmp 심볼릭 링크를 쓰는지 아닌지 여부
private String trainType; // 일반 : GENERAL, 전이 : TRANSFER
private UUID beforeModelUUID;
private Integer
beforeBestEpoch; // 전이학습인 경우, before 모델 베스트 pth 경로를 보내야 해서 before 모델의 bestEpoch 을 조회하기
public String getOutputFolder() {
return String.valueOf(this.outputFolder);
}

View File

@@ -379,9 +379,30 @@ public class DockerTrainService {
addArg(c, "--saturation-range", req.getSaturationRange());
addArg(c, "--hue-delta", req.getHueDelta());
// 재시작을 위한 파라미터
if (req.getResumeFrom() != null && !req.getResumeFrom().isBlank()) {
c.add("--resume");
addArg(c, "--load-from", req.getResumeFrom());
} else if ("TRANSFER".equals(req.getTrainType())
&& req.getBeforeModelUUID() != null
&& req.getBeforeBestEpoch() != null) {
// 재시작이 아닌데 전이학습인 경우, 전이학습을 위한 파라미터
String finalLoadFromPath;
String bestModelFileName = "best_changed_fscore_epoch_" + req.getBeforeBestEpoch() + ".pth";
String epochFileName = "epoch_" + req.getBeforeBestEpoch() + ".pth";
// 이전 모델 response 경로
Path beforeModelDir = Path.of(responseDir).resolve(req.getBeforeModelUUID().toString());
// 이전 모델 response 의 베스트 fscore pth 모델 파일 경로
Path bestFilePath = beforeModelDir.resolve(bestModelFileName);
// 베스트 fscore 파일이 있으면 그 경로를 사용, 없으면 epoch_00.pth 파일 경로 사용
if (Files.exists(bestFilePath)) {
finalLoadFromPath = bestFilePath.toString();
} else {
finalLoadFromPath = beforeModelDir.resolve(epochFileName).toString();
}
addArg(c, "--load-from", finalLoadFromPath);
}
addArg(c, "--save-interval", 1);
@@ -498,6 +519,10 @@ public class DockerTrainService {
c.add("docker");
c.add("run");
c.add("--rm");
c.add("--user"); // root 권한 문제를 해결하기 위해 임시 추가
c.add("0:0");
c.add("--gpus");
c.add("all");
@@ -515,7 +540,8 @@ public class DockerTrainService {
}
c.add("-v");
c.add(responseDir + ":/checkpoints");
// c.add(responseDir + ":/checkpoints");
c.add(responseDir + "/" + req.getOutputFolder() + ":/checkpoints");
c.add("kamco-cd-train:latest");
@@ -523,7 +549,7 @@ public class DockerTrainService {
c.add("/workspace/change-detection-code/run_evaluation_pipeline.py");
addArg(c, "--dataset-folder", req.getDatasetFolder());
addArg(c, "--output-folder", req.getOutputFolder());
// addArg(c, "--output-folder", req.getOutputFolder());
c.add("--epoch");
c.add(modelFile);

View File

@@ -111,4 +111,34 @@ public class UploadApiController {
return ApiResponseDto.ok(uploadService.getUploadStatus(statusReq));
}
*/
@Operation(summary = "분류 데이터셋 대용량 파일 분할 전송", description = "분류 데이터셋 파일 대용량 파일을 청크 단위로 전송합니다.")
@ApiResponses(
value = {
@ApiResponse(responseCode = "200", description = "청크 업로드 성공", content = @Content),
@ApiResponse(responseCode = "400", description = "잘못된 요청 데이터", content = @Content),
@ApiResponse(responseCode = "404", description = "업로드 세션을 찾을 수 없음", content = @Content),
@ApiResponse(responseCode = "500", description = "서버 오류", content = @Content)
})
@PostMapping(value = "/chunk-upload-cls", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
public ApiResponseDto<UploadDto.UploadRes> uploadChunkClsDataSetFile(
@RequestParam("fileName") String fileName,
@RequestParam("fileSize") long fileSize,
@RequestParam("chunkIndex") Integer chunkIndex,
@RequestParam("chunkTotalIndex") Integer chunkTotalIndex,
@RequestPart("chunkFile") MultipartFile chunkFile,
@RequestParam("uuid") UUID uuid) {
String uploadDivi = "cls-dataset";
UploadDto.UploadAddReq upAddReqDto = new UploadDto.UploadAddReq();
upAddReqDto.setFileName(fileName);
upAddReqDto.setFileSize(fileSize);
upAddReqDto.setChunkIndex(chunkIndex);
upAddReqDto.setChunkTotalIndex(chunkTotalIndex);
upAddReqDto.setUploadDivi(uploadDivi);
upAddReqDto.setUuid(uuid);
return ApiResponseDto.ok(uploadService.uploadChunk(upAddReqDto, chunkFile));
}
}

View File

@@ -32,6 +32,12 @@ public class UploadService {
@Value("${file.dataset-tmp-dir}")
private String datasetTmpDir;
@Value("${file.dataset-cls-dir}")
private String datasetClsDir;
@Value("${file.dataset-cls-tmp-dir}")
private String datasetClsTmpDir;
// TODO 미사용시작
@Transactional
public DmlReturn initUpload(UploadDto.InitReq initReq) {
@@ -60,9 +66,12 @@ public class UploadService {
Integer chunkTotalIndex = upAddReqDto.getChunkTotalIndex();
String status = "UPLOADING";
if (uploadDivi.equals("dataset")) {
if (uploadDivi.equals("dataset")) { // 탐지데이터셋
tmpDataSetDir = datasetTmpDir;
fianlDir = datasetDir;
} else if (uploadDivi.equals("cls-dataset")) { // 분류데이터셋
tmpDataSetDir = datasetClsTmpDir;
fianlDir = datasetClsDir;
}
// 리턴용 파일 값

View File

@@ -50,16 +50,19 @@ swagger:
file:
dataset-dir: /data/training/request/
base_path: /backup/data/training
dataset-dir: ${file.base_path}/request/
dataset-tmp-dir: ${file.dataset-dir}tmp/
dataset-cls-dir: ${file.base_path}/request/cls/
dataset-cls-tmp-dir: ${file.dataset-cls-dir}tmp/
pt-path: /data/training/response/v6-cls-checkpoints/
pt-path: ${file.base_path}/response/v6-cls-checkpoints/
pt-FileName: yolov8_6th-6m.pt
train:
docker:
image: kamco-cd-train:latest
base_path: /data/training
base_path: /backup/data/training
request_dir: ${train.docker.base_path}/request
response_dir: ${train.docker.base_path}/response
symbolic_link_dir: ${train.docker.base_path}/tmp

View File

@@ -27,6 +27,8 @@ swagger:
file:
dataset-dir: /data/training/request/
dataset-tmp-dir: ${file.dataset-dir}tmp/
dataset-cls-dir: ${file.base_path}/request/cls/
dataset-cls-tmp-dir: ${file.dataset-cls-dir}tmp/
pt-path: /data/training/response/v6-cls-checkpoints/
pt-FileName: yolov8_6th-6m.pt

View File

@@ -37,8 +37,8 @@ spring:
max-file-size: 10GB
max-request-size: 10GB
transaction:
default-timeout: 300 # 5분 트랜잭션 타임아웃
#transaction:
# default-timeout: 300 # 5분 트랜잭션 타임아웃
logging:
level: