학습결과 파일 베스트 에폭 제외 삭제 추가, 납품데이터 등록 비동기 수정

src/main/java/com/kamco/cd/training/dataset/DatasetApiController.java

@@ -1,12 +1,12 @@
 package com.kamco.cd.training.dataset;
 
 import com.kamco.cd.training.config.api.ApiResponseDto;
-import com.kamco.cd.training.config.api.ApiResponseDto.ResponseObj;
 import com.kamco.cd.training.dataset.dto.DatasetDto;
 import com.kamco.cd.training.dataset.dto.DatasetDto.AddDeliveriesReq;
 import com.kamco.cd.training.dataset.dto.DatasetObjDto;
 import com.kamco.cd.training.dataset.dto.DatasetObjDto.DatasetClass;
 import com.kamco.cd.training.dataset.dto.DatasetObjDto.DatasetStorage;
+import com.kamco.cd.training.dataset.service.DatasetAsyncService;
 import com.kamco.cd.training.dataset.service.DatasetService;
 import com.kamco.cd.training.model.dto.FileDto.FoldersDto;
 import com.kamco.cd.training.model.dto.FileDto.SrchFoldersDto;
@@ -37,6 +37,7 @@ import org.springframework.web.bind.annotation.*;
 public class DatasetApiController {
 
   private final DatasetService datasetService;
+  private final DatasetAsyncService datasetAsyncService;
 
   @Operation(summary = "학습데이터 관리 목록 조회", description = "학습데이터 목록을 조회합니다.")
   @ApiResponses(
@@ -262,7 +263,7 @@ public class DatasetApiController {
             content =
                 @Content(
                     mediaType = "application/json",
-                    schema = @Schema(implementation = Page.class))),
+                    schema = @Schema(implementation = FoldersDto.class))),
         @ApiResponse(responseCode = "404", description = "조회 오류", content = @Content),
         @ApiResponse(responseCode = "500", description = "서버 오류", content = @Content)
       })
@@ -281,12 +282,13 @@ public class DatasetApiController {
             content =
                 @Content(
                     mediaType = "application/json",
-                    schema = @Schema(implementation = Page.class))),
+                    schema = @Schema(implementation = String.class))),
         @ApiResponse(responseCode = "404", description = "조회 오류", content = @Content),
         @ApiResponse(responseCode = "500", description = "서버 오류", content = @Content)
       })
   @PostMapping("/deliveries")
-  public ApiResponseDto<ResponseObj> insertDeliveriesDataset(@RequestBody AddDeliveriesReq req) {
-    return ApiResponseDto.createOK(datasetService.insertDeliveriesDataset(req));
+  public ApiResponseDto<String> insertDeliveriesDataset(@RequestBody AddDeliveriesReq req) {
+    datasetAsyncService.insertDeliveriesDatasetAsync(req);
+    return ApiResponseDto.createOK("ok");
   }
 }

src/main/java/com/kamco/cd/training/dataset/service/DatasetAsyncService.java

@@ -0,0 +1,134 @@
+package com.kamco.cd.training.dataset.service;
+
+import com.kamco.cd.training.common.enums.LearnDataRegister;
+import com.kamco.cd.training.dataset.dto.DatasetDto.AddDeliveriesReq;
+import com.kamco.cd.training.dataset.dto.DatasetDto.DatasetMngRegDto;
+import com.kamco.cd.training.postgres.core.DatasetCoreService;
+import java.util.UUID;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.log4j.Log4j2;
+import org.springframework.scheduling.annotation.Async;
+import org.springframework.stereotype.Service;
+
+@Service
+@Log4j2
+@RequiredArgsConstructor
+public class DatasetAsyncService {
+
+  private final DatasetService datasetService;
+  private final DatasetCoreService datasetCoreService;
+
+  private static final String LOG_PREFIX = "[납품 데이터셋]";
+
+  /**
+   * 납품 데이터셋 등록 비동기 업로드 처리 1) 데이터셋 구조/파일 검증 2) UID 생성 및 마스터 데이터 저장 3) 상태를 UPLOADING으로 변경 4) 실제
+   * 데이터(train/val/test) 등록 5) 완료 시 COMPLETED 상태로 변경 6) 실패 시 상태를 UPLOAD_FAILED로 변경 후 데이터 정리(삭제)
+   *
+   * @param req
+   */
+  @Async("datasetExecutor")
+  public void insertDeliveriesDatasetAsync(AddDeliveriesReq req) {
+
+    long startTime = System.currentTimeMillis();
+
+    log.info("{} 업로드 시작 ==========", LOG_PREFIX);
+    log.info(
+        "{} filePath={}, targetYyyy={}, compareYyyy={}, roundNo={}",
+        LOG_PREFIX,
+        req.getFilePath(),
+        req.getTargetYyyy(),
+        req.getCompareYyyy(),
+        req.getRoundNo());
+
+    Long datasetUid = null;
+
+    try {
+
+      // ===== 1. 폴더/파일 검증 =====
+      long validateStart = System.currentTimeMillis();
+
+      // 폴더 구조 검증
+      DatasetService.validateTrainValTestDirs(req.getFilePath());
+      // 파일 개수 검증
+      DatasetService.validateDirFileCount(req.getFilePath());
+
+      log.info("{} 데이터셋 검증 완료. ({} ms)", LOG_PREFIX, System.currentTimeMillis() - validateStart);
+
+      // ===== 2. UID 생성 =====
+      String uid = UUID.randomUUID().toString().replace("-", "").toUpperCase();
+      log.info("{} 생성된 UID: {}", LOG_PREFIX, uid);
+
+      // ===== 3. 마스터 데이터 생성 =====
+      String title = req.getTitle();
+
+      if (title == null || title.isBlank()) {
+        Integer compareYyyy = req.getCompareYyyy();
+        Integer targetYyyy = req.getTargetYyyy();
+
+        if (compareYyyy != null && targetYyyy != null) {
+          title = compareYyyy + "-" + targetYyyy;
+        } else {
+          title = null;
+        }
+      }
+
+      DatasetMngRegDto datasetMngRegDto = new DatasetMngRegDto();
+      datasetMngRegDto.setUid(uid);
+      datasetMngRegDto.setDataType("DELIVER");
+      datasetMngRegDto.setCompareYyyy(req.getCompareYyyy() == null ? 0 : req.getCompareYyyy());
+      datasetMngRegDto.setTargetYyyy(req.getTargetYyyy() == null ? 0 : req.getTargetYyyy());
+      datasetMngRegDto.setRoundNo(req.getRoundNo());
+      datasetMngRegDto.setTitle(title);
+      datasetMngRegDto.setMemo(req.getMemo());
+      datasetMngRegDto.setDatasetPath(req.getFilePath());
+
+      // 마스터 저장
+      datasetUid = datasetCoreService.insertDatasetMngData(datasetMngRegDto);
+
+      log.info("{} 마스터 저장 완료. datasetUid={}", LOG_PREFIX, datasetUid);
+
+      // ===== 4. 상태 변경 (업로드중) =====
+      datasetCoreService.updateDatasetUploadStatus(datasetUid, LearnDataRegister.UPLOADING);
+      log.info("{} 상태 변경 → UPLOADING. datasetUid={}", LOG_PREFIX, datasetUid);
+
+      // ===== 5. 데이터 등록 =====
+      long insertStart = System.currentTimeMillis();
+
+      // 납품 데이터 obj 등록
+      datasetService.insertDeliveriesDataset(req, datasetUid);
+
+      log.info(
+          "{} 데이터 등록 완료. datasetUid={}, 소요시간={} ms",
+          LOG_PREFIX,
+          datasetUid,
+          System.currentTimeMillis() - insertStart);
+
+      // ===== 6. 상태 변경 (완료) =====
+      datasetCoreService.updateDatasetUploadStatus(datasetUid, LearnDataRegister.COMPLETED);
+      log.info("{} 상태 변경 → COMPLETED. datasetUid={}", LOG_PREFIX, datasetUid);
+
+      log.info(
+          "{} 업로드 완료. 총 소요시간={} ms ==========", LOG_PREFIX, System.currentTimeMillis() - startTime);
+
+    } catch (Exception e) {
+
+      log.error(
+          "{} 업로드 실패. datasetUid={}, filePath={}", LOG_PREFIX, datasetUid, req.getFilePath(), e);
+
+      if (datasetUid != null) {
+        try {
+          // ===== 실패 처리 =====
+          datasetCoreService.updateDatasetUploadStatus(datasetUid, LearnDataRegister.UPLOAD_FAILED);
+          log.error("{} 상태 변경 → 업로드 실패. datasetUid={}", LOG_PREFIX, datasetUid);
+
+          // 실패 시 데이터 정리
+          datasetCoreService.deleteAllDatasetObj(datasetUid);
+          log.error("{} 데이터 정리 완료. datasetUid={}", LOG_PREFIX, datasetUid);
+
+        } catch (Exception ex) {
+          log.error("{} 실패 후 정리 작업 중 오류. datasetUid={}", LOG_PREFIX, datasetUid, ex);
+        }
+      }
+    }
+  }
+}

src/main/java/com/kamco/cd/training/dataset/service/DatasetBatchService.java

@@ -0,0 +1,152 @@
+package com.kamco.cd.training.dataset.service;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.kamco.cd.training.dataset.dto.DatasetObjDto.DatasetObjRegDto;
+import com.kamco.cd.training.postgres.core.DatasetCoreService;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.Map;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.log4j.Log4j2;
+import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
+
+@Service
+@Log4j2
+@RequiredArgsConstructor
+public class DatasetBatchService {
+
+  private final DatasetCoreService datasetCoreService;
+  private final ObjectMapper mapper;
+
+  /**
+   * 배치 단위 데이터 저장
+   *
+   * <p>- 전달받은 데이터 목록을 순회하며 개별 insert 처리 - batch 단위로 트랜잭션 관리
+   */
+  @Transactional
+  public void saveBatch(List<Map<String, Object>> batch, Long datasetUid, String type) {
+    for (Map<String, Object> map : batch) {
+      insertTrainTestData(map, datasetUid, type);
+    }
+  }
+
+  /**
+   * 단일 데이터 처리 및 insert DTO 생성
+   *
+   * <p>처리 흐름: 1) 경로/JSON 데이터 추출 2) 파일명에서 연도 및 도엽번호 파싱 3) label JSON → feature 단위 분리 4) feature별 DTO
+   * 생성 후 DB insert
+   */
+  private void insertTrainTestData(Map<String, Object> map, Long datasetUid, String subDir) {
+
+    String comparePath = (String) map.get("input1");
+    String targetPath = (String) map.get("input2");
+    String labelPath = (String) map.get("label");
+    String geojsonPath = (String) map.get("geojson_path");
+    Object labelJson = map.get("label-json");
+
+    // JSON 파싱
+    JsonNode json = parseJson(labelJson);
+
+    // 파일명 파싱
+    String fileName = Paths.get(comparePath).getFileName().toString();
+    String[] fileNameStr = fileName.split("_");
+
+    if (fileNameStr.length < 4) {
+      log.error("파일명 파싱 실패: {}", fileName);
+      throw new IllegalArgumentException("잘못된 파일명 형식: " + fileName);
+    }
+
+    int compareYyyy = parseInt(fileNameStr[1], "compareYyyy", fileName);
+    int targetYyyy = parseInt(fileNameStr[2], "targetYyyy", fileName);
+    String mapSheetNum = fileNameStr[3];
+
+    // JSON 유효성 체크
+    JsonNode featuresNode = json.path("features");
+
+    if (featuresNode.isMissingNode() || !featuresNode.isArray() || featuresNode.isEmpty()) {
+      return; // skip
+    }
+
+    ObjectNode base = mapper.createObjectNode();
+    base.put("type", "FeatureCollection");
+
+    for (JsonNode feature : featuresNode) {
+
+      JsonNode prop = feature.path("properties");
+
+      String compareClassCd = prop.path("before").asText(null);
+      String targetClassCd = prop.path("after").asText(null);
+
+      ArrayNode arr = mapper.createArrayNode();
+      arr.add(feature);
+
+      ObjectNode root = base.deepCopy();
+      root.set("features", arr);
+
+      DatasetObjRegDto objRegDto =
+          DatasetObjRegDto.builder()
+              .datasetUid(datasetUid)
+              .compareYyyy(compareYyyy)
+              .compareClassCd(compareClassCd)
+              .targetYyyy(targetYyyy)
+              .targetClassCd(targetClassCd)
+              .comparePath(comparePath)
+              .targetPath(targetPath)
+              .labelPath(labelPath)
+              .mapSheetNum(mapSheetNum)
+              .geojson(root)
+              .geojsonPath(geojsonPath)
+              .fileName(fileName)
+              .build();
+
+      // 데이터 타입별 insert
+      insertByType(subDir, objRegDto);
+    }
+  }
+
+  /** 데이터 타입별 insert 처리 - type 값에 따라 대상 테이블 분기 - 잘못된 타입 입력 시 예외 발생 */
+  private void insertByType(String type, DatasetObjRegDto dto) {
+
+    switch (type) {
+      case "train" -> datasetCoreService.insertDatasetObj(dto);
+      case "val" -> datasetCoreService.insertDatasetValObj(dto);
+      case "test" -> datasetCoreService.insertDatasetTestObj(dto);
+      default -> throw new IllegalArgumentException("잘못된 타입: " + type);
+    }
+  }
+
+  /**
+   * label_json → JsonNode 변환
+   *
+   * <p>- JsonNode면 그대로 사용 - 문자열이면 파싱 수행 - 실패 시 로그 후 예외 발생
+   */
+  private JsonNode parseJson(Object labelJson) {
+    try {
+      if (labelJson instanceof JsonNode jn) {
+        return jn;
+      }
+      return mapper.readTree(labelJson.toString());
+    } catch (Exception e) {
+      log.error("label_json parse error: {}", labelJson, e);
+      throw new RuntimeException("label_json parse error", e);
+    }
+  }
+
+  /**
+   * 문자열 → 정수 변환
+   *
+   * <p>- 파싱 실패 시 어떤 필드/파일에서 발생했는지 로그 기록 - 잘못된 데이터는 즉시 예외 처리
+   */
+  private int parseInt(String value, String field, String fileName) {
+    try {
+      return Integer.parseInt(value);
+    } catch (NumberFormatException e) {
+      log.error("{} 파싱 실패. fileName={}, value={}", field, fileName, value);
+      throw new IllegalArgumentException(field + " 파싱 실패: " + fileName);
+    }
+  }
+}

src/main/java/com/kamco/cd/training/dataset/service/DatasetService.java

@@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.ArrayNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.kamco.cd.training.common.enums.LearnDataRegister;
 import com.kamco.cd.training.common.enums.LearnDataType;
 import com.kamco.cd.training.common.exception.CustomApiException;
 import com.kamco.cd.training.common.service.FormatStorage;
@@ -22,8 +23,6 @@ import com.kamco.cd.training.dataset.dto.DatasetObjDto.SearchReq;
 import com.kamco.cd.training.model.dto.FileDto.FoldersDto;
 import com.kamco.cd.training.model.dto.FileDto.SrchFoldersDto;
 import com.kamco.cd.training.postgres.core.DatasetCoreService;
-import jakarta.persistence.EntityManager;
-import jakarta.persistence.PersistenceContext;
 import jakarta.validation.Valid;
 import java.io.File;
 import java.io.IOException;
@@ -45,7 +44,6 @@ import java.util.stream.Collectors;
 import java.util.stream.Stream;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
-import org.springframework.beans.factory.annotation.Value;
 import org.springframework.core.io.InputStreamResource;
 import org.springframework.core.io.Resource;
 import org.springframework.data.domain.Page;
@@ -62,11 +60,7 @@ import org.springframework.transaction.annotation.Transactional;
 public class DatasetService {
 
   private final DatasetCoreService datasetCoreService;
-  @PersistenceContext private EntityManager em;
-  private final ObjectMapper mapper = new ObjectMapper();
-
-  @Value("${file.dataset-dir}")
-  private String datasetDir;
+  private final DatasetBatchService datasetBatchService;
 
   private static final List<String> LABEL_DIRS = List.of("label-json", "label", "input1", "input2");
   private static final List<String> REQUIRED_DIRS = Arrays.asList("train", "val", "test");
@@ -143,7 +137,6 @@ public class DatasetService {
     return datasetCoreService.searchDatasetObjectList(searchReq);
   }
 
-  @Transactional
   public UUID deleteDatasetObjByUuid(UUID uuid) {
     return datasetCoreService.deleteDatasetObjByUuid(uuid);
   }
@@ -241,7 +234,7 @@ public class DatasetService {
       return new ResponseObj(ApiResponseCode.INTERNAL_SERVER_ERROR, e.getMessage());
     }
 
-    datasetCoreService.updateDatasetUploadStatus(datasetUid);
+    datasetCoreService.updateDatasetUploadStatus(datasetUid, LearnDataRegister.COMPLETED);
     return new ResponseObj(ApiResponseCode.OK, "업로드 성공하였습니다.");
   }
 
@@ -616,146 +609,65 @@ public class DatasetService {
    * 납품 데이터 등록
    *
    * @param req 폴더경로, 메모
-   * @return 성공/실패 여부
+   * @return 성공/실패 여부0
    */
-  public ResponseObj insertDeliveriesDataset(AddDeliveriesReq req) {
+  public void insertDeliveriesDataset(AddDeliveriesReq req, Long datasetUid) {
 
     long startTime = System.currentTimeMillis();
 
-    log.info("========== 납품 데이터셋 업로드 시작 ==========");
-    log.info("filePath: {}", req.getFilePath());
-
-    DatasetMngRegDto datasetMngRegDto = new DatasetMngRegDto();
-
-    String uid = UUID.randomUUID().toString().replace("-", "").toUpperCase();
-
-    datasetMngRegDto.setUid(uid);
-    datasetMngRegDto.setDataType("DELIVER");
-    datasetMngRegDto.setCompareYyyy(req.getCompareYyyy() == null ? 0 : req.getCompareYyyy());
-    datasetMngRegDto.setTargetYyyy(req.getTargetYyyy() == null ? 0 : req.getTargetYyyy());
-    datasetMngRegDto.setRoundNo(req.getRoundNo() == null ? null : req.getRoundNo());
-    datasetMngRegDto.setTitle(req.getTitle() == null ? null : req.getTitle());
-    datasetMngRegDto.setMemo(req.getMemo() == null ? null : req.getMemo());
-    datasetMngRegDto.setDatasetPath(req.getFilePath());
-
-    // 마스터 저장 (트랜잭션 내부)
-    Long datasetUid = datasetCoreService.insertDatasetMngData(datasetMngRegDto);
-
-    log.info("납품 Dataset 마스터 저장 완료. datasetUid: {}", datasetUid);
-
-    // 검증
-    validateTrainValTestDirs(req.getFilePath());
-    validateDirFileCount(req.getFilePath());
-
     // 처리
     processType(req.getFilePath(), datasetUid, "train");
     processType(req.getFilePath(), datasetUid, "val");
     processType(req.getFilePath(), datasetUid, "test");
 
-    datasetCoreService.updateDatasetUploadStatus(datasetUid);
-
     log.info("========== 전체 완료. 총 소요시간: {} ms ==========", System.currentTimeMillis() - startTime);
-
-    return new ResponseObj(ApiResponseCode.OK, "업로드 성공하였습니다.");
   }
 
+  /**
+   * 납품 데이터 등록 처리
+   *
+   * @param path
+   * @param datasetUid
+   * @param type
+   */
   private void processType(String path, Long datasetUid, String type) {
-
     long start = System.currentTimeMillis();
 
-    log.info("[{}] 데이터 처리 시작", type.toUpperCase());
+    log.info("[납품 데이터 등록 처리][{}] 시작", type.toUpperCase());
 
     List<Map<String, Object>> list = getUnzipDatasetFiles(path, type);
 
-    log.info("[{}] 파일 개수: {}", type.toUpperCase(), list.size());
+    int batchSize = 1000;
+    int total = list.size();
+    int processed = 0;
 
-    int count = 0;
+    for (int i = 0; i < total; i += batchSize) {
 
-    for (Map<String, Object> map : list) {
-      insertTrainTestData(map, datasetUid, type);
-      count++;
+      List<Map<String, Object>> batch = list.subList(i, Math.min(i + batchSize, total));
 
-      if (count % 1000 == 0 || count == list.size()) {
-        log.info("[{}] 진행건수: {}", type.toUpperCase(), count);
+      try {
+        log.info("[납품 데이터 등록 처리][{}] batch 시작: {} ~ {}", type, i, i + batch.size());
+
+        datasetBatchService.saveBatch(batch, datasetUid, type);
+
+        processed += batch.size();
+      } catch (Exception e) {
+        log.error("batch 실패 row 데이터: {}", batch);
+        log.error(
+            "[납품 데이터 등록 처리][{}] batch 실패. range: {} ~ {}, datasetUid={}",
+            type,
+            i,
+            i + batch.size(),
+            datasetUid,
+            e);
+        throw e;
       }
     }
 
     log.info(
-        "[{}] 완료. 총 {}건, 소요시간: {} ms",
-        type.toUpperCase(),
-        count,
+        "[납품 데이터 등록 처리][{}] 완료. 총 {}건, 소요시간: {} ms",
+        type,
+        total,
         System.currentTimeMillis() - start);
   }
-
-  @Transactional
-  public void insertTrainTestData(Map<String, Object> map, Long datasetUid, String subDir) {
-
-    String comparePath = (String) map.get("input1");
-    String targetPath = (String) map.get("input2");
-    String labelPath = (String) map.get("label");
-    String geojsonPath = (String) map.get("geojson_path");
-    Object labelJson = map.get("label-json");
-
-    JsonNode json;
-
-    try {
-      if (labelJson instanceof JsonNode jn) {
-        json = jn;
-      } else {
-        json = mapper.readTree(labelJson.toString());
-      }
-    } catch (Exception e) {
-      throw new RuntimeException("label_json parse error", e);
-    }
-
-    String fileName = Paths.get(comparePath).getFileName().toString();
-    String[] fileNameStr = fileName.split("_");
-
-    String compareYyyy = fileNameStr[1];
-    String targetYyyy = fileNameStr[2];
-    String mapSheetNum = fileNameStr[3];
-
-    if (json != null && json.path("features") != null && !json.path("features").isEmpty()) {
-
-      for (JsonNode feature : json.path("features")) {
-
-        JsonNode prop = feature.path("properties");
-
-        String compareClassCd = prop.path("before").asText(null);
-        String targetClassCd = prop.path("after").asText(null);
-
-        ObjectNode root = mapper.createObjectNode();
-        root.put("type", "FeatureCollection");
-
-        ArrayNode features = mapper.createArrayNode();
-        features.add(feature);
-        root.set("features", features);
-
-        DatasetObjRegDto objRegDto =
-            DatasetObjRegDto.builder()
-                .datasetUid(datasetUid)
-                .compareYyyy(Integer.parseInt(compareYyyy))
-                .compareClassCd(compareClassCd)
-                .targetYyyy(Integer.parseInt(targetYyyy))
-                .targetClassCd(targetClassCd)
-                .comparePath(comparePath)
-                .targetPath(targetPath)
-                .labelPath(labelPath)
-                .mapSheetNum(mapSheetNum)
-                .geojson(root)
-                .geojsonPath(geojsonPath)
-                .fileName(fileName)
-                .build();
-
-        // insert
-        if (subDir.equals("train")) {
-          datasetCoreService.insertDatasetObj(objRegDto);
-        } else if (subDir.equals("val")) {
-          datasetCoreService.insertDatasetValObj(objRegDto);
-        } else {
-          datasetCoreService.insertDatasetTestObj(objRegDto);
-        }
-      }
-    }
-  }
 }