Merge pull request '납품데이터 등록 수정' (#171) from feat/training_260324 into develop

Reviewed-on: #171

src/main/java/com/kamco/cd/training/config/api/ApiLogFilter.java

@@ -16,6 +16,12 @@ public class ApiLogFilter extends OncePerRequestFilter {
   protected void doFilterInternal(
       HttpServletRequest request, HttpServletResponse response, FilterChain filterChain)
       throws ServletException, IOException {
+    String uri = request.getRequestURI();
+    if (uri.contains("/download/")) {
+      filterChain.doFilter(request, response);
+      return;
+    }
+
     ContentCachingRequestWrapper wrappedRequest = new ContentCachingRequestWrapper(request);
 
     ContentCachingResponseWrapper wrappedResponse = new ContentCachingResponseWrapper(response);

src/main/java/com/kamco/cd/training/dataset/DatasetApiController.java

@@ -1,11 +1,15 @@
 package com.kamco.cd.training.dataset;
 
 import com.kamco.cd.training.config.api.ApiResponseDto;
+import com.kamco.cd.training.config.api.ApiResponseDto.ResponseObj;
 import com.kamco.cd.training.dataset.dto.DatasetDto;
+import com.kamco.cd.training.dataset.dto.DatasetDto.AddDeliveriesReq;
 import com.kamco.cd.training.dataset.dto.DatasetObjDto;
 import com.kamco.cd.training.dataset.dto.DatasetObjDto.DatasetClass;
 import com.kamco.cd.training.dataset.dto.DatasetObjDto.DatasetStorage;
 import com.kamco.cd.training.dataset.service.DatasetService;
+import com.kamco.cd.training.model.dto.FileDto.FoldersDto;
+import com.kamco.cd.training.model.dto.FileDto.SrchFoldersDto;
 import io.swagger.v3.oas.annotations.Operation;
 import io.swagger.v3.oas.annotations.Parameter;
 import io.swagger.v3.oas.annotations.media.Content;
@@ -248,4 +252,42 @@ public class DatasetApiController {
     String path = datasetService.getFilePathByUUIDPathType(uuid, pathType);
     return datasetService.getFilePathByFile(path);
   }
+
+  @Operation(summary = "납품 폴더 조회", description = "납품 폴더 조회 API")
+  @ApiResponses(
+      value = {
+        @ApiResponse(
+            responseCode = "200",
+            description = "조회 성공",
+            content =
+                @Content(
+                    mediaType = "application/json",
+                    schema = @Schema(implementation = Page.class))),
+        @ApiResponse(responseCode = "404", description = "조회 오류", content = @Content),
+        @ApiResponse(responseCode = "500", description = "서버 오류", content = @Content)
+      })
+  @PostMapping("/folder-list")
+  public ApiResponseDto<FoldersDto> getDir(@RequestBody SrchFoldersDto srchDto) throws IOException {
+
+    return ApiResponseDto.createOK(datasetService.getFolderAll(srchDto));
+  }
+
+  @Operation(summary = "납품 학습데이터셋 등록", description = "납품 학습데이터셋 등록 API")
+  @ApiResponses(
+      value = {
+        @ApiResponse(
+            responseCode = "200",
+            description = "등록 성공",
+            content =
+                @Content(
+                    mediaType = "application/json",
+                    schema = @Schema(implementation = Page.class))),
+        @ApiResponse(responseCode = "404", description = "조회 오류", content = @Content),
+        @ApiResponse(responseCode = "500", description = "서버 오류", content = @Content)
+      })
+  @PostMapping("/deliveries")
+  public ApiResponseDto<ResponseObj> insertDeliveriesDataset(@RequestBody AddDeliveriesReq req)
+      throws IOException {
+    return ApiResponseDto.createOK(datasetService.insertDeliveriesDataset(req));
+  }
 }

src/main/java/com/kamco/cd/training/dataset/dto/DatasetDto.java

@@ -532,4 +532,13 @@ public class DatasetDto {
     private Long totalObjectCount;
     private String datasetPath;
   }
+
+  @Getter
+  @Setter
+  @NoArgsConstructor
+  @AllArgsConstructor
+  public static class AddDeliveriesReq {
+    @Schema(description = "경로", example = "/")
+    private String filePath;
+  }
 }

src/main/java/com/kamco/cd/training/dataset/service/DatasetService.java

@@ -11,6 +11,7 @@ import com.kamco.cd.training.common.utils.FIleChecker;
 import com.kamco.cd.training.config.api.ApiResponseDto.ApiResponseCode;
 import com.kamco.cd.training.config.api.ApiResponseDto.ResponseObj;
 import com.kamco.cd.training.dataset.dto.DatasetDto;
+import com.kamco.cd.training.dataset.dto.DatasetDto.AddDeliveriesReq;
 import com.kamco.cd.training.dataset.dto.DatasetDto.AddReq;
 import com.kamco.cd.training.dataset.dto.DatasetDto.DatasetMngRegDto;
 import com.kamco.cd.training.dataset.dto.DatasetObjDto;
@@ -18,8 +19,13 @@ import com.kamco.cd.training.dataset.dto.DatasetObjDto.DatasetClass;
 import com.kamco.cd.training.dataset.dto.DatasetObjDto.DatasetObjRegDto;
 import com.kamco.cd.training.dataset.dto.DatasetObjDto.DatasetStorage;
 import com.kamco.cd.training.dataset.dto.DatasetObjDto.SearchReq;
+import com.kamco.cd.training.model.dto.FileDto.FoldersDto;
+import com.kamco.cd.training.model.dto.FileDto.SrchFoldersDto;
 import com.kamco.cd.training.postgres.core.DatasetCoreService;
+import jakarta.persistence.EntityManager;
+import jakarta.persistence.PersistenceContext;
 import jakarta.validation.Valid;
+import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.file.Files;
@@ -27,6 +33,8 @@ import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -51,10 +59,11 @@ import org.springframework.transaction.annotation.Transactional;
 @Slf4j
 @Service
 @RequiredArgsConstructor
-@Transactional
 public class DatasetService {
 
   private final DatasetCoreService datasetCoreService;
+  @PersistenceContext private EntityManager em;
+  private final ObjectMapper mapper = new ObjectMapper();
 
   @Value("${file.dataset-dir}")
   private String datasetDir;
@@ -134,6 +143,7 @@ public class DatasetService {
     return datasetCoreService.searchDatasetObjectList(searchReq);
   }
 
+  @Transactional
   public UUID deleteDatasetObjByUuid(UUID uuid) {
     return datasetCoreService.deleteDatasetObjByUuid(uuid);
   }
@@ -506,4 +516,243 @@ public class DatasetService {
       }
     }
   }
+
+  /**
+   * 폴더 조회
+   *
+   * @param srchDto 폴더 경로
+   * @return 폴더 리스트
+   * @throws IOException
+   */
+  public FoldersDto getFolderAll(SrchFoldersDto srchDto) throws IOException {
+
+    File dir = new File(srchDto.getDirPath() == null ? "/" : srchDto.getDirPath());
+
+    // 존재 + 디렉토리 체크
+    if (!dir.exists() || !dir.isDirectory()) {
+      throw new CustomApiException("BAD_REQUEST", HttpStatus.BAD_REQUEST, "잘못된 경로입니다.");
+    }
+
+    // 권한 없을때
+    if (!dir.canRead()) {
+      throw new CustomApiException(
+          ApiResponseCode.FORBIDDEN.getId(), HttpStatus.FORBIDDEN, "디렉토리에 접근할 권한이 없습니다.");
+    }
+
+    String canonicalPath = dir.getCanonicalPath();
+
+    File[] files = dir.listFiles();
+
+    if (files == null) {
+      return new FoldersDto(canonicalPath, 0, 0, Collections.emptyList());
+    }
+
+    List<FIleChecker.Folder> folders = new ArrayList<>();
+
+    int folderTotCnt = 0;
+    int folderErrTotCnt = 0;
+
+    for (File f : files) {
+
+      // 숨김 제외
+      if (f.isHidden()) continue;
+
+      if (f.isDirectory()) {
+
+        // 폴더 개수 증가
+        folderTotCnt++;
+
+        // 폴더 유효성 여부 (기본 true, 이후 검증 로직으로 변경 가능)
+        boolean isValid = true;
+
+        // 유효하지 않은 폴더 카운트 증가
+        if (!isValid) folderErrTotCnt++;
+
+        // 현재 폴더 이름 (ex: train, images 등)
+        String folderNm = f.getName();
+
+        // 부모 경로 (ex: /data/datasets)
+        String parentPath = f.getParent();
+
+        // 부모 폴더 이름 (ex: datasets)
+        String parentFolderNm = new File(parentPath).getName();
+
+        // 전체 절대 경로 (ex: /data/datasets/train)
+        String fullPath = f.getAbsolutePath();
+
+        // 폴더 깊이 (경로 기준 depth)
+        // ex: /a/b/c → depth = 3
+        int depth = f.toPath().getNameCount();
+
+        // 하위 폴더 개수
+        long childCnt = FIleChecker.getChildFolderCount(f);
+
+        // 마지막 수정 시간 (문자열 포맷)
+        String lastModified = FIleChecker.getLastModified(f);
+
+        // Folder DTO 생성 및 리스트에 추가
+        folders.add(
+            new FIleChecker.Folder(
+                folderNm, // 폴더명
+                parentFolderNm, // 부모 폴더명
+                parentPath, // 부모 경로
+                fullPath, // 전체 경로
+                depth, // 깊이
+                childCnt, // 하위 폴더 개수
+                lastModified, // 수정일시
+                isValid // 유효성 여부
+                ));
+      }
+    }
+
+    // 폴더 정렬
+    folders.sort(
+        Comparator.comparing(FIleChecker.Folder::getFolderNm, String.CASE_INSENSITIVE_ORDER));
+
+    return new FoldersDto(canonicalPath, folderTotCnt, folderErrTotCnt, folders);
+  }
+
+  /**
+   * 납품 데이터 등록
+   *
+   * @param req 폴더경로, 메모
+   * @return 성공/실패 여부
+   */
+  public ResponseObj insertDeliveriesDataset(AddDeliveriesReq req) {
+
+    long startTime = System.currentTimeMillis();
+
+    log.info("========== 납품 데이터셋 업로드 시작 ==========");
+    log.info("filePath: {}", req.getFilePath());
+
+    DatasetMngRegDto datasetMngRegDto = new DatasetMngRegDto();
+
+    String uid = UUID.randomUUID().toString().replace("-", "").toUpperCase();
+
+    datasetMngRegDto.setUid(uid);
+    datasetMngRegDto.setDataType("DELIVER");
+    datasetMngRegDto.setCompareYyyy(0);
+    datasetMngRegDto.setTargetYyyy(0);
+    datasetMngRegDto.setDatasetPath(req.getFilePath());
+
+    // 마스터 저장 (트랜잭션 내부)
+    Long datasetUid = datasetCoreService.insertDatasetMngData(datasetMngRegDto);
+
+    log.info("납품 Dataset 마스터 저장 완료. datasetUid: {}", datasetUid);
+
+    // 검증
+    validateTrainValTestDirs(req.getFilePath());
+    validateDirFileCount(req.getFilePath());
+
+    // 처리
+    processType(req.getFilePath(), datasetUid, "train");
+    processType(req.getFilePath(), datasetUid, "val");
+    processType(req.getFilePath(), datasetUid, "test");
+
+    datasetCoreService.updateDatasetUploadStatus(datasetUid);
+
+    log.info("========== 전체 완료. 총 소요시간: {} ms ==========", System.currentTimeMillis() - startTime);
+
+    return new ResponseObj(ApiResponseCode.OK, "업로드 성공하였습니다.");
+  }
+
+  private void processType(String path, Long datasetUid, String type) {
+
+    long start = System.currentTimeMillis();
+
+    log.info("[{}] 데이터 처리 시작", type.toUpperCase());
+
+    List<Map<String, Object>> list = getUnzipDatasetFiles(path, type);
+
+    log.info("[{}] 파일 개수: {}", type.toUpperCase(), list.size());
+
+    int count = 0;
+
+    for (Map<String, Object> map : list) {
+      insertTrainTestData(map, datasetUid, type);
+      count++;
+
+      if (count % 1000 == 0 || count == list.size()) {
+        log.info("[{}] 진행건수: {}", type.toUpperCase(), count);
+      }
+    }
+
+    log.info(
+        "[{}] 완료. 총 {}건, 소요시간: {} ms",
+        type.toUpperCase(),
+        count,
+        System.currentTimeMillis() - start);
+  }
+
+  @Transactional
+  public void insertTrainTestData(Map<String, Object> map, Long datasetUid, String subDir) {
+
+    String comparePath = (String) map.get("input1");
+    String targetPath = (String) map.get("input2");
+    String labelPath = (String) map.get("label");
+    String geojsonPath = (String) map.get("geojson_path");
+    Object labelJson = map.get("label-json");
+
+    JsonNode json;
+
+    try {
+      if (labelJson instanceof JsonNode jn) {
+        json = jn;
+      } else {
+        json = mapper.readTree(labelJson.toString());
+      }
+    } catch (Exception e) {
+      throw new RuntimeException("label_json parse error", e);
+    }
+
+    String fileName = Paths.get(comparePath).getFileName().toString();
+    String[] fileNameStr = fileName.split("_");
+
+    String compareYyyy = fileNameStr[1];
+    String targetYyyy = fileNameStr[2];
+    String mapSheetNum = fileNameStr[3];
+
+    if (json != null && json.path("features") != null && !json.path("features").isEmpty()) {
+
+      for (JsonNode feature : json.path("features")) {
+
+        JsonNode prop = feature.path("properties");
+
+        String compareClassCd = prop.path("before").asText(null);
+        String targetClassCd = prop.path("after").asText(null);
+
+        ObjectNode root = mapper.createObjectNode();
+        root.put("type", "FeatureCollection");
+
+        ArrayNode features = mapper.createArrayNode();
+        features.add(feature);
+        root.set("features", features);
+
+        DatasetObjRegDto objRegDto =
+            DatasetObjRegDto.builder()
+                .datasetUid(datasetUid)
+                .compareYyyy(Integer.parseInt(compareYyyy))
+                .compareClassCd(compareClassCd)
+                .targetYyyy(Integer.parseInt(targetYyyy))
+                .targetClassCd(targetClassCd)
+                .comparePath(comparePath)
+                .targetPath(targetPath)
+                .labelPath(labelPath)
+                .mapSheetNum(mapSheetNum)
+                .geojson(root)
+                .geojsonPath(geojsonPath)
+                .fileName(fileName)
+                .build();
+
+        // insert
+        if (subDir.equals("train")) {
+          datasetCoreService.insertDatasetObj(objRegDto);
+        } else if (subDir.equals("val")) {
+          datasetCoreService.insertDatasetValObj(objRegDto);
+        } else {
+          datasetCoreService.insertDatasetTestObj(objRegDto);
+        }
+      }
+    }
+  }
 }

src/main/java/com/kamco/cd/training/model/dto/FileDto.java

@@ -0,0 +1,157 @@
+package com.kamco.cd.training.model.dto;
+
+import com.kamco.cd.training.common.utils.FIleChecker;
+import io.swagger.v3.oas.annotations.media.Schema;
+import jakarta.validation.constraints.NotNull;
+import java.util.List;
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+
+public class FileDto {
+
+  @Getter
+  @Setter
+  @NoArgsConstructor
+  @AllArgsConstructor
+  public static class SrchFoldersDto {
+    @Schema(description = "디렉토리경로(ROOT:/)", example = "")
+    @NotNull
+    private String dirPath = "/";
+  }
+
+  @Getter
+  @Setter
+  @NoArgsConstructor
+  @AllArgsConstructor
+  public static class SrchFilesDto {
+    @Schema(description = "디렉토리경로", example = "D:\\kamco\\2022\\캠코_2021_2022_34602060_D1")
+    @NotNull
+    private String dirPath;
+
+    @Schema(description = "전체(*), cpg,dbf,geojson등", example = "*")
+    @NotNull
+    private String extension;
+
+    @Schema(description = "파일명(name), 최종수정일(date)", example = "name")
+    @NotNull
+    private String sortType;
+
+    @Schema(description = "파일시작위치", example = "1")
+    @NotNull
+    private Integer startPos;
+
+    @Schema(description = "파일종료위치", example = "100")
+    @NotNull
+    private Integer endPos;
+  }
+
+  @Getter
+  @Setter
+  @NoArgsConstructor
+  @AllArgsConstructor
+  public static class SrchFilesDepthDto extends SrchFilesDto {
+    @Schema(description = "최대폴더Depth", example = "5")
+    @NotNull
+    private Integer maxDepth;
+  }
+
+  @Schema(name = "FolderDto", description = "폴더 정보")
+  @Getter
+  public static class FolderDto {
+    private final String folderNm;
+    private final String parentFolderNm;
+    private final String parentPath;
+    private final String fullPath;
+    private final int depth;
+    private final long childCnt;
+    private final String lastModified;
+    private final Boolean isValid;
+
+    public FolderDto(
+        String folderNm,
+        String parentFolderNm,
+        String parentPath,
+        String fullPath,
+        int depth,
+        long childCnt,
+        String lastModified,
+        Boolean isValid) {
+      this.folderNm = folderNm;
+      this.parentFolderNm = parentFolderNm;
+      this.parentPath = parentPath;
+      this.fullPath = fullPath;
+      this.depth = depth;
+      this.childCnt = childCnt;
+      this.lastModified = lastModified;
+      this.isValid = isValid;
+    }
+  }
+
+  @Schema(name = "FoldersDto", description = "폴더목록 정보")
+  @Getter
+  public static class FoldersDto {
+    private final String dirPath;
+    private final int folderTotCnt;
+    private final int folderErrTotCnt;
+    private final List<FIleChecker.Folder> folders;
+
+    public FoldersDto(
+        String dirPath, int folderTotCnt, int folderErrTotCnt, List<FIleChecker.Folder> folders) {
+
+      this.dirPath = dirPath;
+      this.folderTotCnt = folderTotCnt;
+      this.folderErrTotCnt = folderErrTotCnt;
+      this.folders = folders;
+    }
+  }
+
+  @Schema(name = "File Basic", description = "파일 기본 정보")
+  @Getter
+  public static class Basic {
+
+    private final String fileNm;
+    private final String parentFolderNm;
+    private final String parentPath;
+    private final String fullPath;
+    private final String extension;
+    private final long fileSize;
+    private final String lastModified;
+
+    public Basic(
+        String fileNm,
+        String parentFolderNm,
+        String parentPath,
+        String fullPath,
+        String extension,
+        long fileSize,
+        String lastModified) {
+      this.fileNm = fileNm;
+      this.parentFolderNm = parentFolderNm;
+      this.parentPath = parentPath;
+      this.fullPath = fullPath;
+      this.extension = extension;
+      this.fileSize = fileSize;
+      this.lastModified = lastModified;
+    }
+  }
+
+  @Schema(name = "FilesDto", description = "파일 목록 정보")
+  @Getter
+  public static class FilesDto {
+    private final String dirPath;
+    private final int fileTotCnt;
+    private final long fileTotSize;
+    private final List<FIleChecker.Basic> files;
+
+    public FilesDto(
+        String dirPath, int fileTotCnt, long fileTotSize, List<FIleChecker.Basic> files) {
+
+      this.dirPath = dirPath;
+      this.fileTotCnt = fileTotCnt;
+      this.fileTotSize = fileTotSize;
+      this.files = files;
+    }
+  }
+}

src/main/java/com/kamco/cd/training/postgres/core/DatasetCoreService.java

@@ -15,7 +15,6 @@ import com.kamco.cd.training.postgres.entity.DatasetEntity;
 import com.kamco.cd.training.postgres.entity.DatasetObjEntity;
 import com.kamco.cd.training.postgres.repository.dataset.DatasetObjRepository;
 import com.kamco.cd.training.postgres.repository.dataset.DatasetRepository;
-import jakarta.transaction.Transactional;
 import java.time.ZonedDateTime;
 import java.util.List;
 import java.util.UUID;
@@ -24,6 +23,7 @@ import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.data.domain.Page;
 import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
 
 @Service
 @RequiredArgsConstructor
@@ -230,10 +230,12 @@ public class DatasetCoreService
     return datasetObjRepository.getFilePathByUUIDPathType(uuid, pathType);
   }
 
+  @Transactional
   public void insertDatasetTestObj(DatasetObjRegDto objRegDto) {
     datasetObjRepository.insertDatasetTestObj(objRegDto);
   }
 
+  @Transactional
   public void updateDatasetUploadStatus(Long datasetUid) {
     DatasetEntity entity =
         datasetRepository
@@ -243,6 +245,7 @@ public class DatasetCoreService
     entity.setStatus(LearnDataRegister.COMPLETED.getId());
   }
 
+  @Transactional
   public void insertDatasetValObj(DatasetObjRegDto objRegDto) {
     datasetObjRepository.insertDatasetValObj(objRegDto);
   }