학습데이터 업로드, unzip 로직 진행중

src/main/java/com/kamco/cd/training/dataset/DatasetApiController.java

@@ -14,10 +14,16 @@ import io.swagger.v3.oas.annotations.responses.ApiResponse;
 import io.swagger.v3.oas.annotations.responses.ApiResponses;
 import io.swagger.v3.oas.annotations.tags.Tag;
 import jakarta.validation.Valid;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.List;
 import java.util.UUID;
 import lombok.RequiredArgsConstructor;
+import org.springframework.core.io.Resource;
+import org.springframework.core.io.UrlResource;
 import org.springframework.data.domain.Page;
+import org.springframework.http.MediaType;
+import org.springframework.http.ResponseEntity;
 import org.springframework.web.bind.annotation.*;
 
 @Tag(name = "학습데이터 관리", description = "어드민 홈 > 학습데이터관리 > 전체데이터 API")
@@ -235,4 +241,25 @@ public class DatasetApiController {
   public ApiResponseDto<DatasetStorage> getUsableBytes() {
     return ApiResponseDto.ok(datasetService.getUsableBytes());
   }
+
+  @Operation(summary = "데이터셋 등록", description = "데이터셋을 등록 합니다.")
+  @PostMapping
+  public ApiResponseDto<ApiResponseDto.ResponseObj> insertDataset(
+      @RequestBody @Valid DatasetDto.AddReq addReq) {
+
+    return ApiResponseDto.ok(datasetService.insertDataset(addReq));
+  }
+
+  @Operation(summary = "파일 Path 조회", description = "파일 Path 조회")
+  @GetMapping("/files")
+  public ResponseEntity<Resource> getFile(@RequestParam UUID uuid, @RequestParam String pathType)
+      throws Exception {
+
+    String path = datasetService.getFilePathByUUIDPathType(uuid, pathType);
+    Path filePath = Paths.get(path);
+
+    Resource resource = new UrlResource(filePath.toUri());
+
+    return ResponseEntity.ok().contentType(MediaType.APPLICATION_OCTET_STREAM).body(resource);
+  }
 }

src/main/java/com/kamco/cd/training/dataset/dto/DatasetDto.java

@@ -13,6 +13,7 @@ import java.time.ZonedDateTime;
 import java.util.List;
 import java.util.UUID;
 import lombok.AllArgsConstructor;
+import lombok.Builder;
 import lombok.Getter;
 import lombok.NoArgsConstructor;
 import lombok.Setter;
@@ -132,9 +133,7 @@ public class DatasetDto {
     private int size = 20;
 
     public Pageable toPageable() {
-      // API에서는 1부터 시작하지만 내부적으로는 0부터 시작
-      int pageIndex = Math.max(0, page - 1);
-      return PageRequest.of(pageIndex, size, Sort.by(Sort.Direction.DESC, "createdDttm"));
+      return PageRequest.of(page, size, Sort.by(Sort.Direction.DESC, "createdDttm"));
     }
   }
 
@@ -308,4 +307,45 @@ public class DatasetDto {
     Long id;
     List<Long> ids;
   }
+
+  @Getter
+  @Setter
+  @NoArgsConstructor
+  @AllArgsConstructor
+  public static class AddReq {
+    private Long stage;
+    private String title;
+    private String fileName;
+    private String filePath;
+    private Long fileSize;
+    private String memo;
+  }
+
+  @Getter
+  @Setter
+  @NoArgsConstructor
+  @AllArgsConstructor
+  @Builder
+  public static class DatasetFileDto {
+    private String fileName;
+    private String filePath;
+  }
+
+  @Getter
+  @Setter
+  @NoArgsConstructor
+  @AllArgsConstructor
+  @Builder
+  public static class DatasetMngRegDto {
+    private String uid;
+    private String dataType;
+    private Integer compareYyyy;
+    private Integer targetYyyy;
+    private Long roundNo;
+    private String title;
+    private String memo;
+    private Long totalSize;
+    private Long totalObjectCount;
+    private String datasetPath;
+  }
 }

src/main/java/com/kamco/cd/training/dataset/dto/DatasetObjDto.java

@@ -1,11 +1,13 @@
 package com.kamco.cd.training.dataset.dto;
 
+import com.fasterxml.jackson.databind.JsonNode;
 import com.kamco.cd.training.common.enums.DetectionClassification;
 import com.kamco.cd.training.common.utils.interfaces.JsonFormatDttm;
 import io.swagger.v3.oas.annotations.media.Schema;
 import java.time.ZonedDateTime;
 import java.util.UUID;
 import lombok.AllArgsConstructor;
+import lombok.Builder;
 import lombok.Getter;
 import lombok.NoArgsConstructor;
 import lombok.Setter;
@@ -38,6 +40,7 @@ public class DatasetObjDto {
     private Long createdUid;
     private Boolean deleted;
     private UUID uuid;
+    private String geoJsonb;
 
     public Basic(
         Long objId,
@@ -54,7 +57,8 @@ public class DatasetObjDto {
         ZonedDateTime createdDttm,
         Long createdUid,
         Boolean deleted,
-        UUID uuid) {
+        UUID uuid,
+        String geoJsonb) {
       this.objId = objId;
       this.datasetUid = datasetUid;
       this.targetYyyy = targetYyyy;
@@ -70,6 +74,7 @@ public class DatasetObjDto {
       this.createdUid = createdUid;
       this.deleted = deleted;
       this.uuid = uuid;
+      this.geoJsonb = geoJsonb;
     }
   }
 
@@ -104,9 +109,7 @@ public class DatasetObjDto {
     private int size = 20;
 
     public Pageable toPageable() {
-      // API에서는 1부터 시작하지만 내부적으로는 0부터 시작
-      int pageIndex = Math.max(0, page - 1);
-      return PageRequest.of(pageIndex, size, Sort.by(Sort.Direction.DESC, "createdDttm"));
+      return PageRequest.of(page, size, Sort.by(Sort.Direction.DESC, "createdDttm"));
     }
   }
 
@@ -127,4 +130,23 @@ public class DatasetObjDto {
   public static class DatasetStorage {
     private String usableBytes;
   }
+
+  @Getter
+  @Setter
+  @NoArgsConstructor
+  @AllArgsConstructor
+  @Builder
+  public static class DatasetObjRegDto {
+    private Long datasetUid;
+    private Integer compareYyyy;
+    private String compareClassCd;
+    private Integer targetYyyy;
+    private String targetClassCd;
+    private String comparePath;
+    private String targetPath;
+    private String labelPath;
+    private String mapSheetNum;
+    private JsonNode geojson;
+    private String fileName;
+  }
 }

src/main/java/com/kamco/cd/training/dataset/service/DatasetService.java

@@ -1,18 +1,36 @@
 package com.kamco.cd.training.dataset.service;
 
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.kamco.cd.training.common.enums.LearnDataType;
 import com.kamco.cd.training.common.exception.CustomApiException;
 import com.kamco.cd.training.common.service.FormatStorage;
+import com.kamco.cd.training.common.utils.FIleChecker;
+import com.kamco.cd.training.config.api.ApiResponseDto.ApiResponseCode;
+import com.kamco.cd.training.config.api.ApiResponseDto.ResponseObj;
 import com.kamco.cd.training.dataset.dto.DatasetDto;
+import com.kamco.cd.training.dataset.dto.DatasetDto.AddReq;
+import com.kamco.cd.training.dataset.dto.DatasetDto.DatasetMngRegDto;
 import com.kamco.cd.training.dataset.dto.DatasetObjDto;
 import com.kamco.cd.training.dataset.dto.DatasetObjDto.DatasetClass;
+import com.kamco.cd.training.dataset.dto.DatasetObjDto.DatasetObjRegDto;
 import com.kamco.cd.training.dataset.dto.DatasetObjDto.DatasetStorage;
 import com.kamco.cd.training.dataset.dto.DatasetObjDto.SearchReq;
 import com.kamco.cd.training.postgres.core.DatasetCoreService;
+import jakarta.validation.Valid;
+import java.io.IOException;
+import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.UUID;
+import java.util.stream.Stream;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.data.domain.Page;
 import org.springframework.http.HttpStatus;
 import org.springframework.stereotype.Service;
@@ -21,11 +39,16 @@ import org.springframework.transaction.annotation.Transactional;
 @Slf4j
 @Service
 @RequiredArgsConstructor
-@Transactional(readOnly = true)
+@Transactional
 public class DatasetService {
 
   private final DatasetCoreService datasetCoreService;
 
+  @Value("${file.dataset-dir}")
+  private String datasetDir;
+
+  private static final List<String> LABEL_DIRS = List.of("label-json", "label", "input1", "input2");
+
   /**
    * 데이터셋 목록 조회
    *
@@ -132,4 +155,162 @@ public class DatasetService {
       throw new CustomApiException("NOT_FOUND", HttpStatus.NOT_FOUND);
     }
   }
+
+  public ResponseObj insertDataset(@Valid AddReq addReq) {
+
+    Long datasetUid = 0L; // master id 값, 등록하면서 가져올 예정
+
+    try {
+      // 압축 해제
+      FIleChecker.unzip(addReq.getFileName(), addReq.getFilePath());
+
+      // 해제한 폴더 읽어서 데이터 저장
+      List<Map<String, Object>> list =
+          getUnzipDatasetFiles(addReq.getFilePath() + addReq.getFileName().replace(".zip", ""));
+
+      int idx = 0;
+      for (Map<String, Object> map : list) {
+        String comparePath = (String) map.get("input1");
+        String targetPath = (String) map.get("input2");
+        String labelPath = (String) map.get("label");
+        JsonNode json = (JsonNode) map.get("label-json");
+
+        String fileName = Paths.get(comparePath).getFileName().toString();
+        String[] fileNameStr = fileName.split("_");
+        String compareYyyy = fileNameStr[1];
+        String targetYyyy = fileNameStr[2];
+        String mapSheetNum = fileNameStr[3];
+
+        if (idx == 0) {
+          String title = compareYyyy + "-" + targetYyyy;
+          String dataType = LearnDataType.PRODUCTION.getId(); // 만들어 넣는 건 다 제작
+          Long stage =
+              datasetCoreService.getDatasetMaxStage(
+                      Integer.parseInt(compareYyyy), Integer.parseInt(targetYyyy))
+                  + 1;
+          String uid = Paths.get(addReq.getFilePath()).getParent().getFileName().toString();
+
+          DatasetMngRegDto mngRegDto =
+              DatasetMngRegDto.builder()
+                  .uid(uid)
+                  .dataType(dataType)
+                  .compareYyyy(Integer.parseInt(compareYyyy))
+                  .targetYyyy(Integer.parseInt(targetYyyy))
+                  .title(title)
+                  .memo(addReq.getMemo())
+                  .roundNo(stage)
+                  .totalSize(addReq.getFileSize())
+                  .datasetPath(addReq.getFilePath())
+                  .build();
+
+          datasetUid = datasetCoreService.insertDatasetMngData(mngRegDto); // tb_dataset 에 insert
+        }
+
+        // datasetUid 로 obj 도 등록하기
+        // Json 갯수만큼 for문 돌려서 insert 해야 함
+        for (JsonNode feature : json.path("features")) {
+          JsonNode prop = feature.path("properties");
+          String compareClassCd = prop.path("before").asText(null);
+          String targetClassCd = prop.path("after").asText(null);
+
+          DatasetObjRegDto objRegDto =
+              DatasetObjRegDto.builder()
+                  .datasetUid(datasetUid)
+                  .compareYyyy(Integer.parseInt(compareYyyy))
+                  .compareClassCd(compareClassCd)
+                  .targetYyyy(Integer.parseInt(targetYyyy))
+                  .targetClassCd(targetClassCd)
+                  .comparePath(comparePath)
+                  .targetPath(targetPath)
+                  .labelPath(labelPath)
+                  .mapSheetNum(mapSheetNum)
+                  .geojson(feature)
+                  .fileName(fileName)
+                  .build();
+
+          datasetCoreService.insertDatasetObj(objRegDto);
+        }
+
+        idx++;
+      }
+
+    } catch (IOException e) {
+      log.error(e.getMessage());
+      return new ResponseObj(ApiResponseCode.INTERNAL_SERVER_ERROR, e.getMessage());
+    }
+    return new ResponseObj(ApiResponseCode.OK, "업로드 성공하였습니다.");
+  }
+
+  private List<Map<String, Object>> getUnzipDatasetFiles(String unzipRootPath) {
+
+    Path root = Paths.get(unzipRootPath).resolve("train");
+    Map<String, Map<String, Object>> grouped = new HashMap<>();
+    long total_file_cnt = 0;
+
+    for (String dirName : LABEL_DIRS) {
+      Path dir = root.resolve(dirName);
+
+      if (!Files.isDirectory(dir)) {
+        throw new IllegalStateException("폴더가 존재하지 않습니다 : " + dir);
+      }
+
+      // 파일 갯수는 같아서 비교년도일 때만 file_cnt 를 가지고 오기
+      //      if(dirName.equals("input1")){
+      //        try (Stream<Path> stream = Files.list(dir)) {
+      //          total_file_cnt = stream.filter(Files::isRegularFile).count();
+      //        } catch (IOException e) {
+      //          throw new RuntimeException(e);
+      //        }
+      //      }
+
+      try (Stream<Path> stream = Files.list(dir)) {
+        stream
+            .filter(Files::isRegularFile)
+            .forEach(
+                path -> {
+                  String fileName = path.getFileName().toString();
+                  String baseName = getBaseName(fileName);
+
+                  // baseName 기준 Map 생성
+                  Map<String, Object> data =
+                      grouped.computeIfAbsent(baseName, k -> new HashMap<>());
+
+                  // 공통 메타
+                  data.put("baseName", baseName);
+
+                  // 폴더별 처리
+                  if ("label-json".equals(dirName)) {
+                    // json 파일이면 파싱
+                    data.put("label-json", readJson(path));
+                  } else {
+                    // 나머지는 경로 or 파일명
+                    data.put(dirName, path.toAbsolutePath().toString());
+                  }
+                });
+
+      } catch (IOException e) {
+        throw new RuntimeException(e);
+      }
+    }
+
+    return new ArrayList<>(grouped.values());
+  }
+
+  private String getBaseName(String fileName) {
+    int idx = fileName.lastIndexOf('.');
+    return (idx > 0) ? fileName.substring(0, idx) : fileName;
+  }
+
+  private JsonNode readJson(Path jsonPath) {
+    try {
+      ObjectMapper mapper = new ObjectMapper();
+      return mapper.readTree(jsonPath.toFile());
+    } catch (IOException e) {
+      throw new RuntimeException("JSON 읽기 실패: " + jsonPath, e);
+    }
+  }
+
+  public String getFilePathByUUIDPathType(UUID uuid, String pathType) {
+    return datasetCoreService.getFilePathByUUIDPathType(uuid, pathType);
+  }
 }