chunk업로드 공통, 모델관리 수정

src/main/java/com/kamco/cd/kamcoback/upload/UploadApiController.java

@@ -0,0 +1,144 @@
+package com.kamco.cd.kamcoback.upload;
+
+import com.kamco.cd.kamcoback.config.api.ApiResponseDto;
+import com.kamco.cd.kamcoback.upload.dto.UploadDto;
+import com.kamco.cd.kamcoback.upload.service.UploadService;
+import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.media.Content;
+import io.swagger.v3.oas.annotations.responses.ApiResponse;
+import io.swagger.v3.oas.annotations.responses.ApiResponses;
+import io.swagger.v3.oas.annotations.tags.Tag;
+import java.util.UUID;
+import lombok.RequiredArgsConstructor;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.http.MediaType;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.PutMapping;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RequestPart;
+import org.springframework.web.bind.annotation.RestController;
+import org.springframework.web.multipart.MultipartFile;
+
+@Tag(name = "파일 업로드", description = "대용량 파일 업로드 API")
+@RestController
+@RequestMapping("/api/upload")
+@RequiredArgsConstructor
+public class UploadApiController {
+
+  private final UploadService uploadService;
+
+  @Value("${file.sync-root-dir}")
+  private String syncRootDir;
+
+  @Value("${file.sync-tmp-dir}")
+  private String syncTmpDir;
+
+  @Value("${file.sync-file-extention}")
+  private String syncFileExtention;
+
+  @Value("${file.dataset-dir}")
+  private String datasetDir;
+
+  @Value("${file.dataset-tmp-dir}")
+  private String datasetTmpDir;
+
+  @Value("${file.model-dir}")
+  private String modelDir;
+
+  @Value("${file.model-tmp-dir}")
+  private String modelTmpDir;
+
+  /*
+    @Operation(summary = "데이터셋 대용량 업로드 세션 시작", description = "데이터셋 대용량 파일 업로드 세션을 시작합니다.")
+    @ApiResponses(
+        value = {
+          @ApiResponse(
+              responseCode = "201",
+              description = "세션 생성 성공",
+              content =
+                  @Content(
+                      mediaType = "application/json",
+                      schema = @Schema(implementation = UploadDto.InitRes.class))),
+          @ApiResponse(responseCode = "400", description = "잘못된 요청 데이터", content = @Content),
+          @ApiResponse(responseCode = "500", description = "서버 오류", content = @Content)
+        })
+    @PostMapping("/chunk-upload-init")
+    public ApiResponseDto<DmlReturn> initUpload(
+        @RequestBody @Valid UploadDto.InitReq initReq) {
+      return ApiResponseDto.createOK(uploadService.initUpload(initReq));
+    }
+  */
+
+  @Operation(summary = "데이터셋 대용량 파일 분할 전송", description = "데이터셋 파일 대용량 파일을 청크 단위로 전송합니다.")
+  @ApiResponses(
+      value = {
+        @ApiResponse(responseCode = "200", description = "청크 업로드 성공", content = @Content),
+        @ApiResponse(responseCode = "400", description = "잘못된 요청 데이터", content = @Content),
+        @ApiResponse(responseCode = "404", description = "업로드 세션을 찾을 수 없음", content = @Content),
+        @ApiResponse(responseCode = "500", description = "서버 오류", content = @Content)
+      })
+  @PostMapping(value = "/file-chunk-upload", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
+  public ApiResponseDto<UploadDto.UploadRes> fileChunkUpload(
+    @RequestParam("uuid") UUID uuid,
+    @RequestParam("fileName") String fileName,
+    @RequestParam("fileSize") long fileSize,
+    // @RequestParam("fileHash") String fileHash,
+    @RequestParam("chunkIndex") Integer chunkIndex,
+    @RequestParam("chunkTotalIndex") Integer chunkTotalIndex,
+    @RequestPart("chunkFile") MultipartFile chunkFile) {
+
+    String uploadDivi = "dataset";
+
+    UploadDto.UploadAddReq upAddReqDto = new UploadDto.UploadAddReq();
+    upAddReqDto.setDatasetId(0L);
+    upAddReqDto.setUuid(uuid);
+    upAddReqDto.setFileName(fileName);
+    upAddReqDto.setFileSize(fileSize);
+    upAddReqDto.setChunkIndex(chunkIndex);
+    upAddReqDto.setChunkTotalIndex(chunkTotalIndex);
+    upAddReqDto.setUploadDivi(uploadDivi);
+    upAddReqDto.setFinalPath(datasetDir);
+    upAddReqDto.setTempPath(datasetTmpDir);
+
+    System.out.println("uuid === "+ uuid);
+
+    return ApiResponseDto.ok(uploadService.uploadChunk(upAddReqDto, chunkFile));
+  }
+
+  @Operation(summary = "업로드 완료된 파일 병합", description = "업로드 완료 및 파일 병합을 요청합니다.")
+  @ApiResponses(
+      value = {
+        @ApiResponse(responseCode = "200", description = "업로드 완료 성공", content = @Content),
+        @ApiResponse(responseCode = "400", description = "잘못된 요청", content = @Content),
+        @ApiResponse(responseCode = "404", description = "업로드 세션을 찾을 수 없음", content = @Content),
+        @ApiResponse(responseCode = "500", description = "서버 오류", content = @Content)
+      })
+  @PutMapping("/chunk-upload-complete/{uuid}")
+  public ApiResponseDto<UploadDto.UploadRes> completeUpload(
+    @PathVariable UUID uuid) {
+    return ApiResponseDto.ok(uploadService.completeUpload(uuid));
+  }
+
+  /*
+  @Operation(summary = "업로드 상태 조회", description = "업로드 진행 상태를 조회합니다.")
+  @ApiResponses(
+      value = {
+        @ApiResponse(
+            responseCode = "200",
+            description = "조회 성공",
+            content =
+                @Content(
+                    mediaType = "application/json",
+                    schema = @Schema(implementation = UploadDto.Status.class))),
+        @ApiResponse(responseCode = "404", description = "업로드 세션을 찾을 수 없음", content = @Content),
+        @ApiResponse(responseCode = "500", description = "서버 오류", content = @Content)
+      })
+  @PostMapping("/status")
+  public ApiResponseDto<UploadDto.Status> getUploadStatus(
+      @RequestBody @Valid UploadDto.StatusReq statusReq) {
+    return ApiResponseDto.ok(uploadService.getUploadStatus(statusReq));
+  }
+  */
+}

src/main/java/com/kamco/cd/kamcoback/upload/dto/UploadDto.java

@@ -0,0 +1,224 @@
+package com.kamco.cd.kamcoback.upload.dto;
+
+import io.swagger.v3.oas.annotations.media.Schema;
+import jakarta.validation.constraints.NotBlank;
+import java.util.UUID;
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+
+public class UploadDto {
+
+  @Schema(name = "InitReq", description = "업로드(Chunk) 세션 초기화")
+  @Getter
+  @Setter
+  @NoArgsConstructor
+  @AllArgsConstructor
+  public static class InitReq {
+
+    @Schema(description = "파일명", example = "data.zip")
+    private String fileName;
+
+    @Schema(description = "파일 크기 (bytes)", example = "10737418240")
+    private Long fileSize;
+
+    @Schema(description = "총 청크 수", example = "100")
+    private Integer chunkTotalIndex;
+
+    /*
+    @Schema(
+        description = "파일해쉬",
+        example = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
+    private String fileHash;
+    */
+
+    @Schema(description = "업로드구분", example = "model")
+    private String uploadDivi;
+  }
+
+  @Schema(name = "UploadAddReq", description = "업로드 요청")
+  @Getter
+  @Setter
+  @NoArgsConstructor
+  @AllArgsConstructor
+  public static class UploadAddReq {
+    @Schema(description = "업로드 ID", example = "각데이터의 식별키")
+    private String uploadId;
+
+    @Schema(description = "데이터식별키", example = "129227333")
+    private Long datasetId;
+
+    @Schema(description = "파일명", example = "data.zip")
+    private String fileName;
+
+    @Schema(description = "파일 크기 (bytes)", example = "10737418240")
+    private Long fileSize;
+
+    @Schema(description = "파일명", example = "data.zip")
+    private String finalPath;
+
+    @Schema(description = "업로드구분", example = "dataset")
+    private String uploadDivi;
+
+    @Schema(description = "상태", example = "UPLOADING")
+    private String status;
+
+    @Schema(description = "임시저장경로")
+    private String tempPath;
+
+    @Schema(description = "업로드 청크 Index", example = "50")
+    private Integer chunkIndex;
+
+    @Schema(description = "총 청크 수", example = "100")
+    private Integer chunkTotalIndex;
+
+    @Schema(
+      description = "파일해쉬",
+      example = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
+    private String fileHash;
+
+    @Schema(description = "uuid", example = "303d4e24-1726-4272-bbc7-01ab85692b80")
+    private UUID uuid;
+  }
+
+  @Schema(name = "UploadCompleteReq", description = "업로드 완료 요청")
+  @Getter
+  @Setter
+  @NoArgsConstructor
+  @AllArgsConstructor
+  public static class CompleteReq {
+
+    @NotBlank(message = "업로드 ID는 필수입니다")
+    @Schema(description = "업로드 ID", example = "upload_20241218_123456_abc123")
+    private String uploadId;
+  }
+
+  @Schema(name = "UploadStatusReq", description = "업로드 상태 조회 요청")
+  @Getter
+  @Setter
+  @NoArgsConstructor
+  @AllArgsConstructor
+  public static class StatusReq {
+
+    @NotBlank(message = "업로드 ID는 필수입니다")
+    @Schema(description = "업로드 ID", example = "upload_20241218_123456_abc123")
+    private String uploadId;
+  }
+
+  @Schema(name = "UploadStatus", description = "업로드 상태 정보")
+  @Getter
+  @Setter
+  @NoArgsConstructor
+  @AllArgsConstructor
+  public static class Status {
+
+    @Schema(description = "업로드 ID", example = "upload_20241218_123456_abc123")
+    private String uploadId;
+
+    @Schema(description = "파일명", example = "data.zip")
+    private String fileName;
+
+    @Schema(description = "파일 크기 (bytes)", example = "10737418240")
+    private Long fileSize;
+
+    @Schema(description = "상태", example = "UPLOADING")
+    private String status;
+
+    @Schema(description = "총 청크 수", example = "100")
+    private Integer totalChunks;
+
+    @Schema(description = "업로드된 청크 수", example = "50")
+    private Integer uploadedChunks;
+
+    @Schema(description = "진행률 (%)", example = "50.0")
+    private Double progress;
+
+    @Schema(description = "에러 메시지", example = "")
+    private String errorMessage;
+  }
+
+  @Schema(name = "UploadAddReq", description = "업로드 요청")
+  @Getter
+  @Setter
+  @NoArgsConstructor
+  @AllArgsConstructor
+  public static class uploadDto {
+    @Schema(description = "업로드 ID", example = "각데이터의 식별키")
+    private String uploadId;
+
+    @Schema(description = "데이터식별키", example = "129227333")
+    private Long datasetId;
+
+    @Schema(description = "파일명", example = "data.zip")
+    private String fileName;
+
+
+    @Schema(description = "파일 크기 (bytes)", example = "10737418240")
+    private Long fileSize;
+
+    @Schema(description = "파일명", example = "data.zip")
+    private String finalPath;
+
+    @Schema(description = "업로드구분", example = "dataset")
+    private String uploadDivi;
+
+    @Schema(description = "상태", example = "UPLOADING")
+    private String status;
+
+    @Schema(description = "임시저장경로")
+    private String tempPath;
+
+    @Schema(description = "업로드 청크 Index", example = "50")
+    private Integer chunkIndex;
+
+    @Schema(description = "총 청크 Index", example = "100")
+    private Integer chunkTotalIndex;
+
+
+    @Schema(
+      description = "파일해쉬",
+      example = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
+    private String fileHash;
+
+    @Schema(description = "uuid")
+    private UUID uuid;
+
+  }
+
+  @Schema(name = "UploadRes", description = "업로드 수행 후 리턴")
+  @Getter
+  @Setter
+  @NoArgsConstructor
+  @AllArgsConstructor
+  public static class UploadRes {
+    private String res;
+    private String resMsg;
+    private UUID uuid;
+    private String filePath;
+    private String fileName;
+    private int chunkIndex;
+    private int chunkTotalIndex;
+
+    public double getUploadRate() {
+      if (this.chunkTotalIndex == 0) {
+        return 0.0;
+      }
+      return (double) (this.chunkIndex+1) / (this.chunkTotalIndex+1) * 100.0;
+    }
+
+  }
+
+
+
+  @Schema(name = "DmlReturn", description = "수행 후 리턴")
+  @Getter
+  @Setter
+  @NoArgsConstructor
+  @AllArgsConstructor
+  public static class DmlReturn {
+
+    private String flag;
+    private String message;
+  }
+}

src/main/java/com/kamco/cd/kamcoback/upload/service/UploadService.java

@@ -0,0 +1,252 @@
+package com.kamco.cd.kamcoback.upload.service;
+
+import com.kamco.cd.kamcoback.common.enums.FileUploadStatus;
+import com.kamco.cd.kamcoback.common.utils.FIleChecker;
+import com.kamco.cd.kamcoback.postgres.core.UploadSessionCoreService;
+import com.kamco.cd.kamcoback.upload.dto.UploadDto;
+import com.kamco.cd.kamcoback.upload.dto.UploadDto.DmlReturn;
+import java.io.IOException;
+import java.nio.channels.FileChannel;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+import java.util.UUID;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
+import org.springframework.web.multipart.MultipartFile;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+public class UploadService {
+
+  private final UploadSessionCoreService uploadSessionCoreService;
+
+  @Value("${file.sync-root-dir}")
+  private String syncRootDir;
+
+  @Value("${file.sync-tmp-dir}")
+  private String syncTmpDir;
+
+  @Value("${file.sync-file-extention}")
+  private String syncFileExtention;
+
+  @Value("${file.dataset-dir}")
+  private String datasetDir;
+
+  @Value("${file.dataset-tmp-dir}")
+  private String datasetTmpDir;
+
+  @Value("${file.model-dir}")
+  private String modelDir;
+
+  @Value("${file.model-tmp-dir}")
+  private String modelTmpDir;
+
+  @Transactional
+  public DmlReturn initUpload(UploadDto.InitReq initReq) {
+
+    return new DmlReturn("success", "UPLOAD CHUNK INIT");
+  }
+
+  @Transactional
+  public UploadDto.UploadRes uploadChunk(UploadDto.UploadAddReq upAddReqDto, MultipartFile file) {
+
+    UploadDto.UploadRes upRes = new UploadDto.UploadRes();
+
+    long datasetId = 0;
+    if( upAddReqDto.getDatasetId() != null )datasetId = upAddReqDto.getDatasetId();
+    String uploadId = System.currentTimeMillis()+"";
+    //UUID uuid = UUID.randomUUID();
+    UUID uuid = upAddReqDto.getUuid();
+    String tmpDataSetDir = upAddReqDto.getTempPath()+uuid;
+    String fianlDir = upAddReqDto.getFinalPath()+uuid;
+    String uploadDivi = upAddReqDto.getUploadDivi();
+    //String fileName = file.getOriginalFilename();
+    String fileName = upAddReqDto.getFileName();
+    Integer chunkIndex = upAddReqDto.getChunkIndex();
+    Integer chunkTotalIndex  = upAddReqDto.getChunkTotalIndex();
+    String status = FileUploadStatus.INIT.name();
+
+    upRes.setUuid(uuid);
+    upRes.setFilePath(fianlDir);
+    upRes.setFileName(fileName);
+
+    upAddReqDto.setUuid(uuid);
+    upAddReqDto.setUploadId(uploadId);
+    upAddReqDto.setStatus(status);
+    upAddReqDto.setFileName(fileName);
+    upAddReqDto.setTempPath(tmpDataSetDir);
+    upAddReqDto.setFinalPath(fianlDir);
+
+    //세션 신규,중복체크(초기화 포함)
+    UploadDto.uploadDto dto = this.checkUploadSession(upAddReqDto, upRes);
+    if( !upRes.getRes().equals("success") )return upRes;
+
+
+    status = FileUploadStatus.UPLOADING.name();
+    upAddReqDto.setStatus(status);
+
+    if( dto != null )
+    {
+      tmpDataSetDir =  dto.getTempPath();
+      fianlDir = dto.getFinalPath();
+    }
+
+    //폴더 생성 및 체크
+    if( ! checkChunkFoler(upRes, tmpDataSetDir, fianlDir) )return upRes;
+
+    //chunk저장하기
+    if( ! FIleChecker.multipartChunkSaveTo(file, tmpDataSetDir, chunkIndex ) )
+    {
+      upRes.setRes("fail");
+      upRes.setResMsg("chunkIndex:"+chunkIndex+" 업로드 애러");
+    }
+
+    if( chunkIndex == chunkTotalIndex ) {
+
+      upAddReqDto.setUploadId(dto.getUploadId());
+      upAddReqDto.setStatus(FileUploadStatus.DONE.name());
+      uploadSessionCoreService.updateUploadSessionStatus(upAddReqDto);
+
+
+      try {
+        this.mergeChunks(tmpDataSetDir, fianlDir, fileName, chunkTotalIndex);
+
+        upAddReqDto.setUploadId(dto.getUploadId());
+        upAddReqDto.setStatus("MERGED");
+        uploadSessionCoreService.updateUploadSessionStatus(upAddReqDto);
+
+      } catch (IOException e) {
+        //throw new RuntimeException(e);
+        upRes.setRes("fail");
+        upRes.setResMsg("파일Chunk 병합(merge) 애러");
+        return upRes;
+      }
+
+
+
+    }
+
+    return upRes;
+  }
+
+  @Transactional
+  public UploadDto.UploadRes completeUpload(UUID uuid) {
+
+    UploadDto.uploadDto dto = uploadSessionCoreService.findByUuid(uuid);
+    UploadDto.UploadRes upRes = new UploadDto.UploadRes();
+    upRes.setRes("success");
+    upRes.setResMsg("병합(merge) 정상처리되었습니다.");
+    upRes.setUuid(uuid);
+    upRes.setFilePath(dto.getFinalPath());
+    upRes.setFileName(dto.getFileName());
+    upRes.setChunkIndex(dto.getChunkIndex());
+    upRes.setChunkTotalIndex(dto.getChunkTotalIndex());
+
+    try {
+      this.mergeChunks(dto.getTempPath(), dto.getFinalPath(), dto.getFileName(), dto.getChunkTotalIndex());
+    } catch (IOException e) {
+
+      upRes.setRes("fail");
+      upRes.setResMsg("병합(merge) 애러");
+
+      return upRes;
+    }
+
+    return upRes;
+
+  }
+
+  public boolean checkChunkFoler(UploadDto.UploadRes upRes, String tmpDataSetDir, String fianlDir)
+  {
+
+    if( ! FIleChecker.mkDir(tmpDataSetDir) )
+    {
+      upRes.setRes("fail");
+      upRes.setRes("CHUNK 폴더 생성 ERROR");
+      return false;
+    }
+
+    if( ! FIleChecker.mkDir(fianlDir) )
+    {
+      upRes.setRes("fail");
+      upRes.setRes("업로드 완료 폴더 생성 ERROR");
+      return false;
+    }
+
+    return true;
+  }
+
+
+  public UploadDto.uploadDto checkUploadSession(UploadDto.UploadAddReq upAddReqDto, UploadDto.UploadRes upRes) {
+
+    upRes.setRes("success");
+    upRes.setResMsg("정상처리되었습니다.");
+
+    UploadDto.uploadDto dto = uploadSessionCoreService.findByUuid(upAddReqDto.getUuid());
+
+    if( upAddReqDto.getChunkIndex() == 0 ) {
+      if( dto != null )
+      {
+        upRes.setRes("duplicate");
+        upRes.setResMsg("이미 진행중인 업로드세션입니다.");
+        return dto;
+      }
+
+      upAddReqDto.setStatus("UPLOADING");
+      upRes.setUuid( upAddReqDto.getUuid() );
+      uploadSessionCoreService.createUploadSession(upAddReqDto);
+    }
+    else {
+      if( dto == null ){
+        upRes.setRes("nosession");
+        upRes.setResMsg("업로드 세션이 존재하지 않습니다.");
+        return dto;
+      }
+
+      upAddReqDto.setStatus("UPLOADING");
+      upAddReqDto.setUploadId(dto.getUploadId());
+      uploadSessionCoreService.updateUploadSessionStatus(upAddReqDto);
+    }
+
+    if( dto != null )upRes.setUuid( dto.getUuid() );
+
+    upRes.setChunkIndex(upAddReqDto.getChunkIndex());
+    upRes.setChunkTotalIndex(upAddReqDto.getChunkTotalIndex());
+
+
+    return dto;
+
+  }
+
+  public void mergeChunks(String tmpDir, String fianlDir, String fileName, int chunkTotalIndex) throws IOException {
+
+    Path outputPath = Paths.get(fianlDir, fileName);
+    try (FileChannel outChannel = FileChannel.open(outputPath, StandardOpenOption.CREATE, StandardOpenOption.WRITE)) {
+      for (int i = 0; i <= chunkTotalIndex; i++) {
+        Path chunkPath = Paths.get(tmpDir, i+"");
+
+        try (FileChannel inChannel = FileChannel.open(chunkPath, StandardOpenOption.READ)) {
+          long transferred = 0;
+          long size = inChannel.size();
+          while (transferred < size) {
+            transferred += inChannel.transferTo(transferred, size - transferred, outChannel);
+          }
+        }
+        // 병합 후 즉시 삭제하여 디스크 공간 확보
+        Files.delete(chunkPath);
+      }
+    }
+
+    //병합후 임시 폴더 삭제
+    FIleChecker.deleteFolder(tmpDir);
+
+  }
+
+
+}