중복 경고

Reviewed-on: #142

.gitignore

@@ -72,3 +72,9 @@ docker-compose.override.yml
 *.swo
 *~
 !/CLAUDE.md
+
+### SSL Certificates ###
+nginx/ssl/
+*.crt
+*.key
+*.pem

src/main/java/com/kamco/cd/training/common/utils/FIleChecker.java

@@ -3,6 +3,7 @@ package com.kamco.cd.training.common.utils;
 import static java.lang.String.CASE_INSENSITIVE_ORDER;
 
 import com.jcraft.jsch.ChannelExec;
+import com.jcraft.jsch.ChannelSftp;
 import com.jcraft.jsch.JSch;
 import com.jcraft.jsch.Session;
 import com.kamco.cd.training.common.exception.CustomApiException;
@@ -719,26 +720,18 @@ public class FIleChecker {
   public static void unzip(String fileName, String destDirectory) throws IOException {
     String zipFilePath = destDirectory + File.separator + fileName;
 
-    log.info("fileName : {}", fileName);
-    log.info("destDirectory : {}", destDirectory);
-    log.info("zipFilePath : {}", zipFilePath);
     // zip 이름으로 폴더 생성 (확장자 제거)
     String folderName =
         fileName.endsWith(".zip") ? fileName.substring(0, fileName.length() - 4) : fileName;
-    log.info("folderName : {}", folderName);
 
     File destDir = new File(destDirectory, folderName);
-    log.info("destDir : {}", destDir);
 
     // 동일 폴더가 이미 있으면 삭제
-    log.info("111 destDir.exists() : {}", destDir.exists());
     if (destDir.exists()) {
       deleteDirectoryRecursively(destDir.toPath());
     }
 
-    log.info("222 destDir.exists() : {}", destDir.exists());
     if (!destDir.exists()) {
-      log.info("mkdirs : {}", destDir.exists());
       destDir.mkdirs();
     }
 
@@ -794,6 +787,92 @@ public class FIleChecker {
     return destFile;
   }
 
+  public static void uploadTo86(Path localFile) {
+
+    String host = "192.168.2.86";
+    int port = 22;
+    String username = "kcomu";
+    String password = "Kamco2025!";
+
+    String remoteDir = "/home/kcomu/data/request";
+
+    Session session = null;
+    ChannelSftp channel = null;
+
+    try {
+      JSch jsch = new JSch();
+
+      session = jsch.getSession(username, host, port);
+      session.setPassword(password);
+
+      Properties config = new Properties();
+      config.put("StrictHostKeyChecking", "no");
+      session.setConfig(config);
+
+      session.connect(10_000);
+
+      channel = (ChannelSftp) session.openChannel("sftp");
+      channel.connect(10_000);
+
+      // 목적지 디렉토리 이동
+      channel.cd(remoteDir);
+
+      // 업로드
+      channel.put(localFile.toString(), localFile.getFileName().toString());
+
+    } catch (Exception e) {
+      throw new RuntimeException("SFTP upload failed", e);
+    } finally {
+      if (channel != null) channel.disconnect();
+      if (session != null) session.disconnect();
+    }
+  }
+
+  public static void unzipOn86Server(String zipPath, String targetDir) {
+
+    String host = "192.168.2.86";
+    String user = "kcomu";
+    String password = "Kamco2025!";
+
+    Session session = null;
+    ChannelExec channel = null;
+
+    try {
+      JSch jsch = new JSch();
+
+      session = jsch.getSession(user, host, 22);
+      session.setPassword(password);
+
+      Properties config = new Properties();
+      config.put("StrictHostKeyChecking", "no");
+      session.setConfig(config);
+
+      session.connect(10_000);
+
+      String command = "unzip -o " + zipPath + " -d " + targetDir;
+
+      channel = (ChannelExec) session.openChannel("exec");
+      channel.setCommand(command);
+      channel.setErrStream(System.err);
+
+      InputStream in = channel.getInputStream();
+      channel.connect();
+
+      // 출력 읽기(선택)
+      try (BufferedReader br = new BufferedReader(new InputStreamReader(in))) {
+        while (br.readLine() != null) {
+          // 필요하면 로그
+        }
+      }
+
+    } catch (Exception e) {
+      throw new RuntimeException(e);
+    } finally {
+      if (channel != null) channel.disconnect();
+      if (session != null) session.disconnect();
+    }
+  }
+
   public static List<String> execCommandAndReadLines(String command) {
 
     List<String> result = new ArrayList<>();

src/main/java/com/kamco/cd/training/dataset/service/DatasetService.java

@@ -417,7 +417,6 @@ public class DatasetService {
   }
 
   private String escape(String path) {
-    // 쉘 커맨드에서 안전하게 사용할 수 있도록 문자열을 작은따옴표로 감싸면서, 내부의 작은따옴표를 이스케이프 처리
     return "'" + path.replace("'", "'\"'\"'") + "'";
   }
 

src/main/java/com/kamco/cd/training/train/service/TmpDatasetService.java

@@ -20,187 +20,71 @@ public class TmpDatasetService {
   @Value("${train.docker.basePath}")
   private String trainBaseDir;
 
-  /**
+  private final DataSetCountersService dataSetCountersService;
-   * train, val, test 폴더별로 link
-   *
-   * @param uid 임시폴더 uuid
-   * @param type train, val, test
-   * @param links tif pull path
-   * @return
-   * @throws IOException
-   */
-  public void buildTmpDatasetHardlink(String uid, String type, List<ModelTrainLinkDto> links)
-      throws IOException {
-
-    if (links == null || links.isEmpty()) {
-      throw new IOException("links is empty");
-    }
-
-    Path tmp = Path.of(trainBaseDir, "tmp", uid);
-
-    long hardlinksMade = 0;
-
-    for (ModelTrainLinkDto dto : links) {
-
-      if (type == null) {
-        log.warn("SKIP - trainType null: {}", dto);
-        continue;
-      }
-
-      // type별 디렉토리 생성
-      Files.createDirectories(tmp.resolve(type).resolve("input1"));
-      Files.createDirectories(tmp.resolve(type).resolve("input2"));
-      Files.createDirectories(tmp.resolve(type).resolve("label"));
-      Files.createDirectories(tmp.resolve(type).resolve("label-json"));
-
-      // comparePath → input1
-      hardlinksMade += link(tmp, type, "input1", dto.getComparePath());
-
-      // targetPath → input2
-      hardlinksMade += link(tmp, type, "input2", dto.getTargetPath());
-
-      // labelPath → label
-      hardlinksMade += link(tmp, type, "label", dto.getLabelPath());
-
-      // geoJsonPath -> label-json
-      hardlinksMade += link(tmp, type, "label-json", dto.getGeoJsonPath());
-    }
-
-    if (hardlinksMade == 0) {
-      throw new IOException("No hardlinks created.");
-    }
-
-    log.info("tmp dataset created: {}, hardlinksMade={}", tmp, hardlinksMade);
-  }
-
-  private long link(Path tmp, String type, String part, String fullPath) throws IOException {
-
-    if (fullPath == null || fullPath.isBlank()) return 0;
-
-    Path src = Path.of(fullPath);
-
-    if (!Files.isRegularFile(src)) {
-      log.warn("SKIP (not file): {}", src);
-      return 0;
-    }
-
-    String fileName = src.getFileName().toString();
-    Path dst = tmp.resolve(type).resolve(part).resolve(fileName);
-
-    // 충돌 시 덮어쓰기
-    if (Files.exists(dst)) {
-      Files.delete(dst);
-    }
-
-    Files.createLink(dst, src);
-
-    return 1;
-  }
-
-  private String safe(String s) {
-    return (s == null || s.isBlank()) ? null : s.trim();
-  }
 
   /**
-   * request 전체 폴더 link
+   * 다른 데이터셋 파일과 이름이 겹치면 그 파일은 skip함
    *
    * @param uid
-   * @param datasetUids
+   * @param type
+   * @param links
    * @return
    * @throws IOException
    */
-  public String buildTmpDatasetSymlink(String uid, List<String> datasetUids) throws IOException {
+  public String buildTmpDatasetSymlink(String uid, String type, List<ModelTrainLinkDto> links)
+      throws IOException {
 
-    log.info("========== buildTmpDatasetHardlink START ==========");
+    if (uid == null || uid.isBlank()) throw new IOException("uid is empty");
-    log.info("uid={}", uid);
+    if (type == null || type.isBlank()) throw new IOException("type is empty");
-    log.info("datasetUids={}", datasetUids);
+    if (links == null || links.isEmpty()) throw new IOException("links is empty");
-    log.info("requestDir(raw)={}", requestDir);
+
+    log.info("========== buildTmpDatasetHardlink MERGE START ==========");
+    log.info("uid={}, type={}, links.size={}", uid, type, links.size());
 
     Path BASE = toPath(requestDir);
     Path tmp = Path.of(trainBaseDir, "tmp", uid);
 
-    log.info("BASE={}", BASE);
+    long hardlinksMade = 0;
-    log.info("BASE exists? {}", Files.isDirectory(BASE));
+    long skippedCollision = 0;
-    log.info("tmp={}", tmp);
+    long noDir = 0;
 
-    long noDir = 0, scannedDirs = 0, regularFiles = 0, hardlinksMade = 0;
+    // tmp/<type>/<part> 준비
-
-    // tmp 디렉토리 준비
-    for (String type : List.of("train", "val", "test")) {
     for (String part : List.of("input1", "input2", "label", "label-json")) {
-        Path dir = tmp.resolve(type).resolve(part);
+      Files.createDirectories(tmp.resolve(type).resolve(part));
-        Files.createDirectories(dir);
-        log.info("createDirectories: {}", dir);
-      }
     }
 
-    // 하드링크는 "같은 파일시스템"에서만 가능하므로 BASE/tmp가 같은 FS인지 미리 확인(권장)
+    for (ModelTrainLinkDto dto : links) {
-    try {
+      String datasetUid = safe(dto.getDatasetUid());
-      var baseStore = Files.getFileStore(BASE);
+      if (datasetUid == null) {
-      var tmpStore = Files.getFileStore(tmp.getParent()); // BASE/tmp
+        log.warn("SKIP dto (datasetUid null): {}", dto);
-      if (!baseStore.name().equals(tmpStore.name()) || !baseStore.type().equals(tmpStore.type())) {
+        continue;
-        throw new IOException(
-            "Hardlink requires same filesystem. baseStore="
-                + baseStore.name()
-                + "("
-                + baseStore.type()
-                + "), tmpStore="
-                + tmpStore.name()
-                + "("
-                + tmpStore.type()
-                + ")");
-      }
-    } catch (Exception e) {
-      // FileStore 비교가 환경마다 애매할 수 있어서, 여기서는 경고만 주고 실제 createLink에서 최종 판단하게 둘 수도 있음.
-      log.warn("FileStore check skipped/failed (will rely on createLink): {}", e.toString());
       }
 
-    for (String id : datasetUids) {
+      Path srcRoot = BASE.resolve(datasetUid);
-      Path srcRoot = BASE.resolve(id);
-      log.info("---- dataset id={} srcRoot={} exists? {}", id, srcRoot, Files.isDirectory(srcRoot));
 
-      for (String type : List.of("train", "val", "test")) {
       for (String part : List.of("input1", "input2", "label", "label-json")) {
 
         Path srcDir = srcRoot.resolve(type).resolve(part);
         if (!Files.isDirectory(srcDir)) {
-            log.warn("SKIP (not directory): {}", srcDir);
           noDir++;
           continue;
         }
 
-          scannedDirs++;
+        // 하위폴더까지 전부
-          log.info("SCAN dir={}", srcDir);
+        try (var walk = Files.walk(srcDir)) {
+          for (Path f : walk.filter(Files::isRegularFile).toList()) {
 
-          try (DirectoryStream<Path> stream = Files.newDirectoryStream(srcDir)) {
+            String fileName = f.getFileName().toString();
-            for (Path f : stream) {
+            Path dst = tmp.resolve(type).resolve(part).resolve(fileName);
-              if (!Files.isRegularFile(f)) {
+
-                log.debug("skip non-regular file: {}", f);
+            // 이름 유지 + 충돌은 skip
+            if (Files.exists(dst)) {
+              skippedCollision++;
               continue;
             }
 
-              regularFiles++;
-
-              String dstName = id + "__" + f.getFileName();
-              Path dst = tmp.resolve(type).resolve(part).resolve(dstName);
-
-              // dst가 남아있으면 삭제(심볼릭링크든 파일이든)
-              if (Files.exists(dst) || Files.isSymbolicLink(dst)) {
-                Files.delete(dst);
-                log.debug("deleted existing: {}", dst);
-              }
-
-              try {
-                // 하드링크 생성 (dst가 새 파일로 생기지만 inode는 f와 동일)
             Files.createLink(dst, f);
             hardlinksMade++;
-                log.debug("created hardlink: {} => {}", dst, f);
-              } catch (IOException e) {
-                // 여기서 바로 실패시키면 “tmp는 만들었는데 내용은 0개” 같은 상태를 방지할 수 있음
-                log.error("FAILED create hardlink: {} => {}", dst, f, e);
-                throw e;
-              }
-            }
           }
         }
       }
@@ -208,29 +92,35 @@ public class TmpDatasetService {
 
     if (hardlinksMade == 0) {
       throw new IOException(
-          "No hardlinks created. regularFiles="
+          "No hardlinks created. noDir=" + noDir + ", skippedCollision=" + skippedCollision);
-              + regularFiles
-              + ", scannedDirs="
-              + scannedDirs
-              + ", noDir="
-              + noDir);
     }
 
-    log.info("tmp dataset created: {}", tmp);
     log.info(
-        "summary: scannedDirs={}, noDir={}, regularFiles={}, hardlinksMade={}",
+        "tmp dataset merged: {} (type={}), hardlinksMade={}, skippedCollision={}, noDir={}",
-        scannedDirs,
+        tmp,
-        noDir,
+        type,
-        regularFiles,
+        hardlinksMade,
-        hardlinksMade);
+        skippedCollision,
+        noDir);
 
     return uid;
   }
 
   private static Path toPath(String p) {
-    if (p.startsWith("~/")) {
+    if (p == null || p.isBlank()) {
-      return Paths.get(System.getProperty("user.home")).resolve(p.substring(2)).normalize();
+      throw new IllegalArgumentException("path is null or blank");
     }
-    return Paths.get(p).toAbsolutePath().normalize();
+    String trimmed = p.trim();
+    if (trimmed.startsWith("~/")) {
+      return Paths.get(System.getProperty("user.home"))
+          .resolve(trimmed.substring(2))
+          .toAbsolutePath()
+          .normalize();
+    }
+    return Paths.get(trimmed).toAbsolutePath().normalize();
+  }
+
+  private static String safe(String s) {
+    return (s == null || s.isBlank()) ? null : s.trim();
   }
 }

src/main/java/com/kamco/cd/training/train/service/TrainJobService.java

@@ -266,8 +266,6 @@ public class TrainJobService {
     List<String> uids = modelTrainMngCoreService.findDatasetUid(datasetIds);
 
     try {
-      // 데이터셋 심볼링크 생성
-      //      String pathUid = tmpDatasetService.buildTmpDatasetSymlink(raw, uids);
       // train path
       List<ModelTrainLinkDto> trainList = modelTrainMngCoreService.findDatasetTrainPath(modelId);
       // validation path
@@ -276,12 +274,13 @@ public class TrainJobService {
       List<ModelTrainLinkDto> testList = modelTrainMngCoreService.findDatasetTestPath(modelId);
 
       // train 데이터셋 심볼링크 생성
-      tmpDatasetService.buildTmpDatasetHardlink(raw, "train", trainList);
+      tmpDatasetService.buildTmpDatasetSymlink(raw, "train", trainList);
       // val 데이터셋 심볼링크 생성
-      tmpDatasetService.buildTmpDatasetHardlink(raw, "val", valList);
+      tmpDatasetService.buildTmpDatasetSymlink(raw, "val", valList);
       // test 데이터셋 심볼링크 생성
-      tmpDatasetService.buildTmpDatasetHardlink(raw, "test", testList);
+      tmpDatasetService.buildTmpDatasetSymlink(raw, "test", testList);
-
+      // 카운트 로그
+      dataSetCounters.getCount(modelId);
       ModelTrainMngDto.UpdateReq updateReq = new ModelTrainMngDto.UpdateReq();
       updateReq.setRequestPath(raw);