diff --git a/src/main/java/com/kamco/cd/training/train/service/TmpDatasetService.java b/src/main/java/com/kamco/cd/training/train/service/TmpDatasetService.java index 27b59ae..978fe2b 100644 --- a/src/main/java/com/kamco/cd/training/train/service/TmpDatasetService.java +++ b/src/main/java/com/kamco/cd/training/train/service/TmpDatasetService.java @@ -20,71 +20,187 @@ public class TmpDatasetService { @Value("${train.docker.basePath}") private String trainBaseDir; - private final DataSetCountersService dataSetCountersService; - /** - * 다른 데이터셋 파일과 이름이 겹치면 그 파일은 skip함 + * train, val, test 폴더별로 link * - * @param uid - * @param type - * @param links + * @param uid 임시폴더 uuid + * @param type train, val, test + * @param links tif pull path * @return * @throws IOException */ - public String buildTmpDatasetSymlink(String uid, String type, List links) + public void buildTmpDatasetHardlink(String uid, String type, List links) throws IOException { - if (uid == null || uid.isBlank()) throw new IOException("uid is empty"); - if (type == null || type.isBlank()) throw new IOException("type is empty"); - if (links == null || links.isEmpty()) throw new IOException("links is empty"); + if (links == null || links.isEmpty()) { + throw new IOException("links is empty"); + } - log.info("========== buildTmpDatasetHardlink MERGE START =========="); - log.info("uid={}, type={}, links.size={}", uid, type, links.size()); + Path tmp = Path.of(trainBaseDir, "tmp", uid); + + long hardlinksMade = 0; + + for (ModelTrainLinkDto dto : links) { + + if (type == null) { + log.warn("SKIP - trainType null: {}", dto); + continue; + } + + // type별 디렉토리 생성 + Files.createDirectories(tmp.resolve(type).resolve("input1")); + Files.createDirectories(tmp.resolve(type).resolve("input2")); + Files.createDirectories(tmp.resolve(type).resolve("label")); + Files.createDirectories(tmp.resolve(type).resolve("label-json")); + + // comparePath → input1 + hardlinksMade += link(tmp, type, "input1", dto.getComparePath()); + + // targetPath → input2 + hardlinksMade += link(tmp, type, "input2", dto.getTargetPath()); + + // labelPath → label + hardlinksMade += link(tmp, type, "label", dto.getLabelPath()); + + // geoJsonPath -> label-json + hardlinksMade += link(tmp, type, "label-json", dto.getGeoJsonPath()); + } + + if (hardlinksMade == 0) { + throw new IOException("No hardlinks created."); + } + + log.info("tmp dataset created: {}, hardlinksMade={}", tmp, hardlinksMade); + } + + private long link(Path tmp, String type, String part, String fullPath) throws IOException { + + if (fullPath == null || fullPath.isBlank()) return 0; + + Path src = Path.of(fullPath); + + if (!Files.isRegularFile(src)) { + log.warn("SKIP (not file): {}", src); + return 0; + } + + String fileName = src.getFileName().toString(); + Path dst = tmp.resolve(type).resolve(part).resolve(fileName); + + // 충돌 시 덮어쓰기 + if (Files.exists(dst)) { + Files.delete(dst); + } + + Files.createLink(dst, src); + + return 1; + } + + private String safe(String s) { + return (s == null || s.isBlank()) ? null : s.trim(); + } + + /** + * request 전체 폴더 link + * + * @param uid + * @param datasetUids + * @return + * @throws IOException + */ + public String buildTmpDatasetSymlink(String uid, List datasetUids) throws IOException { + + log.info("========== buildTmpDatasetHardlink START =========="); + log.info("uid={}", uid); + log.info("datasetUids={}", datasetUids); + log.info("requestDir(raw)={}", requestDir); Path BASE = toPath(requestDir); Path tmp = Path.of(trainBaseDir, "tmp", uid); - long hardlinksMade = 0; - long skippedCollision = 0; - long noDir = 0; + log.info("BASE={}", BASE); + log.info("BASE exists? {}", Files.isDirectory(BASE)); + log.info("tmp={}", tmp); - // tmp// 준비 - for (String part : List.of("input1", "input2", "label", "label-json")) { - Files.createDirectories(tmp.resolve(type).resolve(part)); + long noDir = 0, scannedDirs = 0, regularFiles = 0, hardlinksMade = 0; + + // tmp 디렉토리 준비 + for (String type : List.of("train", "val", "test")) { + for (String part : List.of("input1", "input2", "label", "label-json")) { + Path dir = tmp.resolve(type).resolve(part); + Files.createDirectories(dir); + log.info("createDirectories: {}", dir); + } } - for (ModelTrainLinkDto dto : links) { - String datasetUid = safe(dto.getDatasetUid()); - if (datasetUid == null) { - log.warn("SKIP dto (datasetUid null): {}", dto); - continue; + // 하드링크는 "같은 파일시스템"에서만 가능하므로 BASE/tmp가 같은 FS인지 미리 확인(권장) + try { + var baseStore = Files.getFileStore(BASE); + var tmpStore = Files.getFileStore(tmp.getParent()); // BASE/tmp + if (!baseStore.name().equals(tmpStore.name()) || !baseStore.type().equals(tmpStore.type())) { + throw new IOException( + "Hardlink requires same filesystem. baseStore=" + + baseStore.name() + + "(" + + baseStore.type() + + "), tmpStore=" + + tmpStore.name() + + "(" + + tmpStore.type() + + ")"); } + } catch (Exception e) { + // FileStore 비교가 환경마다 애매할 수 있어서, 여기서는 경고만 주고 실제 createLink에서 최종 판단하게 둘 수도 있음. + log.warn("FileStore check skipped/failed (will rely on createLink): {}", e.toString()); + } - Path srcRoot = BASE.resolve(datasetUid); + for (String id : datasetUids) { + Path srcRoot = BASE.resolve(id); + log.info("---- dataset id={} srcRoot={} exists? {}", id, srcRoot, Files.isDirectory(srcRoot)); - for (String part : List.of("input1", "input2", "label", "label-json")) { + for (String type : List.of("train", "val", "test")) { + for (String part : List.of("input1", "input2", "label", "label-json")) { - Path srcDir = srcRoot.resolve(type).resolve(part); - if (!Files.isDirectory(srcDir)) { - noDir++; - continue; - } + Path srcDir = srcRoot.resolve(type).resolve(part); + if (!Files.isDirectory(srcDir)) { + log.warn("SKIP (not directory): {}", srcDir); + noDir++; + continue; + } - // 하위폴더까지 전부 - try (var walk = Files.walk(srcDir)) { - for (Path f : walk.filter(Files::isRegularFile).toList()) { + scannedDirs++; + log.info("SCAN dir={}", srcDir); - String fileName = f.getFileName().toString(); - Path dst = tmp.resolve(type).resolve(part).resolve(fileName); + try (DirectoryStream stream = Files.newDirectoryStream(srcDir)) { + for (Path f : stream) { + if (!Files.isRegularFile(f)) { + log.debug("skip non-regular file: {}", f); + continue; + } - // 이름 유지 + 충돌은 skip - if (Files.exists(dst)) { - skippedCollision++; - continue; + regularFiles++; + + String dstName = id + "__" + f.getFileName(); + Path dst = tmp.resolve(type).resolve(part).resolve(dstName); + + // dst가 남아있으면 삭제(심볼릭링크든 파일이든) + if (Files.exists(dst) || Files.isSymbolicLink(dst)) { + Files.delete(dst); + log.debug("deleted existing: {}", dst); + } + + try { + // 하드링크 생성 (dst가 새 파일로 생기지만 inode는 f와 동일) + Files.createLink(dst, f); + hardlinksMade++; + log.debug("created hardlink: {} => {}", dst, f); + } catch (IOException e) { + // 여기서 바로 실패시키면 “tmp는 만들었는데 내용은 0개” 같은 상태를 방지할 수 있음 + log.error("FAILED create hardlink: {} => {}", dst, f, e); + throw e; + } } - - Files.createLink(dst, f); - hardlinksMade++; } } } @@ -92,35 +208,29 @@ public class TmpDatasetService { if (hardlinksMade == 0) { throw new IOException( - "No hardlinks created. noDir=" + noDir + ", skippedCollision=" + skippedCollision); + "No hardlinks created. regularFiles=" + + regularFiles + + ", scannedDirs=" + + scannedDirs + + ", noDir=" + + noDir); } + log.info("tmp dataset created: {}", tmp); log.info( - "tmp dataset merged: {} (type={}), hardlinksMade={}, skippedCollision={}, noDir={}", - tmp, - type, - hardlinksMade, - skippedCollision, - noDir); + "summary: scannedDirs={}, noDir={}, regularFiles={}, hardlinksMade={}", + scannedDirs, + noDir, + regularFiles, + hardlinksMade); return uid; } private static Path toPath(String p) { - if (p == null || p.isBlank()) { - throw new IllegalArgumentException("path is null or blank"); + if (p.startsWith("~/")) { + return Paths.get(System.getProperty("user.home")).resolve(p.substring(2)).normalize(); } - String trimmed = p.trim(); - if (trimmed.startsWith("~/")) { - return Paths.get(System.getProperty("user.home")) - .resolve(trimmed.substring(2)) - .toAbsolutePath() - .normalize(); - } - return Paths.get(trimmed).toAbsolutePath().normalize(); - } - - private static String safe(String s) { - return (s == null || s.isBlank()) ? null : s.trim(); + return Paths.get(p).toAbsolutePath().normalize(); } } diff --git a/src/main/java/com/kamco/cd/training/train/service/TrainJobService.java b/src/main/java/com/kamco/cd/training/train/service/TrainJobService.java index 5193cda..d2dd5a6 100644 --- a/src/main/java/com/kamco/cd/training/train/service/TrainJobService.java +++ b/src/main/java/com/kamco/cd/training/train/service/TrainJobService.java @@ -266,6 +266,8 @@ public class TrainJobService { List uids = modelTrainMngCoreService.findDatasetUid(datasetIds); try { + // 데이터셋 심볼링크 생성 + // String pathUid = tmpDatasetService.buildTmpDatasetSymlink(raw, uids); // train path List trainList = modelTrainMngCoreService.findDatasetTrainPath(modelId); // validation path @@ -274,13 +276,12 @@ public class TrainJobService { List testList = modelTrainMngCoreService.findDatasetTestPath(modelId); // train 데이터셋 심볼링크 생성 - tmpDatasetService.buildTmpDatasetSymlink(raw, "train", trainList); + tmpDatasetService.buildTmpDatasetHardlink(raw, "train", trainList); // val 데이터셋 심볼링크 생성 - tmpDatasetService.buildTmpDatasetSymlink(raw, "val", valList); + tmpDatasetService.buildTmpDatasetHardlink(raw, "val", valList); // test 데이터셋 심볼링크 생성 - tmpDatasetService.buildTmpDatasetSymlink(raw, "test", testList); - // 카운트 로그 - dataSetCounters.getCount(modelId); + tmpDatasetService.buildTmpDatasetHardlink(raw, "test", testList); + ModelTrainMngDto.UpdateReq updateReq = new ModelTrainMngDto.UpdateReq(); updateReq.setRequestPath(raw);