운영환경처리

This commit is contained in:
2026-03-10 17:22:27 +09:00
parent fe6d37456d
commit 997e85c0cc

View File

@@ -38,7 +38,7 @@ public class TmpDatasetService {
Path tmp = Path.of(trainBaseDir, "tmp", uid); Path tmp = Path.of(trainBaseDir, "tmp", uid);
long hardlinksMade = 0; long linksMade = 0;
for (ModelTrainLinkDto dto : links) { for (ModelTrainLinkDto dto : links) {
@@ -54,23 +54,23 @@ public class TmpDatasetService {
Files.createDirectories(tmp.resolve(type).resolve("label-json")); Files.createDirectories(tmp.resolve(type).resolve("label-json"));
// comparePath → input1 // comparePath → input1
hardlinksMade += link(tmp, type, "input1", dto.getComparePath()); linksMade += link(tmp, type, "input1", dto.getComparePath());
// targetPath → input2 // targetPath → input2
hardlinksMade += link(tmp, type, "input2", dto.getTargetPath()); linksMade += link(tmp, type, "input2", dto.getTargetPath());
// labelPath → label // labelPath → label
hardlinksMade += link(tmp, type, "label", dto.getLabelPath()); linksMade += link(tmp, type, "label", dto.getLabelPath());
// geoJsonPath -> label-json // geoJsonPath -> label-json
hardlinksMade += link(tmp, type, "label-json", dto.getGeoJsonPath()); linksMade += link(tmp, type, "label-json", dto.getGeoJsonPath());
} }
if (hardlinksMade == 0) { if (linksMade == 0) {
throw new IOException("No hardlinks created."); throw new IOException("No symlinks created.");
} }
log.info("tmp dataset created: {}, hardlinksMade={}", tmp, hardlinksMade); log.info("tmp dataset created: {}, symlinksMade={}", tmp, linksMade);
} }
private long link(Path tmp, String type, String part, String fullPath) throws IOException { private long link(Path tmp, String type, String part, String fullPath) throws IOException {
@@ -88,24 +88,12 @@ public class TmpDatasetService {
Files.createDirectories(dst.getParent()); Files.createDirectories(dst.getParent());
if (Files.exists(dst)) { if (Files.exists(dst) || Files.isSymbolicLink(dst)) {
Files.delete(dst); Files.delete(dst);
} }
try { Files.createSymbolicLink(dst, src);
Files.createLink(dst, src); log.info("symlink created: {} -> {}", dst, src);
log.info("hardlink created: {} -> {}", dst, src);
} catch (FileSystemException e) {
if (e.getMessage() != null && e.getMessage().contains("Invalid cross-device link")) {
log.warn(
"Hardlink failed due to cross-device link. Fallback to symlink. src={}, dst={}",
src,
dst);
Files.createSymbolicLink(dst, src);
} else {
throw e;
}
}
return 1; return 1;
} }
@@ -124,7 +112,7 @@ public class TmpDatasetService {
*/ */
public String buildTmpDatasetSymlink(String uid, List<String> datasetUids) throws IOException { public String buildTmpDatasetSymlink(String uid, List<String> datasetUids) throws IOException {
log.info("========== buildTmpDatasetHardlink START =========="); log.info("========== buildTmpDatasetSymlink START ==========");
log.info("uid={}", uid); log.info("uid={}", uid);
log.info("datasetUids={}", datasetUids); log.info("datasetUids={}", datasetUids);
log.info("requestDir(raw)={}", requestDir); log.info("requestDir(raw)={}", requestDir);
@@ -136,7 +124,7 @@ public class TmpDatasetService {
log.info("BASE exists? {}", Files.isDirectory(BASE)); log.info("BASE exists? {}", Files.isDirectory(BASE));
log.info("tmp={}", tmp); log.info("tmp={}", tmp);
long noDir = 0, scannedDirs = 0, regularFiles = 0, hardlinksMade = 0; long noDir = 0, scannedDirs = 0, regularFiles = 0, symlinksMade = 0;
// tmp 디렉토리 준비 // tmp 디렉토리 준비
for (String type : List.of("train", "val", "test")) { for (String type : List.of("train", "val", "test")) {
@@ -147,26 +135,7 @@ public class TmpDatasetService {
} }
} }
// 하드링크는 "같은 파일시스템"에서만 가능하므로 BASE/tmp가 같은 FS인지 미리 확인(권장) // 심볼릭 링크는 파일시스템이 달라도 작동하므로 FileStore 체크 불필요
try {
var baseStore = Files.getFileStore(BASE);
var tmpStore = Files.getFileStore(tmp.getParent()); // BASE/tmp
if (!baseStore.name().equals(tmpStore.name()) || !baseStore.type().equals(tmpStore.type())) {
throw new IOException(
"Hardlink requires same filesystem. baseStore="
+ baseStore.name()
+ "("
+ baseStore.type()
+ "), tmpStore="
+ tmpStore.name()
+ "("
+ tmpStore.type()
+ ")");
}
} catch (Exception e) {
// FileStore 비교가 환경마다 애매할 수 있어서, 여기서는 경고만 주고 실제 createLink에서 최종 판단하게 둘 수도 있음.
log.warn("FileStore check skipped/failed (will rely on createLink): {}", e.toString());
}
for (String id : datasetUids) { for (String id : datasetUids) {
Path srcRoot = BASE.resolve(id); Path srcRoot = BASE.resolve(id);
@@ -204,13 +173,12 @@ public class TmpDatasetService {
} }
try { try {
// 하드링크 생성 (dst가 새 파일로 생기지만 inode는 f와 동일) // 심볼릭 링크 생성 (파일시스템이 달라도 작동)
Files.createLink(dst, f); Files.createSymbolicLink(dst, f);
hardlinksMade++; symlinksMade++;
log.debug("created hardlink: {} => {}", dst, f); log.debug("created symlink: {} => {}", dst, f);
} catch (IOException e) { } catch (IOException e) {
// 여기서 바로 실패시키면 “tmp는 만들었는데 내용은 0개” 같은 상태를 방지할 수 있음 log.error("FAILED create symlink: {} => {}", dst, f, e);
log.error("FAILED create hardlink: {} => {}", dst, f, e);
throw e; throw e;
} }
} }
@@ -219,9 +187,9 @@ public class TmpDatasetService {
} }
} }
if (hardlinksMade == 0) { if (symlinksMade == 0) {
throw new IOException( throw new IOException(
"No hardlinks created. regularFiles=" "No symlinks created. regularFiles="
+ regularFiles + regularFiles
+ ", scannedDirs=" + ", scannedDirs="
+ scannedDirs + scannedDirs
@@ -231,11 +199,11 @@ public class TmpDatasetService {
log.info("tmp dataset created: {}", tmp); log.info("tmp dataset created: {}", tmp);
log.info( log.info(
"summary: scannedDirs={}, noDir={}, regularFiles={}, hardlinksMade={}", "summary: scannedDirs={}, noDir={}, regularFiles={}, symlinksMade={}",
scannedDirs, scannedDirs,
noDir, noDir,
regularFiles, regularFiles,
hardlinksMade); symlinksMade);
return uid; return uid;
} }