tmp 하드링크 수정
This commit is contained in:
@@ -1,5 +1,6 @@
|
||||
package com.kamco.cd.training.train.service;
|
||||
|
||||
import com.kamco.cd.training.train.dto.ModelTrainLinkDto;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.*;
|
||||
import java.util.List;
|
||||
@@ -19,6 +20,92 @@ public class TmpDatasetService {
|
||||
@Value("${train.docker.basePath}")
|
||||
private String trainBaseDir;
|
||||
|
||||
/**
|
||||
* train, val, test 폴더별로 link
|
||||
*
|
||||
* @param uid 임시폴더 uuid
|
||||
* @param type train, val, test
|
||||
* @param links tif pull path
|
||||
* @return
|
||||
* @throws IOException
|
||||
*/
|
||||
public String buildTmpDatasetHardlink(String uid, String type, List<ModelTrainLinkDto> links)
|
||||
throws IOException {
|
||||
|
||||
if (links == null || links.isEmpty()) {
|
||||
throw new IOException("links is empty");
|
||||
}
|
||||
|
||||
Path tmp = Path.of(trainBaseDir, "tmp", uid);
|
||||
|
||||
long hardlinksMade = 0;
|
||||
|
||||
for (ModelTrainLinkDto dto : links) {
|
||||
|
||||
if (type == null) {
|
||||
log.warn("SKIP - trainType null: {}", dto);
|
||||
continue;
|
||||
}
|
||||
|
||||
// type별 디렉토리 생성
|
||||
Files.createDirectories(tmp.resolve(type).resolve("input1"));
|
||||
Files.createDirectories(tmp.resolve(type).resolve("input2"));
|
||||
Files.createDirectories(tmp.resolve(type).resolve("label"));
|
||||
|
||||
// comparePath → input1
|
||||
hardlinksMade += link(tmp, type, "input1", dto.getComparePath());
|
||||
|
||||
// targetPath → input2
|
||||
hardlinksMade += link(tmp, type, "input2", dto.getTargetPath());
|
||||
|
||||
// labelPath → label
|
||||
hardlinksMade += link(tmp, type, "label", dto.getLabelPath());
|
||||
}
|
||||
|
||||
if (hardlinksMade == 0) {
|
||||
throw new IOException("No hardlinks created.");
|
||||
}
|
||||
|
||||
log.info("tmp dataset created: {}, hardlinksMade={}", tmp, hardlinksMade);
|
||||
return uid;
|
||||
}
|
||||
|
||||
private long link(Path tmp, String type, String part, String fullPath) throws IOException {
|
||||
|
||||
if (fullPath == null || fullPath.isBlank()) return 0;
|
||||
|
||||
Path src = Path.of(fullPath);
|
||||
|
||||
if (!Files.isRegularFile(src)) {
|
||||
log.warn("SKIP (not file): {}", src);
|
||||
return 0;
|
||||
}
|
||||
|
||||
String fileName = src.getFileName().toString();
|
||||
Path dst = tmp.resolve(type).resolve(part).resolve(fileName);
|
||||
|
||||
// 충돌 시 덮어쓰기
|
||||
if (Files.exists(dst)) {
|
||||
Files.delete(dst);
|
||||
}
|
||||
|
||||
Files.createLink(dst, src);
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
private String safe(String s) {
|
||||
return (s == null || s.isBlank()) ? null : s.trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* request 전체 폴더 link
|
||||
*
|
||||
* @param uid
|
||||
* @param datasetUids
|
||||
* @return
|
||||
* @throws IOException
|
||||
*/
|
||||
public String buildTmpDatasetSymlink(String uid, List<String> datasetUids) throws IOException {
|
||||
|
||||
log.info("========== buildTmpDatasetHardlink START ==========");
|
||||
@@ -37,7 +124,7 @@ public class TmpDatasetService {
|
||||
|
||||
// tmp 디렉토리 준비
|
||||
for (String type : List.of("train", "val", "test")) {
|
||||
for (String part : List.of("input1", "input2", "label", "label-json")) {
|
||||
for (String part : List.of("input1", "input2", "label")) {
|
||||
Path dir = tmp.resolve(type).resolve(part);
|
||||
Files.createDirectories(dir);
|
||||
log.info("createDirectories: {}", dir);
|
||||
@@ -70,7 +157,7 @@ public class TmpDatasetService {
|
||||
log.info("---- dataset id={} srcRoot={} exists? {}", id, srcRoot, Files.isDirectory(srcRoot));
|
||||
|
||||
for (String type : List.of("train", "val", "test")) {
|
||||
for (String part : List.of("input1", "input2", "label", "label-json")) {
|
||||
for (String part : List.of("input1", "input2", "label")) {
|
||||
|
||||
Path srcDir = srcRoot.resolve(type).resolve(part);
|
||||
if (!Files.isDirectory(srcDir)) {
|
||||
|
||||
@@ -7,6 +7,7 @@ import com.kamco.cd.training.postgres.core.ModelTrainJobCoreService;
|
||||
import com.kamco.cd.training.postgres.core.ModelTrainMngCoreService;
|
||||
import com.kamco.cd.training.train.dto.ModelTrainJobDto;
|
||||
import com.kamco.cd.training.train.dto.ModelTrainJobQueuedEvent;
|
||||
import com.kamco.cd.training.train.dto.ModelTrainLinkDto;
|
||||
import com.kamco.cd.training.train.dto.TrainRunRequest;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
@@ -234,16 +235,30 @@ public class TrainJobService {
|
||||
UUID tmpUuid = UUID.randomUUID();
|
||||
String raw = tmpUuid.toString().toUpperCase().replace("-", "");
|
||||
|
||||
// MODELID 가져오기
|
||||
// model id 가져오기
|
||||
Long modelId = modelTrainMngCoreService.findModelIdByUuid(modelUuid);
|
||||
// model 에 연결된 dataset id 가져오기
|
||||
List<Long> datasetIds = modelTrainMngCoreService.findModelDatasetMapp(modelId);
|
||||
List<String> uids = modelTrainMngCoreService.findDatasetUid(datasetIds);
|
||||
|
||||
try {
|
||||
// 데이터셋 심볼링크 생성
|
||||
String pathUid = tmpDatasetService.buildTmpDatasetSymlink(raw, uids);
|
||||
// train path
|
||||
List<ModelTrainLinkDto> trainList = modelTrainMngCoreService.findDatasetTrainPath(modelId);
|
||||
// validation path
|
||||
List<ModelTrainLinkDto> valList = modelTrainMngCoreService.findDatasetValPath(modelId);
|
||||
// test path
|
||||
List<ModelTrainLinkDto> testList = modelTrainMngCoreService.findDatasetTestPath(modelId);
|
||||
|
||||
// train 데이터셋 심볼링크 생성
|
||||
tmpDatasetService.buildTmpDatasetHardlink(raw, "train", trainList);
|
||||
// val 데이터셋 심볼링크 생성
|
||||
tmpDatasetService.buildTmpDatasetHardlink(raw, "val", valList);
|
||||
// test 데이터셋 심볼링크 생성
|
||||
tmpDatasetService.buildTmpDatasetHardlink(raw, "test", testList);
|
||||
|
||||
ModelTrainMngDto.UpdateReq updateReq = new ModelTrainMngDto.UpdateReq();
|
||||
updateReq.setRequestPath(pathUid);
|
||||
updateReq.setRequestPath(raw);
|
||||
|
||||
// 학습모델을 수정한다.
|
||||
modelTrainMngCoreService.updateModelMaster(modelId, updateReq);
|
||||
} catch (IOException e) {
|
||||
|
||||
Reference in New Issue
Block a user