하드링크 로그 추가
This commit is contained in:
@@ -185,7 +185,7 @@ public class TrainApiController {
|
|||||||
})
|
})
|
||||||
@PostMapping("/create-tmp/{uuid}")
|
@PostMapping("/create-tmp/{uuid}")
|
||||||
public ApiResponseDto<UUID> createTmpFile(
|
public ApiResponseDto<UUID> createTmpFile(
|
||||||
@Parameter(description = "uuid", example = "80a0e544-36ed-4999-b705-97427f23337d")
|
@Parameter(description = "model uuid", example = "80a0e544-36ed-4999-b705-97427f23337d")
|
||||||
@PathVariable
|
@PathVariable
|
||||||
UUID uuid) {
|
UUID uuid) {
|
||||||
|
|
||||||
|
|||||||
@@ -6,13 +6,17 @@ import java.io.IOException;
|
|||||||
import java.nio.file.Files;
|
import java.nio.file.Files;
|
||||||
import java.nio.file.Path;
|
import java.nio.file.Path;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
|
import java.util.HashSet;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
import java.util.Set;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
import lombok.Getter;
|
import lombok.Getter;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
import lombok.extern.log4j.Log4j2;
|
import lombok.extern.log4j.Log4j2;
|
||||||
import org.springframework.beans.factory.annotation.Value;
|
import org.springframework.beans.factory.annotation.Value;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
|
/** 학습실행 파일 하드링크 */
|
||||||
@Service
|
@Service
|
||||||
@Log4j2
|
@Log4j2
|
||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
@@ -42,6 +46,10 @@ public class DataSetCountersService {
|
|||||||
|
|
||||||
// tmp
|
// tmp
|
||||||
Path tmpPath = Path.of(trainBaseDir, "tmp", basic.getRequestPath());
|
Path tmpPath = Path.of(trainBaseDir, "tmp", basic.getRequestPath());
|
||||||
|
|
||||||
|
// 차이나는거
|
||||||
|
diffMergedRequestsVsTmp(uids, tmpPath);
|
||||||
|
|
||||||
DatasetCounters counters2 = countTmpAfterBuild(tmpPath);
|
DatasetCounters counters2 = countTmpAfterBuild(tmpPath);
|
||||||
allLogs
|
allLogs
|
||||||
.append(counters2.prints(basic.getRequestPath(), "TMP"))
|
.append(counters2.prints(basic.getRequestPath(), "TMP"))
|
||||||
@@ -163,4 +171,58 @@ public class DataSetCountersService {
|
|||||||
test + test2);
|
test + test2);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private Set<String> listTifRelative(Path root) throws IOException {
|
||||||
|
if (!Files.isDirectory(root)) return Set.of();
|
||||||
|
|
||||||
|
try (var stream = Files.walk(root)) {
|
||||||
|
return stream
|
||||||
|
.filter(Files::isRegularFile)
|
||||||
|
.filter(p -> p.getFileName().toString().toLowerCase().endsWith(".tif"))
|
||||||
|
.map(p -> root.relativize(p).toString().replace("\\", "/"))
|
||||||
|
.collect(Collectors.toSet());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private Set<String> listTifFileNameOnly(Path root) throws IOException {
|
||||||
|
if (!Files.isDirectory(root)) return Set.of();
|
||||||
|
|
||||||
|
try (var stream = Files.walk(root)) {
|
||||||
|
return stream
|
||||||
|
.filter(Files::isRegularFile)
|
||||||
|
.filter(p -> p.getFileName().toString().toLowerCase().endsWith(".tif"))
|
||||||
|
.map(p -> p.getFileName().toString()) // 파일명만
|
||||||
|
.collect(Collectors.toSet());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void diffMergedRequestsVsTmp(List<String> uids, Path tmpRoot) throws IOException {
|
||||||
|
|
||||||
|
// 1) 요청 uids 전체를 합친 tif "파일명" 집합
|
||||||
|
Set<String> reqAll = new HashSet<>();
|
||||||
|
for (String uid : uids) {
|
||||||
|
Path reqRoot = Path.of(requestDir, uid);
|
||||||
|
|
||||||
|
// ★합본 tmp는 보통 폴더 구조가 바뀌므로 "상대경로" 비교보다 파일명 비교가 먼저 유용합니다.
|
||||||
|
reqAll.addAll(listTifFileNameOnly(reqRoot));
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2) tmp tif 파일명 집합
|
||||||
|
Set<String> tmpAll = listTifFileNameOnly(tmpRoot);
|
||||||
|
|
||||||
|
Set<String> missing = new HashSet<>(reqAll);
|
||||||
|
missing.removeAll(tmpAll);
|
||||||
|
|
||||||
|
Set<String> extra = new HashSet<>(tmpAll);
|
||||||
|
extra.removeAll(reqAll);
|
||||||
|
|
||||||
|
log.info("==== MERGED DIFF (filename-based) ====");
|
||||||
|
log.info("request(all uids) tif = {}", reqAll.size());
|
||||||
|
log.info("tmp tif = {}", tmpAll.size());
|
||||||
|
log.info("missing = {}", missing.size());
|
||||||
|
log.info("extra = {}", extra.size());
|
||||||
|
|
||||||
|
missing.stream().sorted().limit(50).forEach(f -> log.warn("[MISSING] {}", f));
|
||||||
|
extra.stream().sorted().limit(50).forEach(f -> log.warn("[EXTRA] {}", f));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -88,6 +88,7 @@ public class TmpDatasetService {
|
|||||||
|
|
||||||
// 충돌 시 덮어쓰기
|
// 충돌 시 덮어쓰기
|
||||||
if (Files.exists(dst)) {
|
if (Files.exists(dst)) {
|
||||||
|
log.warn("COLLISION overwrite: dst={} src={}", dst, src);
|
||||||
Files.delete(dst);
|
Files.delete(dst);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -171,8 +172,8 @@ public class TmpDatasetService {
|
|||||||
scannedDirs++;
|
scannedDirs++;
|
||||||
log.info("SCAN dir={}", srcDir);
|
log.info("SCAN dir={}", srcDir);
|
||||||
|
|
||||||
try (DirectoryStream<Path> stream = Files.newDirectoryStream(srcDir)) {
|
try (var stream = Files.walk(srcDir)) {
|
||||||
for (Path f : stream) {
|
for (Path f : stream.filter(Files::isRegularFile).toList()) {
|
||||||
if (!Files.isRegularFile(f)) {
|
if (!Files.isRegularFile(f)) {
|
||||||
log.debug("skip non-regular file: {}", f);
|
log.debug("skip non-regular file: {}", f);
|
||||||
continue;
|
continue;
|
||||||
|
|||||||
Reference in New Issue
Block a user