Merge pull request '하드링크 로그 추가' (#138) from feat/training_260202 into develop
Reviewed-on: #138
This commit was merged in pull request #138.
This commit is contained in:
@@ -185,7 +185,7 @@ public class TrainApiController {
|
||||
})
|
||||
@PostMapping("/create-tmp/{uuid}")
|
||||
public ApiResponseDto<UUID> createTmpFile(
|
||||
@Parameter(description = "uuid", example = "80a0e544-36ed-4999-b705-97427f23337d")
|
||||
@Parameter(description = "model uuid", example = "80a0e544-36ed-4999-b705-97427f23337d")
|
||||
@PathVariable
|
||||
UUID uuid) {
|
||||
|
||||
|
||||
@@ -6,13 +6,17 @@ import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
import lombok.Getter;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.log4j.Log4j2;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
/** 학습실행 파일 하드링크 */
|
||||
@Service
|
||||
@Log4j2
|
||||
@RequiredArgsConstructor
|
||||
@@ -42,6 +46,10 @@ public class DataSetCountersService {
|
||||
|
||||
// tmp
|
||||
Path tmpPath = Path.of(trainBaseDir, "tmp", basic.getRequestPath());
|
||||
|
||||
// 차이나는거
|
||||
diffMergedRequestsVsTmp(uids, tmpPath);
|
||||
|
||||
DatasetCounters counters2 = countTmpAfterBuild(tmpPath);
|
||||
allLogs
|
||||
.append(counters2.prints(basic.getRequestPath(), "TMP"))
|
||||
@@ -163,4 +171,58 @@ public class DataSetCountersService {
|
||||
test + test2);
|
||||
}
|
||||
}
|
||||
|
||||
private Set<String> listTifRelative(Path root) throws IOException {
|
||||
if (!Files.isDirectory(root)) return Set.of();
|
||||
|
||||
try (var stream = Files.walk(root)) {
|
||||
return stream
|
||||
.filter(Files::isRegularFile)
|
||||
.filter(p -> p.getFileName().toString().toLowerCase().endsWith(".tif"))
|
||||
.map(p -> root.relativize(p).toString().replace("\\", "/"))
|
||||
.collect(Collectors.toSet());
|
||||
}
|
||||
}
|
||||
|
||||
private Set<String> listTifFileNameOnly(Path root) throws IOException {
|
||||
if (!Files.isDirectory(root)) return Set.of();
|
||||
|
||||
try (var stream = Files.walk(root)) {
|
||||
return stream
|
||||
.filter(Files::isRegularFile)
|
||||
.filter(p -> p.getFileName().toString().toLowerCase().endsWith(".tif"))
|
||||
.map(p -> p.getFileName().toString()) // 파일명만
|
||||
.collect(Collectors.toSet());
|
||||
}
|
||||
}
|
||||
|
||||
public void diffMergedRequestsVsTmp(List<String> uids, Path tmpRoot) throws IOException {
|
||||
|
||||
// 1) 요청 uids 전체를 합친 tif "파일명" 집합
|
||||
Set<String> reqAll = new HashSet<>();
|
||||
for (String uid : uids) {
|
||||
Path reqRoot = Path.of(requestDir, uid);
|
||||
|
||||
// ★합본 tmp는 보통 폴더 구조가 바뀌므로 "상대경로" 비교보다 파일명 비교가 먼저 유용합니다.
|
||||
reqAll.addAll(listTifFileNameOnly(reqRoot));
|
||||
}
|
||||
|
||||
// 2) tmp tif 파일명 집합
|
||||
Set<String> tmpAll = listTifFileNameOnly(tmpRoot);
|
||||
|
||||
Set<String> missing = new HashSet<>(reqAll);
|
||||
missing.removeAll(tmpAll);
|
||||
|
||||
Set<String> extra = new HashSet<>(tmpAll);
|
||||
extra.removeAll(reqAll);
|
||||
|
||||
log.info("==== MERGED DIFF (filename-based) ====");
|
||||
log.info("request(all uids) tif = {}", reqAll.size());
|
||||
log.info("tmp tif = {}", tmpAll.size());
|
||||
log.info("missing = {}", missing.size());
|
||||
log.info("extra = {}", extra.size());
|
||||
|
||||
missing.stream().sorted().limit(50).forEach(f -> log.warn("[MISSING] {}", f));
|
||||
extra.stream().sorted().limit(50).forEach(f -> log.warn("[EXTRA] {}", f));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -88,6 +88,7 @@ public class TmpDatasetService {
|
||||
|
||||
// 충돌 시 덮어쓰기
|
||||
if (Files.exists(dst)) {
|
||||
log.warn("COLLISION overwrite: dst={} src={}", dst, src);
|
||||
Files.delete(dst);
|
||||
}
|
||||
|
||||
@@ -171,8 +172,8 @@ public class TmpDatasetService {
|
||||
scannedDirs++;
|
||||
log.info("SCAN dir={}", srcDir);
|
||||
|
||||
try (DirectoryStream<Path> stream = Files.newDirectoryStream(srcDir)) {
|
||||
for (Path f : stream) {
|
||||
try (var stream = Files.walk(srcDir)) {
|
||||
for (Path f : stream.filter(Files::isRegularFile).toList()) {
|
||||
if (!Files.isRegularFile(f)) {
|
||||
log.debug("skip non-regular file: {}", f);
|
||||
continue;
|
||||
|
||||
Reference in New Issue
Block a user