daniel 작업본 추가

This commit is contained in:
2026-04-09 17:23:59 +09:00
parent 47e93f0d57
commit edff3b0ef8
5 changed files with 122 additions and 48 deletions

View File

@@ -4,6 +4,7 @@ import com.kamco.cd.training.postgres.repository.train.ModelTestMetricsJobReposi
import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.ModelMetricJsonDto; import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.ModelMetricJsonDto;
import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.ModelTestFileName; import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.ModelTestFileName;
import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.ResponsePathDto; import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.ResponsePathDto;
import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.SimpleMetricJsonDto;
import java.time.ZonedDateTime; import java.time.ZonedDateTime;
import java.util.List; import java.util.List;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
@@ -38,6 +39,10 @@ public class ModelTestMetricsJobCoreService {
return modelTestMetricsJobRepository.getTestMetricPackingInfo(modelId); return modelTestMetricsJobRepository.getTestMetricPackingInfo(modelId);
} }
public SimpleMetricJsonDto getSimpleTestMetricPackingInfo(Long modelId) {
return modelTestMetricsJobRepository.getSimpleTestMetricPackingInfo(modelId);
}
public ModelTestFileName findModelTestFileNames(Long modelId) { public ModelTestFileName findModelTestFileNames(Long modelId) {
return modelTestMetricsJobRepository.findModelTestFileNames(modelId); return modelTestMetricsJobRepository.findModelTestFileNames(modelId);
} }

View File

@@ -3,6 +3,7 @@ package com.kamco.cd.training.postgres.repository.train;
import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.ModelMetricJsonDto; import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.ModelMetricJsonDto;
import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.ModelTestFileName; import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.ModelTestFileName;
import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.ResponsePathDto; import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.ResponsePathDto;
import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.SimpleMetricJsonDto;
import java.time.ZonedDateTime; import java.time.ZonedDateTime;
import java.util.List; import java.util.List;
@@ -18,6 +19,14 @@ public interface ModelTestMetricsJobRepositoryCustom {
ModelMetricJsonDto getTestMetricPackingInfo(Long modelId); ModelMetricJsonDto getTestMetricPackingInfo(Long modelId);
/**
* 간단한 형식의 테스트 메트릭 정보 조회 (ZIP 파일용)
*
* @param modelId 모델 ID
* @return 간단한 JSON 형식 DTO
*/
SimpleMetricJsonDto getSimpleTestMetricPackingInfo(Long modelId);
ModelTestFileName findModelTestFileNames(Long modelId); ModelTestFileName findModelTestFileNames(Long modelId);
void updatePackingStart(Long modelId, ZonedDateTime now); void updatePackingStart(Long modelId, ZonedDateTime now);

View File

@@ -11,6 +11,8 @@ import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.ModelMetricJsonDto;
import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.ModelTestFileName; import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.ModelTestFileName;
import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.Properties; import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.Properties;
import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.ResponsePathDto; import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.ResponsePathDto;
import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.SimpleMetricJsonDto;
import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.SimpleProperties;
import com.querydsl.core.types.Projections; import com.querydsl.core.types.Projections;
import com.querydsl.jpa.impl.JPAQueryFactory; import com.querydsl.jpa.impl.JPAQueryFactory;
import java.time.ZonedDateTime; import java.time.ZonedDateTime;
@@ -100,19 +102,19 @@ public class ModelTestMetricsJobRepositoryImpl extends QuerydslRepositorySupport
// TO-BE: modelId, model(best_fscore_10) 같은 데이터가 있으면 update, 없으면 insert // TO-BE: modelId, model(best_fscore_10) 같은 데이터가 있으면 update, 없으면 insert
String updateSql = String updateSql =
""" """
UPDATE tb_model_metrics_test UPDATE tb_model_metrics_test
SET tp=?, fp=?, fn=?, precisions=?, recall=?, f1_score=?, accuracy=?, iou=?, SET tp=?, fp=?, fn=?, precisions=?, recall=?, f1_score=?, accuracy=?, iou=?,
detection_count=?, gt_count=? detection_count=?, gt_count=?
WHERE model_id=? AND model=? WHERE model_id=? AND model=?
"""; """;
String insertSql = String insertSql =
""" """
INSERT INTO tb_model_metrics_test INSERT INTO tb_model_metrics_test
(model_id, model, tp, fp, fn, precisions, recall, f1_score, accuracy, iou, (model_id, model, tp, fp, fn, precisions, recall, f1_score, accuracy, iou,
detection_count, gt_count) detection_count, gt_count)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
"""; """;
// row 단위 처리 (batch 안에서 upsert) // row 단위 처리 (batch 안에서 upsert)
for (Object[] row : batchArgs) { for (Object[] row : batchArgs) {
@@ -156,6 +158,33 @@ public class ModelTestMetricsJobRepositoryImpl extends QuerydslRepositorySupport
.fetchFirst(); .fetchFirst();
} }
@Override
public SimpleMetricJsonDto getSimpleTestMetricPackingInfo(Long modelId) {
return queryFactory
.select(
Projections.constructor(
SimpleMetricJsonDto.class,
modelMasterEntity.modelNo,
modelMasterEntity.modelVer,
Projections.constructor(
SimpleProperties.class,
modelMetricsTestEntity.f1Score,
modelMetricsTestEntity.precisions,
modelMetricsTestEntity.recall,
modelMetricsTestEntity.iou,
modelMetricsTrainEntity.loss)))
.from(modelMetricsTestEntity)
.innerJoin(modelMasterEntity)
.on(modelMetricsTestEntity.model.id.eq(modelMasterEntity.id))
.innerJoin(modelMetricsTrainEntity)
.on(
modelMetricsTestEntity.model.eq(modelMetricsTrainEntity.model),
modelMasterEntity.bestEpoch.eq(modelMetricsTrainEntity.epoch))
.where(modelMetricsTestEntity.model.id.eq(modelId))
.orderBy(modelMetricsTestEntity.createdDttm.desc())
.fetchFirst();
}
@Override @Override
public ModelTestFileName findModelTestFileNames(Long modelId) { public ModelTestFileName findModelTestFileNames(Long modelId) {
return queryFactory return queryFactory

View File

@@ -180,4 +180,35 @@ public class ModelTrainMetricsDto {
private String bestEpochFileName; private String bestEpochFileName;
private String modelVersion; private String modelVersion;
} }
/** 간단한 JSON 형식용 DTO (ZIP 파일 포함용) */
@Getter
@AllArgsConstructor
public static class SimpleMetricJsonDto {
@JsonProperty("cd_model_type")
private String cdModelType;
@JsonProperty("model_version")
private String modelVersion;
private SimpleProperties properties;
}
/** 간단한 Properties (필수 메트릭만 포함) */
@Getter
@AllArgsConstructor
public static class SimpleProperties {
@JsonProperty("f1_score")
private Float f1Score;
private Float precision;
private Float recall;
private Float iou;
private Double loss;
}
} }

View File

@@ -8,6 +8,7 @@ import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.BestPthInfo;
import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.ModelMetricJsonDto; import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.ModelMetricJsonDto;
import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.ModelTestFileName; import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.ModelTestFileName;
import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.ResponsePathDto; import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.ResponsePathDto;
import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.SimpleMetricJsonDto;
import java.io.BufferedReader; import java.io.BufferedReader;
import java.io.IOException; import java.io.IOException;
import java.io.OutputStream; import java.io.OutputStream;
@@ -139,8 +140,9 @@ public class ModelTestMetricsJobService {
// 패키징할 파일 만들기 // 패키징할 파일 만들기
modelTestMetricsJobCoreService.updatePackingStart(modelInfo.getModelId(), ZonedDateTime.now()); modelTestMetricsJobCoreService.updatePackingStart(modelInfo.getModelId(), ZonedDateTime.now());
ModelMetricJsonDto jsonDto = // 간단한 형식의 JSON 생성 (ZIP 파일용)
modelTestMetricsJobCoreService.getTestMetricPackingInfo(modelInfo.getModelId()); SimpleMetricJsonDto jsonDto =
modelTestMetricsJobCoreService.getSimpleTestMetricPackingInfo(modelInfo.getModelId());
try { try {
writeJsonFile( writeJsonFile(
jsonDto, jsonDto,
@@ -255,42 +257,23 @@ public class ModelTestMetricsJobService {
Path individualJsonPath = null; Path individualJsonPath = null;
try { try {
// 3-1. PTH 파일명에서 Epoch과 메트릭 타입 추출 // 파싱 성공 여부와 관계없이 기본 간단한 JSON 사용
BestPthInfo pthInfo = parsePthFileName(pthFileName); // (개별 ZIP도 동일한 간단한 형식 적용)
if (pthInfo == null) { log.debug("PTH 파일: {}, 간단한 JSON 형식 사용", pthFileName);
log.warn("PTH 파일명 파싱 실패, 기본 JSON 사용: {}", pthFileName);
// 파싱 실패 시 기본 메트릭 정보 사용
ModelMetricJsonDto defaultJsonDto =
modelTestMetricsJobCoreService.getTestMetricPackingInfo(modelInfo.getModelId());
individualJsonPath = createIndividualJson(responsePath, pthFileName, defaultJsonDto);
} else {
// 3-2. 해당 Epoch의 메트릭 정보 조회
log.debug(
"PTH 파일 파싱 결과: epoch={}, metricType={}",
pthInfo.getEpoch(),
pthInfo.getMetricType());
ModelMetricJsonDto individualJsonDto = SimpleMetricJsonDto simpleJsonDto =
modelTestMetricsJobCoreService.getMetricsByEpoch( modelTestMetricsJobCoreService.getSimpleTestMetricPackingInfo(modelInfo.getModelId());
modelInfo.getModelId(), pthInfo.getEpoch(), pthInfo.getMetricType());
if (individualJsonDto == null) { if (simpleJsonDto == null) {
log.warn("Epoch={} 메트릭 정보 없음, 기본 JSON 사용: {}", pthInfo.getEpoch(), pthFileName); log.warn("메트릭 정보 없음, 건너뜀: {}", pthFileName);
ModelMetricJsonDto defaultJsonDto = continue;
modelTestMetricsJobCoreService.getTestMetricPackingInfo(modelInfo.getModelId());
individualJsonPath = createIndividualJson(responsePath, pthFileName, defaultJsonDto);
} else {
// 3-3. 개별 JSON 파일 생성
individualJsonPath =
createIndividualJson(responsePath, pthFileName, individualJsonDto);
log.info(
" Epoch별 JSON 생성: epoch={}, file={}",
pthInfo.getEpoch(),
individualJsonPath.getFileName());
}
} }
// 3-4. 개별 ZIP 파일 생성 // 3-2. 개별 JSON 파일 생성
individualJsonPath = createIndividualJson(responsePath, pthFileName, simpleJsonDto);
log.debug("개별 JSON 생성: file={}", individualJsonPath.getFileName());
// 3-3. 개별 ZIP 파일명 생성
// 형식: {modelVersion}.{pthFileNameWithoutExt}.zip // 형식: {modelVersion}.{pthFileNameWithoutExt}.zip
// 예: G1_000001.best_fscore_5.zip, G1_000001.best_precision_7.zip // 예: G1_000001.best_fscore_5.zip, G1_000001.best_precision_7.zip
String pthFileNameWithoutExt = pthFileName.replace(".pth", ""); String pthFileNameWithoutExt = pthFileName.replace(".pth", "");
@@ -302,7 +285,7 @@ public class ModelTestMetricsJobService {
fileInfo.getModelVersion() + "." + pthFileNameWithoutExt + ".zip"; fileInfo.getModelVersion() + "." + pthFileNameWithoutExt + ".zip";
Path individualZipPath = responsePath.resolve(individualZipName); Path individualZipPath = responsePath.resolve(individualZipName);
// 3-5. ZIP에 포함될 파일 목록 구성 // 3-4. ZIP에 포함될 파일 목록 구성
List<Path> zipFileList = new ArrayList<>(); List<Path> zipFileList = new ArrayList<>();
zipFileList.add(bestPthFile); // best*.pth 파일 zipFileList.add(bestPthFile); // best*.pth 파일
zipFileList.add(individualJsonPath); // 개별 JSON 파일 zipFileList.add(individualJsonPath); // 개별 JSON 파일
@@ -313,16 +296,16 @@ public class ModelTestMetricsJobService {
zipFileList.add(modelConfigPath); zipFileList.add(modelConfigPath);
} }
// 3-6. 개별 ZIP 생성 // 3-5. 개별 ZIP 생성
zipFiles(zipFileList, individualZipPath); zipFiles(zipFileList, individualZipPath);
log.info( log.info(
"개별 ZIP 생성 완료: fileName={}, pthFile={}, size={} bytes", "개별 ZIP 생성 완료: fileName={}, pthFile={}, size={} bytes",
individualZipName, individualZipName,
pthFileName, pthFileName,
Files.size(individualZipPath)); Files.size(individualZipPath));
// 3-7. 임시 JSON 파일 정리 // 3-6. 임시 JSON 파일 정리
try { try {
Files.deleteIfExists(individualJsonPath); Files.deleteIfExists(individualJsonPath);
log.debug("임시 JSON 파일 삭제: {}", individualJsonPath.getFileName()); log.debug("임시 JSON 파일 삭제: {}", individualJsonPath.getFileName());
@@ -430,6 +413,23 @@ public class ModelTestMetricsJobService {
return individualJsonPath; return individualJsonPath;
} }
/**
* 개별 JSON 파일 생성 (간단한 형식)
*
* @param responsePath Response 디렉토리 경로
* @param pthFileName PTH 파일명
* @param jsonDto 간단한 JSON 메타데이터
* @return 생성된 JSON 파일 경로
* @throws IOException JSON 쓰기 실패 시
*/
private Path createIndividualJson(
Path responsePath, String pthFileName, SimpleMetricJsonDto jsonDto) throws IOException {
String individualJsonName = pthFileName.replace(".pth", ".json");
Path individualJsonPath = responsePath.resolve(individualJsonName);
writeJsonFile(jsonDto, individualJsonPath);
return individualJsonPath;
}
private void writeJsonFile(Object data, Path outputPath) throws IOException { private void writeJsonFile(Object data, Path outputPath) throws IOException {
Path parent = outputPath.getParent(); Path parent = outputPath.getParent();