From 4d9c9a86b480ce188bea67465a5ee1432662e5f9 Mon Sep 17 00:00:00 2001 From: "gayoun.park" Date: Thu, 12 Feb 2026 21:09:40 +0900 Subject: [PATCH] =?UTF-8?q?=ED=8C=A8=ED=82=A4=EC=A7=95=20zip=ED=8C=8C?= =?UTF-8?q?=EC=9D=BC=20=EB=A7=8C=EB=93=A4=EA=B8=B0=20=EC=BB=A4=EB=B0=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../core/ModelTestMetricsJobCoreService.java | 10 ++ .../ModelTestMetricsJobRepositoryCustom.java | 6 + .../ModelTestMetricsJobRepositoryImpl.java | 44 +++++++ .../train/dto/ModelTrainMetricsDto.java | 36 ++++++ .../service/ModelTestMetricsJobService.java | 109 +++++++++++++++++- src/main/resources/application-dev.yml | 3 + src/main/resources/application-prod.yml | 3 + 7 files changed, 209 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/kamco/cd/training/postgres/core/ModelTestMetricsJobCoreService.java b/src/main/java/com/kamco/cd/training/postgres/core/ModelTestMetricsJobCoreService.java index a19e34c..2f2fec7 100644 --- a/src/main/java/com/kamco/cd/training/postgres/core/ModelTestMetricsJobCoreService.java +++ b/src/main/java/com/kamco/cd/training/postgres/core/ModelTestMetricsJobCoreService.java @@ -1,6 +1,8 @@ package com.kamco.cd.training.postgres.core; import com.kamco.cd.training.postgres.repository.train.ModelTestMetricsJobRepository; +import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.ModelMetricJsonDto; +import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.ModelTestFileName; import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.ResponsePathDto; import java.util.List; import lombok.RequiredArgsConstructor; @@ -26,4 +28,12 @@ public class ModelTestMetricsJobCoreService { public void insertModelMetricsTest(List batchArgs) { modelTestMetricsJobRepository.insertModelMetricsTest(batchArgs); } + + public ModelMetricJsonDto getTestMetricPackingInfo(Long modelId) { + return modelTestMetricsJobRepository.getTestMetricPackingInfo(modelId); + } + + public ModelTestFileName findModelTestFileNames(Long modelId) { + return modelTestMetricsJobRepository.findModelTestFileNames(modelId); + } } diff --git a/src/main/java/com/kamco/cd/training/postgres/repository/train/ModelTestMetricsJobRepositoryCustom.java b/src/main/java/com/kamco/cd/training/postgres/repository/train/ModelTestMetricsJobRepositoryCustom.java index a1a6a04..decfdd8 100644 --- a/src/main/java/com/kamco/cd/training/postgres/repository/train/ModelTestMetricsJobRepositoryCustom.java +++ b/src/main/java/com/kamco/cd/training/postgres/repository/train/ModelTestMetricsJobRepositoryCustom.java @@ -1,5 +1,7 @@ package com.kamco.cd.training.postgres.repository.train; +import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.ModelMetricJsonDto; +import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.ModelTestFileName; import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.ResponsePathDto; import java.util.List; @@ -10,4 +12,8 @@ public interface ModelTestMetricsJobRepositoryCustom { List getTestMetricSaveNotYetModelIds(); void insertModelMetricsTest(List batchArgs); + + ModelMetricJsonDto getTestMetricPackingInfo(Long modelId); + + ModelTestFileName findModelTestFileNames(Long modelId); } diff --git a/src/main/java/com/kamco/cd/training/postgres/repository/train/ModelTestMetricsJobRepositoryImpl.java b/src/main/java/com/kamco/cd/training/postgres/repository/train/ModelTestMetricsJobRepositoryImpl.java index cf6ead8..50594d6 100644 --- a/src/main/java/com/kamco/cd/training/postgres/repository/train/ModelTestMetricsJobRepositoryImpl.java +++ b/src/main/java/com/kamco/cd/training/postgres/repository/train/ModelTestMetricsJobRepositoryImpl.java @@ -1,9 +1,14 @@ package com.kamco.cd.training.postgres.repository.train; import static com.kamco.cd.training.postgres.entity.QModelMasterEntity.modelMasterEntity; +import static com.kamco.cd.training.postgres.entity.QModelMetricsTestEntity.modelMetricsTestEntity; +import static com.kamco.cd.training.postgres.entity.QModelMetricsTrainEntity.modelMetricsTrainEntity; import com.kamco.cd.training.common.enums.TrainStatusType; import com.kamco.cd.training.postgres.entity.ModelMetricsTestEntity; +import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.ModelMetricJsonDto; +import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.ModelTestFileName; +import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.Properties; import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.ResponsePathDto; import com.querydsl.core.types.Projections; import com.querydsl.jpa.impl.JPAQueryFactory; @@ -70,4 +75,43 @@ public class ModelTestMetricsJobRepositoryImpl extends QuerydslRepositorySupport jdbcTemplate.batchUpdate(sql, batchArgs); } + + @Override + public ModelMetricJsonDto getTestMetricPackingInfo(Long modelId) { + return queryFactory + .select( + Projections.constructor( + ModelMetricJsonDto.class, + modelMasterEntity.modelNo, + modelMasterEntity.modelVer, + Projections.constructor( + Properties.class, + modelMetricsTestEntity.f1Score, + modelMetricsTestEntity.precisions, + modelMetricsTestEntity.recall, + modelMetricsTestEntity.iou, + modelMetricsTrainEntity.loss))) + .from(modelMetricsTestEntity) + .innerJoin(modelMasterEntity) + .on(modelMetricsTestEntity.model.id.eq(modelMasterEntity.id)) + .innerJoin(modelMetricsTrainEntity) + .on( + modelMetricsTestEntity.model.eq(modelMetricsTrainEntity.model), + modelMasterEntity.bestEpoch.eq(modelMetricsTrainEntity.epoch)) + .where(modelMetricsTestEntity.model.id.eq(modelId)) + .fetchOne(); + } + + @Override + public ModelTestFileName findModelTestFileNames(Long modelId) { + return queryFactory + .select( + Projections.constructor( + ModelTestFileName.class, modelMetricsTestEntity.model1, modelMasterEntity.modelVer)) + .from(modelMetricsTestEntity) + .innerJoin(modelMasterEntity) + .on(modelMetricsTestEntity.model.id.eq(modelMasterEntity.id)) + .where(modelMetricsTestEntity.model.id.eq(modelId)) + .fetchOne(); + } } diff --git a/src/main/java/com/kamco/cd/training/train/dto/ModelTrainMetricsDto.java b/src/main/java/com/kamco/cd/training/train/dto/ModelTrainMetricsDto.java index cf69372..3e54e4e 100644 --- a/src/main/java/com/kamco/cd/training/train/dto/ModelTrainMetricsDto.java +++ b/src/main/java/com/kamco/cd/training/train/dto/ModelTrainMetricsDto.java @@ -1,6 +1,8 @@ package com.kamco.cd.training.train.dto; +import com.fasterxml.jackson.annotation.JsonProperty; import io.swagger.v3.oas.annotations.media.Schema; +import java.util.Properties; import java.util.UUID; import lombok.AllArgsConstructor; import lombok.Getter; @@ -20,4 +22,38 @@ public class ModelTrainMetricsDto { private String responsePath; private UUID uuid; } + + @Getter + @AllArgsConstructor + public static class ModelMetricJsonDto { + + @JsonProperty("cd_model_type") + private String cdModelType; + + @JsonProperty("model_version") + private String modelVersion; + + private Properties properties; + } + + @Getter + @AllArgsConstructor + public static class Properties { + + @JsonProperty("f1_score") + private Float f1Score; + + private Float precision; + private Float recall; + private Float loss; + private Double iou; + } + + @Getter + @AllArgsConstructor + public static class ModelTestFileName { + + private String bestEpochFileName; + private String modelVersion; + } } diff --git a/src/main/java/com/kamco/cd/training/train/service/ModelTestMetricsJobService.java b/src/main/java/com/kamco/cd/training/train/service/ModelTestMetricsJobService.java index a607d3a..e643673 100644 --- a/src/main/java/com/kamco/cd/training/train/service/ModelTestMetricsJobService.java +++ b/src/main/java/com/kamco/cd/training/train/service/ModelTestMetricsJobService.java @@ -1,14 +1,26 @@ package com.kamco.cd.training.train.service; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; import com.kamco.cd.training.postgres.core.ModelTestMetricsJobCoreService; +import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.ModelMetricJsonDto; +import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.ModelTestFileName; import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.ResponsePathDto; import java.io.BufferedReader; import java.io.IOException; +import java.io.OutputStream; import java.nio.charset.StandardCharsets; import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; import java.util.ArrayList; import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.csv.CSVFormat; @@ -31,6 +43,9 @@ public class ModelTestMetricsJobService { @Value("${train.docker.responseDir}") private String responseDir; + @Value("${file.pt-path}") + private String ptPathDir; + /** * 실행중인 profile * @@ -40,8 +55,8 @@ public class ModelTestMetricsJobService { return "local".equalsIgnoreCase(profile); } - // @Scheduled(cron = "0 * * * * *") - public void findTestValidMetricCsvFiles() { + // @Scheduled(cron = "0 * * * * *") + public void findTestValidMetricCsvFiles() throws IOException { // if (isLocalProfile()) { // return; // } @@ -100,7 +115,97 @@ public class ModelTestMetricsJobService { throw new RuntimeException(e); } + // 패키징할 파일 만들기 + ModelMetricJsonDto jsonDto = + modelTestMetricsJobCoreService.getTestMetricPackingInfo(modelInfo.getModelId()); + try { + writeJsonFile( + jsonDto, + Paths.get( + responseDir + + "/" + + modelInfo.getUuid() + + "/" + + jsonDto.getModelVersion() + + ".json")); + } catch (IOException e) { + throw new RuntimeException(e); + } + + Path responsePath = Paths.get(responseDir + "/" + modelInfo.getUuid()); + + ModelTestFileName fileInfo = + modelTestMetricsJobCoreService.findModelTestFileNames(modelInfo.getModelId()); + + Path zipPath = + Paths.get( + responseDir + "/" + modelInfo.getUuid() + "/" + fileInfo.getModelVersion() + ".zip"); + Set targetNames = + Set.of( + "model_config.py", + fileInfo.getBestEpochFileName() + ".pth", + fileInfo.getModelVersion() + ".json"); + + List files = new ArrayList<>(); + try (Stream s = Files.list(responsePath)) { + files.addAll( + s.filter(Files::isRegularFile) + .filter(p -> targetNames.contains(p.getFileName().toString())) + .collect(Collectors.toList())); + } catch (IOException e) { + throw new RuntimeException(e); + } + + try (Stream s = Files.list(Path.of(ptPathDir))) { + files.addAll( + s.filter(Files::isRegularFile) + .limit(1) // yolov8_6th-6m.pt 파일 1개만 + .collect(Collectors.toList())); + } catch (IOException e) { + throw new RuntimeException(e); + } + + zipFiles(files, zipPath); modelTestMetricsJobCoreService.updateModelMetricsTrainSaveYn(modelInfo.getModelId(), "step2"); } } + + private void writeJsonFile(Object data, Path outputPath) throws IOException { + + Path parent = outputPath.getParent(); + + if (parent != null) { + Files.createDirectories(parent); + } + + ObjectMapper objectMapper = new ObjectMapper(); + objectMapper.enable(SerializationFeature.INDENT_OUTPUT); + + try (OutputStream os = + Files.newOutputStream( + outputPath, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING)) { + objectMapper.writeValue(os, data); + } + } + + private void zipFiles(List files, Path zipPath) throws IOException { + + Path parent = zipPath.getParent(); + if (parent != null) { + Files.createDirectories(parent); + } + + try (ZipOutputStream zos = new ZipOutputStream(Files.newOutputStream(zipPath))) { + + for (Path file : files) { + + ZipEntry entry = new ZipEntry(file.getFileName().toString()); + zos.putNextEntry(entry); + + Files.copy(file, zos); + + zos.closeEntry(); + } + } + } } diff --git a/src/main/resources/application-dev.yml b/src/main/resources/application-dev.yml index caab7a8..d418625 100644 --- a/src/main/resources/application-dev.yml +++ b/src/main/resources/application-dev.yml @@ -58,6 +58,9 @@ file: dataset-dir: /home/kcomu/data/request/ dataset-tmp-dir: ${file.dataset-dir}tmp/ + pt-path: /home/kcomu/data/response/v6-cls-checkpoints/ + pt-FileName: yolov8_6th-6m.pt + train: docker: image: "kamco-cd-train:latest" diff --git a/src/main/resources/application-prod.yml b/src/main/resources/application-prod.yml index 54c1d32..73806e1 100644 --- a/src/main/resources/application-prod.yml +++ b/src/main/resources/application-prod.yml @@ -44,6 +44,9 @@ file: dataset-dir: /home/kcomu/data/request/ dataset-tmp-dir: ${file.dataset-dir}tmp/ + pt-path: /home/kcomu/data/response/v6-cls-checkpoints/ + pt-FileName: yolov8_6th-6m.pt + train: docker: image: "kamco-cd-train:latest"