From b451f697bcb0a3220bcbb036e53aef72426f78d6 Mon Sep 17 00:00:00 2001 From: "gayoun.park" Date: Thu, 12 Feb 2026 14:59:35 +0900 Subject: [PATCH 1/4] =?UTF-8?q?=EB=AA=A8=EB=8D=B8=20=EB=A7=88=EC=8A=A4?= =?UTF-8?q?=ED=84=B0=20=ED=85=8C=EC=9D=B4=EB=B8=94=20request,response=20?= =?UTF-8?q?=EA=B2=BD=EB=A1=9C=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cd/training/postgres/entity/ModelMasterEntity.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/main/java/com/kamco/cd/training/postgres/entity/ModelMasterEntity.java b/src/main/java/com/kamco/cd/training/postgres/entity/ModelMasterEntity.java index 4aca805..cd1124d 100644 --- a/src/main/java/com/kamco/cd/training/postgres/entity/ModelMasterEntity.java +++ b/src/main/java/com/kamco/cd/training/postgres/entity/ModelMasterEntity.java @@ -106,6 +106,12 @@ public class ModelMasterEntity { @Column(name = "best_epoch") private Integer bestEpoch; + @Column(name = "request_path") + private String requestPath; + + @Column(name = "response_path") + private String responsePath; + public ModelTrainMngDto.Basic toDto() { return new ModelTrainMngDto.Basic( this.id, From 2df4a7a80b5a0ebc97c544904602b0c5da36c143 Mon Sep 17 00:00:00 2001 From: "gayoun.park" Date: Thu, 12 Feb 2026 15:24:30 +0900 Subject: [PATCH 2/4] =?UTF-8?q?csv=20=ED=8C=8C=EC=9D=BC=20=EC=9D=BD?= =?UTF-8?q?=EB=8A=94=20=EA=B2=BD=EB=A1=9C=20=EC=9D=BD=EC=96=B4=EC=84=9C=20?= =?UTF-8?q?=EC=88=98=EC=A0=95,=20train=EC=9D=80=20epoch=20+=201=20?= =?UTF-8?q?=ED=95=B4=EC=84=9C=20=EC=A0=80=EC=9E=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../core/ModelTestMetricsJobCoreService.java | 3 +- .../core/ModelTrainMetricsJobCoreService.java | 3 +- .../ModelTestMetricsJobRepositoryCustom.java | 3 +- .../ModelTestMetricsJobRepositoryImpl.java | 8 +- .../ModelTrainMetricsJobRepositoryCustom.java | 3 +- .../ModelTrainMetricsJobRepositoryImpl.java | 12 +- .../train/dto/ModelTrainMetricsDto.java | 21 +++ .../service/ModelTestMetricsJobService.java | 94 ++++++------ .../service/ModelTrainMetricsJobService.java | 136 +++++++++--------- 9 files changed, 160 insertions(+), 123 deletions(-) create mode 100644 src/main/java/com/kamco/cd/training/train/dto/ModelTrainMetricsDto.java diff --git a/src/main/java/com/kamco/cd/training/postgres/core/ModelTestMetricsJobCoreService.java b/src/main/java/com/kamco/cd/training/postgres/core/ModelTestMetricsJobCoreService.java index 6ba1e56..a19e34c 100644 --- a/src/main/java/com/kamco/cd/training/postgres/core/ModelTestMetricsJobCoreService.java +++ b/src/main/java/com/kamco/cd/training/postgres/core/ModelTestMetricsJobCoreService.java @@ -1,6 +1,7 @@ package com.kamco.cd.training.postgres.core; import com.kamco.cd.training.postgres.repository.train.ModelTestMetricsJobRepository; +import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.ResponsePathDto; import java.util.List; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -18,7 +19,7 @@ public class ModelTestMetricsJobCoreService { } // Test 로직 시작 - public List getTestMetricSaveNotYetModelIds() { + public List getTestMetricSaveNotYetModelIds() { return modelTestMetricsJobRepository.getTestMetricSaveNotYetModelIds(); } diff --git a/src/main/java/com/kamco/cd/training/postgres/core/ModelTrainMetricsJobCoreService.java b/src/main/java/com/kamco/cd/training/postgres/core/ModelTrainMetricsJobCoreService.java index 5692017..d7823b1 100644 --- a/src/main/java/com/kamco/cd/training/postgres/core/ModelTrainMetricsJobCoreService.java +++ b/src/main/java/com/kamco/cd/training/postgres/core/ModelTrainMetricsJobCoreService.java @@ -1,6 +1,7 @@ package com.kamco.cd.training.postgres.core; import com.kamco.cd.training.postgres.repository.train.ModelTrainMetricsJobRepository; +import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.ResponsePathDto; import java.util.List; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -12,7 +13,7 @@ public class ModelTrainMetricsJobCoreService { private final ModelTrainMetricsJobRepository modelTrainMetricsJobRepository; - public List getTrainMetricSaveNotYetModelIds() { + public List getTrainMetricSaveNotYetModelIds() { return modelTrainMetricsJobRepository.getTrainMetricSaveNotYetModelIds(); } diff --git a/src/main/java/com/kamco/cd/training/postgres/repository/train/ModelTestMetricsJobRepositoryCustom.java b/src/main/java/com/kamco/cd/training/postgres/repository/train/ModelTestMetricsJobRepositoryCustom.java index bd993e1..a1a6a04 100644 --- a/src/main/java/com/kamco/cd/training/postgres/repository/train/ModelTestMetricsJobRepositoryCustom.java +++ b/src/main/java/com/kamco/cd/training/postgres/repository/train/ModelTestMetricsJobRepositoryCustom.java @@ -1,12 +1,13 @@ package com.kamco.cd.training.postgres.repository.train; +import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.ResponsePathDto; import java.util.List; public interface ModelTestMetricsJobRepositoryCustom { void updateModelMetricsTrainSaveYn(Long modelId, String stepNo); - List getTestMetricSaveNotYetModelIds(); + List getTestMetricSaveNotYetModelIds(); void insertModelMetricsTest(List batchArgs); } diff --git a/src/main/java/com/kamco/cd/training/postgres/repository/train/ModelTestMetricsJobRepositoryImpl.java b/src/main/java/com/kamco/cd/training/postgres/repository/train/ModelTestMetricsJobRepositoryImpl.java index 7804c52..f189150 100644 --- a/src/main/java/com/kamco/cd/training/postgres/repository/train/ModelTestMetricsJobRepositoryImpl.java +++ b/src/main/java/com/kamco/cd/training/postgres/repository/train/ModelTestMetricsJobRepositoryImpl.java @@ -4,6 +4,8 @@ import static com.kamco.cd.training.postgres.entity.QModelMasterEntity.modelMast import com.kamco.cd.training.common.enums.TrainStatusType; import com.kamco.cd.training.postgres.entity.ModelMetricsTestEntity; +import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.ResponsePathDto; +import com.querydsl.core.types.Projections; import com.querydsl.jpa.impl.JPAQueryFactory; import java.util.List; import org.springframework.data.jpa.repository.support.QuerydslRepositorySupport; @@ -36,9 +38,11 @@ public class ModelTestMetricsJobRepositoryImpl extends QuerydslRepositorySupport } @Override - public List getTestMetricSaveNotYetModelIds() { + public List getTestMetricSaveNotYetModelIds() { return queryFactory - .select(modelMasterEntity.id) + .select( + Projections.constructor( + ResponsePathDto.class, modelMasterEntity.id, modelMasterEntity.responsePath)) .from(modelMasterEntity) .where( modelMasterEntity.step2EndDttm.isNotNull(), diff --git a/src/main/java/com/kamco/cd/training/postgres/repository/train/ModelTrainMetricsJobRepositoryCustom.java b/src/main/java/com/kamco/cd/training/postgres/repository/train/ModelTrainMetricsJobRepositoryCustom.java index a10caa8..67517fe 100644 --- a/src/main/java/com/kamco/cd/training/postgres/repository/train/ModelTrainMetricsJobRepositoryCustom.java +++ b/src/main/java/com/kamco/cd/training/postgres/repository/train/ModelTrainMetricsJobRepositoryCustom.java @@ -1,10 +1,11 @@ package com.kamco.cd.training.postgres.repository.train; +import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.ResponsePathDto; import java.util.List; public interface ModelTrainMetricsJobRepositoryCustom { - List getTrainMetricSaveNotYetModelIds(); + List getTrainMetricSaveNotYetModelIds(); void insertModelMetricsTrain(List batchArgs); diff --git a/src/main/java/com/kamco/cd/training/postgres/repository/train/ModelTrainMetricsJobRepositoryImpl.java b/src/main/java/com/kamco/cd/training/postgres/repository/train/ModelTrainMetricsJobRepositoryImpl.java index c20bc73..d650bc7 100644 --- a/src/main/java/com/kamco/cd/training/postgres/repository/train/ModelTrainMetricsJobRepositoryImpl.java +++ b/src/main/java/com/kamco/cd/training/postgres/repository/train/ModelTrainMetricsJobRepositoryImpl.java @@ -4,6 +4,8 @@ import static com.kamco.cd.training.postgres.entity.QModelMasterEntity.modelMast import com.kamco.cd.training.common.enums.TrainStatusType; import com.kamco.cd.training.postgres.entity.ModelMetricsTrainEntity; +import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.ResponsePathDto; +import com.querydsl.core.types.Projections; import com.querydsl.jpa.impl.JPAQueryFactory; import java.util.List; import org.springframework.data.jpa.repository.support.QuerydslRepositorySupport; @@ -23,9 +25,11 @@ public class ModelTrainMetricsJobRepositoryImpl extends QuerydslRepositorySuppor } @Override - public List getTrainMetricSaveNotYetModelIds() { + public List getTrainMetricSaveNotYetModelIds() { return queryFactory - .select(modelMasterEntity.id) + .select( + Projections.constructor( + ResponsePathDto.class, modelMasterEntity.id, modelMasterEntity.responsePath)) .from(modelMasterEntity) .where( modelMasterEntity.step1EndDttm.isNotNull(), @@ -41,7 +45,7 @@ public class ModelTrainMetricsJobRepositoryImpl extends QuerydslRepositorySuppor public void insertModelMetricsTrain(List batchArgs) { String sql = """ - insert into tb_model_matrics_train + insert into tb_model_metrics_train (model_id, epoch, iteration, loss, lr, duration_time) values (?, ?, ?, ?, ?, ?) """; @@ -66,7 +70,7 @@ public class ModelTrainMetricsJobRepositoryImpl extends QuerydslRepositorySuppor public void insertModelMetricsValidation(List batchArgs) { String sql = """ - insert into tb_model_matrics_validation + insert into tb_model_metrics_validation (model_id, epoch, a_acc, m_fscore, m_precision, m_recall, m_iou, m_acc, changed_fscore, changed_precision, changed_recall, unchanged_fscore, unchanged_precision, unchanged_recall ) diff --git a/src/main/java/com/kamco/cd/training/train/dto/ModelTrainMetricsDto.java b/src/main/java/com/kamco/cd/training/train/dto/ModelTrainMetricsDto.java new file mode 100644 index 0000000..ff6bd90 --- /dev/null +++ b/src/main/java/com/kamco/cd/training/train/dto/ModelTrainMetricsDto.java @@ -0,0 +1,21 @@ +package com.kamco.cd.training.train.dto; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +public class ModelTrainMetricsDto { + + @Schema(name = "ResponsePathDto", description = "AI 결과 저장된 path 경로 정보") + @Getter + @Setter + @NoArgsConstructor + @AllArgsConstructor + public static class ResponsePathDto { + + private Long modelId; + private String responsePath; + } +} diff --git a/src/main/java/com/kamco/cd/training/train/service/ModelTestMetricsJobService.java b/src/main/java/com/kamco/cd/training/train/service/ModelTestMetricsJobService.java index c5936bc..167d3a4 100644 --- a/src/main/java/com/kamco/cd/training/train/service/ModelTestMetricsJobService.java +++ b/src/main/java/com/kamco/cd/training/train/service/ModelTestMetricsJobService.java @@ -1,6 +1,7 @@ package com.kamco.cd.training.train.service; import com.kamco.cd.training.postgres.core.ModelTestMetricsJobCoreService; +import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.ResponsePathDto; import java.io.BufferedReader; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -35,66 +36,67 @@ public class ModelTestMetricsJobService { return "local".equalsIgnoreCase(profile); } - // @Scheduled(cron = "0 0/10 * * * *") + // @Scheduled(cron = "0 * * * * *") public void findTestValidMetricCsvFiles() { - if (isLocalProfile()) { - return; - } + // if (isLocalProfile()) { + // return; + // } - List modelIds = - modelTestMetricsJobCoreService - .getTestMetricSaveNotYetModelIds(); // TODO: uid, uuid ? 가져오기로 해야함 + List modelIds = + modelTestMetricsJobCoreService.getTestMetricSaveNotYetModelIds(); if (modelIds.isEmpty()) { return; } - String localPath = "C:\\data\\upload\\test.csv"; - try (BufferedReader reader = - Files.newBufferedReader(Paths.get(localPath), StandardCharsets.UTF_8); ) { + for (ResponsePathDto modelInfo : modelIds) { - log.info("### localPath={}", localPath); - CSVParser parser = CSVFormat.DEFAULT.withFirstRecordAsHeader().parse(reader); + String testPath = modelInfo.getResponsePath() + "/metrics/test.csv"; + try (BufferedReader reader = + Files.newBufferedReader(Paths.get(testPath), StandardCharsets.UTF_8); ) { - List batchArgs = new ArrayList<>(); + CSVParser parser = CSVFormat.DEFAULT.withFirstRecordAsHeader().parse(reader); - for (CSVRecord record : parser) { + List batchArgs = new ArrayList<>(); - String model = record.get("model"); - long TP = Long.parseLong(record.get("TP")); - long FP = Long.parseLong(record.get("FP")); - long FN = Long.parseLong(record.get("FN")); - float precision = Float.parseFloat(record.get("precision")); - float recall = Float.parseFloat(record.get("recall")); - float f1_score = Float.parseFloat(record.get("f1_score")); - float accuracy = Float.parseFloat(record.get("accuracy")); - float iou = Float.parseFloat(record.get("iou")); - long detection_count = Long.parseLong(record.get("detection_count")); - long gt_count = Long.parseLong(record.get("gt_count")); + for (CSVRecord record : parser) { - batchArgs.add( - new Object[] { - modelIds.getFirst(), - model, - TP, - FP, - FN, - precision, - recall, - f1_score, - accuracy, - iou, - detection_count, - gt_count - }); + String model = record.get("model"); + long TP = Long.parseLong(record.get("TP")); + long FP = Long.parseLong(record.get("FP")); + long FN = Long.parseLong(record.get("FN")); + float precision = Float.parseFloat(record.get("precision")); + float recall = Float.parseFloat(record.get("recall")); + float f1_score = Float.parseFloat(record.get("f1_score")); + float accuracy = Float.parseFloat(record.get("accuracy")); + float iou = Float.parseFloat(record.get("iou")); + long detection_count = Long.parseLong(record.get("detection_count")); + long gt_count = Long.parseLong(record.get("gt_count")); + + batchArgs.add( + new Object[] { + modelInfo.getModelId(), + model, + TP, + FP, + FN, + precision, + recall, + f1_score, + accuracy, + iou, + detection_count, + gt_count + }); + } + + modelTestMetricsJobCoreService.insertModelMetricsTest(batchArgs); + + } catch (IOException e) { + throw new RuntimeException(e); } - modelTestMetricsJobCoreService.insertModelMetricsTest(batchArgs); - - } catch (IOException e) { - throw new RuntimeException(e); + modelTestMetricsJobCoreService.updateModelMetricsTrainSaveYn(modelInfo.getModelId(), "step2"); } - - modelTestMetricsJobCoreService.updateModelMetricsTrainSaveYn(modelIds.getFirst(), "step2"); } } diff --git a/src/main/java/com/kamco/cd/training/train/service/ModelTrainMetricsJobService.java b/src/main/java/com/kamco/cd/training/train/service/ModelTrainMetricsJobService.java index 319a5fd..2132af7 100644 --- a/src/main/java/com/kamco/cd/training/train/service/ModelTrainMetricsJobService.java +++ b/src/main/java/com/kamco/cd/training/train/service/ModelTrainMetricsJobService.java @@ -1,6 +1,7 @@ package com.kamco.cd.training.train.service; import com.kamco.cd.training.postgres.core.ModelTrainMetricsJobCoreService; +import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.ResponsePathDto; import java.io.BufferedReader; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -35,96 +36,97 @@ public class ModelTrainMetricsJobService { return "local".equalsIgnoreCase(profile); } - // @Scheduled(cron = "0 0/10 * * * *") + // @Scheduled(cron = "0 * * * * *") public void findTrainValidMetricCsvFiles() { - if (isLocalProfile()) { - return; - } + // if (isLocalProfile()) { + // return; + // } - List modelIds = - modelTrainMetricsJobCoreService - .getTrainMetricSaveNotYetModelIds(); // TODO: uid, uuid ? 가져오기로 해야함 + List modelIds = + modelTrainMetricsJobCoreService.getTrainMetricSaveNotYetModelIds(); if (modelIds.isEmpty()) { return; } - String localPath = "C:\\data\\upload\\train.csv"; - try (BufferedReader reader = - Files.newBufferedReader(Paths.get(localPath), StandardCharsets.UTF_8); ) { + for (ResponsePathDto modelInfo : modelIds) { - log.info("### localPath={}", localPath); - CSVParser parser = CSVFormat.DEFAULT.withFirstRecordAsHeader().parse(reader); + String trainPath = modelInfo.getResponsePath() + "/metrics/train.csv"; + try (BufferedReader reader = + Files.newBufferedReader(Paths.get(trainPath), StandardCharsets.UTF_8); ) { - List batchArgs = new ArrayList<>(); + CSVParser parser = CSVFormat.DEFAULT.withFirstRecordAsHeader().parse(reader); - for (CSVRecord record : parser) { + List batchArgs = new ArrayList<>(); - int epoch = Integer.parseInt(record.get("Epoch")); - long iteration = Long.parseLong(record.get("Iteration")); - double Loss = Double.parseDouble(record.get("Loss")); - double LR = Double.parseDouble(record.get("LR")); - float time = Float.parseFloat(record.get("Time")); + for (CSVRecord record : parser) { - batchArgs.add(new Object[] {modelIds.getFirst(), epoch, iteration, Loss, LR, time}); + int epoch = Integer.parseInt(record.get("Epoch")) + 1; // TODO : 나중에 AI 개발 완료되면 -1 하기 + long iteration = Long.parseLong(record.get("Iteration")); + double Loss = Double.parseDouble(record.get("Loss")); + double LR = Double.parseDouble(record.get("LR")); + float time = Float.parseFloat(record.get("Time")); + + batchArgs.add(new Object[] {modelInfo.getModelId(), epoch, iteration, Loss, LR, time}); + } + + modelTrainMetricsJobCoreService.insertModelMetricsTrain(batchArgs); + + } catch (IOException e) { + throw new RuntimeException(e); } - modelTrainMetricsJobCoreService.insertModelMetricsTrain(batchArgs); + String validationPath = modelInfo.getResponsePath() + "/metrics/val.csv"; + try (BufferedReader reader = + Files.newBufferedReader(Paths.get(validationPath), StandardCharsets.UTF_8); ) { - } catch (IOException e) { - throw new RuntimeException(e); - } + CSVParser parser = CSVFormat.DEFAULT.withFirstRecordAsHeader().parse(reader); - String validationPath = "C:\\data\\upload\\val.csv"; - try (BufferedReader reader = - Files.newBufferedReader(Paths.get(validationPath), StandardCharsets.UTF_8); ) { + List batchArgs = new ArrayList<>(); - log.info("### validationPath={}", validationPath); - CSVParser parser = CSVFormat.DEFAULT.withFirstRecordAsHeader().parse(reader); + for (CSVRecord record : parser) { - List batchArgs = new ArrayList<>(); + int epoch = Integer.parseInt(record.get("Epoch")); + float aAcc = Float.parseFloat(record.get("aAcc")); + float mFscore = Float.parseFloat(record.get("mFscore")); + float mPrecision = Float.parseFloat(record.get("mPrecision")); + float mRecall = Float.parseFloat(record.get("mRecall")); + float mIoU = Float.parseFloat(record.get("mIoU")); + float mAcc = Float.parseFloat(record.get("mAcc")); + float changed_fscore = Float.parseFloat(record.get("changed_fscore")); + float changed_precision = Float.parseFloat(record.get("changed_precision")); + float changed_recall = Float.parseFloat(record.get("changed_recall")); + float unchanged_fscore = Float.parseFloat(record.get("unchanged_fscore")); + float unchanged_precision = Float.parseFloat(record.get("unchanged_precision")); + float unchanged_recall = Float.parseFloat(record.get("unchanged_recall")); - for (CSVRecord record : parser) { + batchArgs.add( + new Object[] { + modelInfo.getModelId(), + epoch, + aAcc, + mFscore, + mPrecision, + mRecall, + mIoU, + mAcc, + changed_fscore, + changed_precision, + changed_recall, + unchanged_fscore, + unchanged_precision, + unchanged_recall + }); + } - int epoch = Integer.parseInt(record.get("Epoch")); - float aAcc = Float.parseFloat(record.get("aAcc")); - float mFscore = Float.parseFloat(record.get("mFscore")); - float mPrecision = Float.parseFloat(record.get("mPrecision")); - float mRecall = Float.parseFloat(record.get("mRecall")); - float mIoU = Float.parseFloat(record.get("mIoU")); - float mAcc = Float.parseFloat(record.get("mAcc")); - float changed_fscore = Float.parseFloat(record.get("changed_fscore")); - float changed_precision = Float.parseFloat(record.get("changed_precision")); - float changed_recall = Float.parseFloat(record.get("changed_recall")); - float unchanged_fscore = Float.parseFloat(record.get("unchanged_fscore")); - float unchanged_precision = Float.parseFloat(record.get("unchanged_precision")); - float unchanged_recall = Float.parseFloat(record.get("unchanged_recall")); + modelTrainMetricsJobCoreService.insertModelMetricsValidation(batchArgs); - batchArgs.add( - new Object[] { - modelIds.getFirst(), - epoch, - aAcc, - mFscore, - mPrecision, - mRecall, - mIoU, - mAcc, - changed_fscore, - changed_precision, - changed_recall, - unchanged_fscore, - unchanged_precision, - unchanged_recall - }); + } catch (IOException e) { + throw new RuntimeException(e); } - modelTrainMetricsJobCoreService.insertModelMetricsValidation(batchArgs); - - } catch (IOException e) { - throw new RuntimeException(e); + modelTrainMetricsJobCoreService.updateModelMetricsTrainSaveYn( + modelInfo.getModelId(), "step1"); } - - modelTrainMetricsJobCoreService.updateModelMetricsTrainSaveYn(modelIds.getFirst(), "step1"); } } From 7a22d8ba7315033c855be90b76540ec0bc529781 Mon Sep 17 00:00:00 2001 From: teddy Date: Thu, 12 Feb 2026 15:39:12 +0900 Subject: [PATCH 3/4] =?UTF-8?q?containerName=20=EC=83=9D=EC=84=B1=20?= =?UTF-8?q?=EB=B3=80=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../training/model/dto/ModelTrainMngDto.java | 11 ++++ .../model/service/ModelTrainMngService.java | 19 ++++++ .../model/service/TmpDatasetService.java | 66 +++++++++++++++++++ .../core/ModelTrainMngCoreService.java | 30 +++++++++ .../dataset/DatasetRepositoryCustom.java | 2 + .../dataset/DatasetRepositoryImpl.java | 5 ++ .../model/ModelMngRepositoryImpl.java | 2 +- .../train/service/DockerTrainService.java | 6 +- 8 files changed, 137 insertions(+), 4 deletions(-) create mode 100644 src/main/java/com/kamco/cd/training/model/service/TmpDatasetService.java diff --git a/src/main/java/com/kamco/cd/training/model/dto/ModelTrainMngDto.java b/src/main/java/com/kamco/cd/training/model/dto/ModelTrainMngDto.java index fa7f983..19202e6 100644 --- a/src/main/java/com/kamco/cd/training/model/dto/ModelTrainMngDto.java +++ b/src/main/java/com/kamco/cd/training/model/dto/ModelTrainMngDto.java @@ -155,6 +155,17 @@ public class ModelTrainMngDto { ModelConfig modelConfig; } + @Schema(name = "addReq", description = "모델학습 관리 등록 파라미터") + @Getter + @Setter + @NoArgsConstructor + @AllArgsConstructor + public static class UpdateReq { + + private String requestPath; + private String responsePath; + } + @Getter @Setter public static class TrainingDataset { diff --git a/src/main/java/com/kamco/cd/training/model/service/ModelTrainMngService.java b/src/main/java/com/kamco/cd/training/model/service/ModelTrainMngService.java index 2438c0d..8b4c832 100644 --- a/src/main/java/com/kamco/cd/training/model/service/ModelTrainMngService.java +++ b/src/main/java/com/kamco/cd/training/model/service/ModelTrainMngService.java @@ -13,6 +13,8 @@ import com.kamco.cd.training.model.dto.ModelTrainMngDto.SearchReq; import com.kamco.cd.training.postgres.core.HyperParamCoreService; import com.kamco.cd.training.postgres.core.ModelTrainMngCoreService; import com.kamco.cd.training.train.service.TrainJobService; +import java.io.IOException; +import java.nio.file.Path; import java.util.List; import java.util.UUID; import lombok.RequiredArgsConstructor; @@ -31,6 +33,7 @@ public class ModelTrainMngService { private final TrainJobService trainJobService; private final ModelTrainMngCoreService modelTrainMngCoreService; private final HyperParamCoreService hyperParamCoreService; + private final TmpDatasetService tmpDatasetService; /** * 모델학습 조회 @@ -90,6 +93,22 @@ public class ModelTrainMngService { // 모델 config 저장 modelTrainMngCoreService.saveModelConfig(modelId, req.getModelConfig()); + UUID tmpUuid = UUID.randomUUID(); + String raw = tmpUuid.toString().replace("-", ""); + + List uids = + modelTrainMngCoreService.findDatasetUid(req.getTrainingDataset().getDatasetList()); + + try { + // 데이터셋 심볼링크 생성 + Path path = tmpDatasetService.buildTmpDatasetSymlink(raw, uids); + ModelTrainMngDto.UpdateReq updateReq = new ModelTrainMngDto.UpdateReq(); + updateReq.setRequestPath(path.toString()); + modelTrainMngCoreService.updateModelMaster(modelId, updateReq); + } catch (IOException e) { + throw new RuntimeException(e); + } + // 저장 다 끝난 뒤에 job enqueue if (Boolean.TRUE.equals(req.getIsStart())) { trainJobService.enqueue(modelId); // job 저장 + 이벤트 발행(실행은 AFTER_COMMIT) diff --git a/src/main/java/com/kamco/cd/training/model/service/TmpDatasetService.java b/src/main/java/com/kamco/cd/training/model/service/TmpDatasetService.java new file mode 100644 index 0000000..c58f135 --- /dev/null +++ b/src/main/java/com/kamco/cd/training/model/service/TmpDatasetService.java @@ -0,0 +1,66 @@ +package com.kamco.cd.training.model.service; + +import java.io.IOException; +import java.nio.file.*; +import java.util.List; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +@Slf4j +@Service +@RequiredArgsConstructor +public class TmpDatasetService { + + @Value("${train.requestDir}") + private String requestDir; + + // 환경에 맞게 yml로 빼는 걸 추천 + private final Path BASE = Paths.get(requestDir); + + @Transactional(readOnly = true) + public Path buildTmpDatasetSymlink(String uid, List uids) throws IOException { + Path tmp = BASE.resolve("tmp").resolve(uid); + + // mkdir -p "$TMP"/train/{input1,input2,label} ... + for (String type : List.of("train", "val")) { + for (String part : List.of("input1", "input2", "label")) { + Files.createDirectories(tmp.resolve(type).resolve(part)); + } + } + + for (String id : uids) { + Path srcRoot = BASE.resolve(id); + + for (String type : List.of("train", "val")) { + for (String part : List.of("input1", "input2", "label")) { + + Path srcDir = srcRoot.resolve(type).resolve(part); + + // zsh NULL_GLOB: 폴더가 없으면 그냥 continue + if (!Files.isDirectory(srcDir)) continue; + + try (DirectoryStream stream = Files.newDirectoryStream(srcDir)) { + for (Path f : stream) { + if (!Files.isRegularFile(f)) continue; + + String dstName = id + "__" + f.getFileName(); + Path dst = tmp.resolve(type).resolve(part).resolve(dstName); + + // 이미 있으면 스킵(원하면 덮어쓰기 로직으로 바꿀 수 있음) + if (Files.exists(dst)) continue; + + // ln -s "$f" "$dst" 와 동일 + Files.createSymbolicLink(dst, f.toAbsolutePath()); + } + } + } + } + } + + log.info("tmp dataset created: {}", tmp); + return tmp; + } +} diff --git a/src/main/java/com/kamco/cd/training/postgres/core/ModelTrainMngCoreService.java b/src/main/java/com/kamco/cd/training/postgres/core/ModelTrainMngCoreService.java index b327132..692f475 100644 --- a/src/main/java/com/kamco/cd/training/postgres/core/ModelTrainMngCoreService.java +++ b/src/main/java/com/kamco/cd/training/postgres/core/ModelTrainMngCoreService.java @@ -149,6 +149,26 @@ public class ModelTrainMngCoreService { modelDatasetRepository.save(datasetEntity); } + /** + * 학습모델 수정 + * + * @param modelId + * @param req + */ + public void updateModelMaster(Long modelId, ModelTrainMngDto.UpdateReq req) { + ModelMasterEntity entity = + modelMngRepository + .findById(modelId) + .orElseThrow(() -> new CustomApiException("NOT_FOUND_DATA", HttpStatus.NOT_FOUND)); + if (req.getRequestPath() != null && !req.getRequestPath().isEmpty()) { + entity.setRequestPath(req.getRequestPath()); + } + + if (req.getResponsePath() != null && !req.getResponsePath().isEmpty()) { + entity.setRequestPath(req.getResponsePath()); + } + } + /** * 모델 데이터셋 mapping 테이블 저장 * @@ -467,4 +487,14 @@ public class ModelTrainMngCoreService { entity.setBestEpoch(epoch); } + + /** + * 데이터셋 uid 조회 + * + * @param datasetIds + * @return + */ + public List findDatasetUid(List datasetIds) { + return datasetRepository.findDatasetUid(datasetIds); + } } diff --git a/src/main/java/com/kamco/cd/training/postgres/repository/dataset/DatasetRepositoryCustom.java b/src/main/java/com/kamco/cd/training/postgres/repository/dataset/DatasetRepositoryCustom.java index adf06e9..1916d58 100644 --- a/src/main/java/com/kamco/cd/training/postgres/repository/dataset/DatasetRepositoryCustom.java +++ b/src/main/java/com/kamco/cd/training/postgres/repository/dataset/DatasetRepositoryCustom.java @@ -22,4 +22,6 @@ public interface DatasetRepositoryCustom { Long getDatasetMaxStage(int compareYyyy, int targetYyyy); Long insertDatasetMngData(DatasetMngRegDto mngRegDto); + + List findDatasetUid(List datasetIds); } diff --git a/src/main/java/com/kamco/cd/training/postgres/repository/dataset/DatasetRepositoryImpl.java b/src/main/java/com/kamco/cd/training/postgres/repository/dataset/DatasetRepositoryImpl.java index 4ae1cee..c41fe37 100644 --- a/src/main/java/com/kamco/cd/training/postgres/repository/dataset/DatasetRepositoryImpl.java +++ b/src/main/java/com/kamco/cd/training/postgres/repository/dataset/DatasetRepositoryImpl.java @@ -242,4 +242,9 @@ public class DatasetRepositoryImpl implements DatasetRepositoryCustom { .where(dataset.uid.eq(mngRegDto.getUid())) .fetchOne(); } + + @Override + public List findDatasetUid(List datasetIds) { + return queryFactory.select(dataset.uid).from(dataset).where(dataset.id.in(datasetIds)).fetch(); + } } diff --git a/src/main/java/com/kamco/cd/training/postgres/repository/model/ModelMngRepositoryImpl.java b/src/main/java/com/kamco/cd/training/postgres/repository/model/ModelMngRepositoryImpl.java index 547d837..3ebbafc 100644 --- a/src/main/java/com/kamco/cd/training/postgres/repository/model/ModelMngRepositoryImpl.java +++ b/src/main/java/com/kamco/cd/training/postgres/repository/model/ModelMngRepositoryImpl.java @@ -94,7 +94,7 @@ public class ModelMngRepositoryImpl implements ModelMngRepositoryCustom { .select( Projections.constructor( TrainRunRequest.class, - modelMasterEntity.uuid, // datasetFolder + modelMasterEntity.requestPath, // datasetFolder modelMasterEntity.uuid, // outputFolder modelHyperParamEntity.inputSize, modelHyperParamEntity.cropSize, diff --git a/src/main/java/com/kamco/cd/training/train/service/DockerTrainService.java b/src/main/java/com/kamco/cd/training/train/service/DockerTrainService.java index be9486b..f04e756 100644 --- a/src/main/java/com/kamco/cd/training/train/service/DockerTrainService.java +++ b/src/main/java/com/kamco/cd/training/train/service/DockerTrainService.java @@ -221,7 +221,7 @@ public class DockerTrainService { c.add("/workspace/change-detection-code/train_wrapper.py"); // ===== 기본 파라미터 ===== - addArg(c, "--dataset-folder", "4BDBBDF99D04477A927CC9EBA760B845" /*req.getDatasetFolder()*/); + addArg(c, "--dataset-folder", req.getDatasetFolder()); addArg(c, "--output-folder", req.getOutputFolder()); addArg(c, "--input-size", req.getInputSize()); addArg(c, "--crop-size", req.getCropSize()); @@ -281,8 +281,8 @@ public class DockerTrainService { if (value == null) return; String s = String.valueOf(value).trim(); if (s.isEmpty()) return; - c.add(key); - c.add(s); + + c.add(key + "=" + s); } /** 컨테이너 강제 종료 및 제거 */ From 8e4bea53daf6e100afaea4d0408c9e9b1e5436b6 Mon Sep 17 00:00:00 2001 From: teddy Date: Thu, 12 Feb 2026 15:43:52 +0900 Subject: [PATCH 4/4] =?UTF-8?q?=EB=AA=A8=EB=8D=B8=EB=93=B1=EB=A1=9D=20?= =?UTF-8?q?=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../training/model/ModelTrainMngApiController.java | 5 ++--- .../training/model/service/ModelTrainMngService.java | 12 ++++++------ .../postgres/core/ModelTrainMngCoreService.java | 9 +++++++-- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/src/main/java/com/kamco/cd/training/model/ModelTrainMngApiController.java b/src/main/java/com/kamco/cd/training/model/ModelTrainMngApiController.java index fce7428..6251ae0 100644 --- a/src/main/java/com/kamco/cd/training/model/ModelTrainMngApiController.java +++ b/src/main/java/com/kamco/cd/training/model/ModelTrainMngApiController.java @@ -92,9 +92,8 @@ public class ModelTrainMngApiController { @ApiResponse(responseCode = "500", description = "서버 오류", content = @Content) }) @PostMapping - public ApiResponseDto createModelTrain(@Valid @RequestBody ModelTrainMngDto.AddReq req) { - modelTrainMngService.createModelTrain(req); - return ApiResponseDto.ok("ok"); + public ApiResponseDto createModelTrain(@Valid @RequestBody ModelTrainMngDto.AddReq req) { + return ApiResponseDto.ok(modelTrainMngService.createModelTrain(req)); } @Operation(summary = "모델학습 config 정보 조회", description = "모델학습 config 정보 조회 API") diff --git a/src/main/java/com/kamco/cd/training/model/service/ModelTrainMngService.java b/src/main/java/com/kamco/cd/training/model/service/ModelTrainMngService.java index 8b4c832..3a82566 100644 --- a/src/main/java/com/kamco/cd/training/model/service/ModelTrainMngService.java +++ b/src/main/java/com/kamco/cd/training/model/service/ModelTrainMngService.java @@ -62,7 +62,7 @@ public class ModelTrainMngService { * @return */ @Transactional - public void createModelTrain(ModelTrainMngDto.AddReq req) { + public UUID createModelTrain(ModelTrainMngDto.AddReq req) { HyperParam hyperParam = req.getHyperParam(); HyperParamDto.Basic hyper = new HyperParamDto.Basic(); @@ -81,7 +81,10 @@ public class ModelTrainMngService { } // 모델학습 테이블 저장 - Long modelId = modelTrainMngCoreService.saveModel(req); + ModelTrainMngDto.Basic modelDto = modelTrainMngCoreService.saveModel(req); + + Long modelId = modelDto.getId(); + UUID modelUuid = modelDto.getUuid(); // 모델학습 데이터셋 저장 modelTrainMngCoreService.saveModelDataset(modelId, req); @@ -109,10 +112,7 @@ public class ModelTrainMngService { throw new RuntimeException(e); } - // 저장 다 끝난 뒤에 job enqueue - if (Boolean.TRUE.equals(req.getIsStart())) { - trainJobService.enqueue(modelId); // job 저장 + 이벤트 발행(실행은 AFTER_COMMIT) - } + return modelUuid; } /** diff --git a/src/main/java/com/kamco/cd/training/postgres/core/ModelTrainMngCoreService.java b/src/main/java/com/kamco/cd/training/postgres/core/ModelTrainMngCoreService.java index 692f475..f28c1ef 100644 --- a/src/main/java/com/kamco/cd/training/postgres/core/ModelTrainMngCoreService.java +++ b/src/main/java/com/kamco/cd/training/postgres/core/ModelTrainMngCoreService.java @@ -78,7 +78,7 @@ public class ModelTrainMngCoreService { * @param addReq * @return */ - public Long saveModel(ModelTrainMngDto.AddReq addReq) { + public ModelTrainMngDto.Basic saveModel(ModelTrainMngDto.AddReq addReq) { ModelMasterEntity entity = new ModelMasterEntity(); ModelHyperParamEntity hyperParamEntity = new ModelHyperParamEntity(); @@ -117,7 +117,12 @@ public class ModelTrainMngCoreService { entity.setCreatedUid(userUtil.getId()); ModelMasterEntity resultEntity = modelMngRepository.save(entity); - return resultEntity.getId(); + + ModelTrainMngDto.Basic result = new ModelTrainMngDto.Basic(); + result.setId(resultEntity.getId()); + result.setUuid(resultEntity.getUuid()); + + return result; } /**