From d7e19abfc9e131126dca8519f18280dc31f9c596 Mon Sep 17 00:00:00 2001 From: "gayoun.park" Date: Wed, 11 Feb 2026 17:06:02 +0900 Subject: [PATCH 1/4] =?UTF-8?q?uploadRate=20=EB=A1=9C=EC=A7=81=20=EC=88=98?= =?UTF-8?q?=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/main/java/com/kamco/cd/training/upload/dto/UploadDto.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/main/java/com/kamco/cd/training/upload/dto/UploadDto.java b/src/main/java/com/kamco/cd/training/upload/dto/UploadDto.java index 99b59c0..cb55752 100644 --- a/src/main/java/com/kamco/cd/training/upload/dto/UploadDto.java +++ b/src/main/java/com/kamco/cd/training/upload/dto/UploadDto.java @@ -199,9 +199,6 @@ public class UploadDto { private String fileName; public double getUploadRate() { - if (this.chunkTotalIndex == 0) { - return 0.0; - } return (double) (this.chunkIndex + 1) / (this.chunkTotalIndex + 1) * 100.0; } } From 2cfa2adcf56ff70e42534753f79394765df563a4 Mon Sep 17 00:00:00 2001 From: "gayoun.park" Date: Wed, 11 Feb 2026 17:21:48 +0900 Subject: [PATCH 2/4] =?UTF-8?q?tb=5Fmodel=5Fmaster=20=EC=BB=AC=EB=9F=BC=20?= =?UTF-8?q?=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cd/training/postgres/entity/ModelMasterEntity.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/main/java/com/kamco/cd/training/postgres/entity/ModelMasterEntity.java b/src/main/java/com/kamco/cd/training/postgres/entity/ModelMasterEntity.java index a06380e..05311dd 100644 --- a/src/main/java/com/kamco/cd/training/postgres/entity/ModelMasterEntity.java +++ b/src/main/java/com/kamco/cd/training/postgres/entity/ModelMasterEntity.java @@ -91,6 +91,12 @@ public class ModelMasterEntity { @Column(name = "before_model_id") private Long beforeModelId; + @Column(name = "step1_metric_save_yn") + private Boolean step1MetricSaveYn; + + @Column(name = "step2_metric_save_yn") + private Boolean step2MetricSaveYn; + public ModelTrainMngDto.Basic toDto() { return new ModelTrainMngDto.Basic( this.id, From b6338bce8e7a2d7f6259adad55ee02a695359d33 Mon Sep 17 00:00:00 2001 From: "gayoun.park" Date: Wed, 11 Feb 2026 18:49:59 +0900 Subject: [PATCH 3/4] =?UTF-8?q?=ED=85=8C=EC=9D=B4=EB=B8=94=20=EA=B5=AC?= =?UTF-8?q?=EC=A1=B0=20=EB=B3=80=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- build.gradle | 2 +- .../core/ModelTrainMetricsJobCoreService.java | 31 ++++ ...ntity.java => ModelMetricsTestEntity.java} | 14 +- ...tity.java => ModelMetricsTrainEntity.java} | 4 +- ...java => ModelMetricsValidationEntity.java} | 4 +- .../ModelTestMetricsJobRepository.java | 9 + .../ModelTestMetricsJobRepositoryCustom.java | 14 ++ .../ModelTestMetricsJobRepositoryImpl.java | 78 +++++++++ .../ModelTrainMetricsJobRepository.java | 9 + .../ModelTrainMetricsJobRepositoryCustom.java | 14 ++ .../ModelTrainMetricsJobRepositoryImpl.java | 78 +++++++++ .../service/ModelTestMetricsJobService.java | 157 ++++++++++++++++++ .../service/ModelTrainMetricsJobService.java | 157 ++++++++++++++++++ 13 files changed, 559 insertions(+), 12 deletions(-) create mode 100644 src/main/java/com/kamco/cd/training/postgres/core/ModelTrainMetricsJobCoreService.java rename src/main/java/com/kamco/cd/training/postgres/entity/{ModelMatricsTestEntity.java => ModelMetricsTestEntity.java} (88%) rename src/main/java/com/kamco/cd/training/postgres/entity/{ModelMatricsTrainEntity.java => ModelMetricsTrainEntity.java} (94%) rename src/main/java/com/kamco/cd/training/postgres/entity/{ModelMatricsValidationEntity.java => ModelMetricsValidationEntity.java} (95%) create mode 100644 src/main/java/com/kamco/cd/training/postgres/repository/schedule/ModelTestMetricsJobRepository.java create mode 100644 src/main/java/com/kamco/cd/training/postgres/repository/schedule/ModelTestMetricsJobRepositoryCustom.java create mode 100644 src/main/java/com/kamco/cd/training/postgres/repository/schedule/ModelTestMetricsJobRepositoryImpl.java create mode 100644 src/main/java/com/kamco/cd/training/postgres/repository/schedule/ModelTrainMetricsJobRepository.java create mode 100644 src/main/java/com/kamco/cd/training/postgres/repository/schedule/ModelTrainMetricsJobRepositoryCustom.java create mode 100644 src/main/java/com/kamco/cd/training/postgres/repository/schedule/ModelTrainMetricsJobRepositoryImpl.java create mode 100644 src/main/java/com/kamco/cd/training/schedule/service/ModelTestMetricsJobService.java create mode 100644 src/main/java/com/kamco/cd/training/schedule/service/ModelTrainMetricsJobService.java diff --git a/build.gradle b/build.gradle index 32a37c3..bb3df95 100644 --- a/build.gradle +++ b/build.gradle @@ -84,7 +84,7 @@ dependencies { implementation 'org.reflections:reflections:0.10.2' implementation 'com.jcraft:jsch:0.1.55' - + implementation 'org.apache.commons:commons-csv:1.10.0' } configurations.configureEach { diff --git a/src/main/java/com/kamco/cd/training/postgres/core/ModelTrainMetricsJobCoreService.java b/src/main/java/com/kamco/cd/training/postgres/core/ModelTrainMetricsJobCoreService.java new file mode 100644 index 0000000..3c6c1f2 --- /dev/null +++ b/src/main/java/com/kamco/cd/training/postgres/core/ModelTrainMetricsJobCoreService.java @@ -0,0 +1,31 @@ +package com.kamco.cd.training.postgres.core; + +import com.kamco.cd.training.postgres.repository.schedule.ModelTrainMetricsJobRepository; +import java.util.List; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +@Service +@RequiredArgsConstructor +public class ModelTrainMetricsJobCoreService { + + private final ModelTrainMetricsJobRepository modelTrainMetricsJobRepository; + + public List getTrainMetricSaveNotYetModelIds() { + return modelTrainMetricsJobRepository.getTrainMetricSaveNotYetModelIds(); + } + + public void insertModelMetricsTrain(List batchArgs) { + modelTrainMetricsJobRepository.insertModelMetricsTrain(batchArgs); + } + + @Transactional + public void updateModelMetricsTrainSaveYn(Long modelId, String stepNo) { + modelTrainMetricsJobRepository.updateModelMetricsTrainSaveYn(modelId, stepNo); + } + + public void insertModelMetricsValidation(List batchArgs) { + modelTrainMetricsJobRepository.insertModelMetricsValidation(batchArgs); + } +} diff --git a/src/main/java/com/kamco/cd/training/postgres/entity/ModelMatricsTestEntity.java b/src/main/java/com/kamco/cd/training/postgres/entity/ModelMetricsTestEntity.java similarity index 88% rename from src/main/java/com/kamco/cd/training/postgres/entity/ModelMatricsTestEntity.java rename to src/main/java/com/kamco/cd/training/postgres/entity/ModelMetricsTestEntity.java index 37e97e3..0909431 100644 --- a/src/main/java/com/kamco/cd/training/postgres/entity/ModelMatricsTestEntity.java +++ b/src/main/java/com/kamco/cd/training/postgres/entity/ModelMetricsTestEntity.java @@ -19,8 +19,8 @@ import org.hibernate.annotations.ColumnDefault; @Getter @Setter @Entity -@Table(name = "tb_model_matrics_test") -public class ModelMatricsTestEntity { +@Table(name = "tb_model_metrics_test") +public class ModelMetricsTestEntity { @Id @GeneratedValue(strategy = GenerationType.IDENTITY) @@ -45,9 +45,6 @@ public class ModelMatricsTestEntity { @Column(name = "fn") private Long fn; - @Column(name = "tn") - private Long tn; - @Column(name = "precisions") private Float precisions; @@ -63,8 +60,11 @@ public class ModelMatricsTestEntity { @Column(name = "iou") private Float iou; - @Column(name = "processed_images") - private Long processedImages; + @Column(name = "detection_count") + private Long detectionCount; + + @Column(name = "gt_count") + private Long gtCount; @ColumnDefault("now()") @Column(name = "created_dttm") diff --git a/src/main/java/com/kamco/cd/training/postgres/entity/ModelMatricsTrainEntity.java b/src/main/java/com/kamco/cd/training/postgres/entity/ModelMetricsTrainEntity.java similarity index 94% rename from src/main/java/com/kamco/cd/training/postgres/entity/ModelMatricsTrainEntity.java rename to src/main/java/com/kamco/cd/training/postgres/entity/ModelMetricsTrainEntity.java index 53537c5..5618c19 100644 --- a/src/main/java/com/kamco/cd/training/postgres/entity/ModelMatricsTrainEntity.java +++ b/src/main/java/com/kamco/cd/training/postgres/entity/ModelMetricsTrainEntity.java @@ -18,8 +18,8 @@ import org.hibernate.annotations.ColumnDefault; @Getter @Setter @Entity -@Table(name = "tb_model_matrics_train") -public class ModelMatricsTrainEntity { +@Table(name = "tb_model_metrics_train") +public class ModelMetricsTrainEntity { @Id @GeneratedValue(strategy = GenerationType.IDENTITY) diff --git a/src/main/java/com/kamco/cd/training/postgres/entity/ModelMatricsValidationEntity.java b/src/main/java/com/kamco/cd/training/postgres/entity/ModelMetricsValidationEntity.java similarity index 95% rename from src/main/java/com/kamco/cd/training/postgres/entity/ModelMatricsValidationEntity.java rename to src/main/java/com/kamco/cd/training/postgres/entity/ModelMetricsValidationEntity.java index bcbdeb4..f79e7b2 100644 --- a/src/main/java/com/kamco/cd/training/postgres/entity/ModelMatricsValidationEntity.java +++ b/src/main/java/com/kamco/cd/training/postgres/entity/ModelMetricsValidationEntity.java @@ -18,8 +18,8 @@ import org.hibernate.annotations.ColumnDefault; @Getter @Setter @Entity -@Table(name = "tb_model_matrics_validation") -public class ModelMatricsValidationEntity { +@Table(name = "tb_model_metrics_validation") +public class ModelMetricsValidationEntity { @Id @GeneratedValue(strategy = GenerationType.IDENTITY) diff --git a/src/main/java/com/kamco/cd/training/postgres/repository/schedule/ModelTestMetricsJobRepository.java b/src/main/java/com/kamco/cd/training/postgres/repository/schedule/ModelTestMetricsJobRepository.java new file mode 100644 index 0000000..48f8e67 --- /dev/null +++ b/src/main/java/com/kamco/cd/training/postgres/repository/schedule/ModelTestMetricsJobRepository.java @@ -0,0 +1,9 @@ +package com.kamco.cd.training.postgres.repository.schedule; + +import com.kamco.cd.training.postgres.entity.ModelMetricsTestEntity; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; + +@Repository +public interface ModelTestMetricsJobRepository + extends JpaRepository, ModelTestMetricsJobRepositoryCustom {} diff --git a/src/main/java/com/kamco/cd/training/postgres/repository/schedule/ModelTestMetricsJobRepositoryCustom.java b/src/main/java/com/kamco/cd/training/postgres/repository/schedule/ModelTestMetricsJobRepositoryCustom.java new file mode 100644 index 0000000..aff4e45 --- /dev/null +++ b/src/main/java/com/kamco/cd/training/postgres/repository/schedule/ModelTestMetricsJobRepositoryCustom.java @@ -0,0 +1,14 @@ +package com.kamco.cd.training.postgres.repository.schedule; + +import java.util.List; + +public interface ModelTestMetricsJobRepositoryCustom { + + List getTrainMetricSaveNotYetModelIds(); + + void insertModelMetricsTrain(List batchArgs); + + void updateModelMetricsTrainSaveYn(Long modelId, String stepNo); + + void insertModelMetricsValidation(List batchArgs); +} diff --git a/src/main/java/com/kamco/cd/training/postgres/repository/schedule/ModelTestMetricsJobRepositoryImpl.java b/src/main/java/com/kamco/cd/training/postgres/repository/schedule/ModelTestMetricsJobRepositoryImpl.java new file mode 100644 index 0000000..7029559 --- /dev/null +++ b/src/main/java/com/kamco/cd/training/postgres/repository/schedule/ModelTestMetricsJobRepositoryImpl.java @@ -0,0 +1,78 @@ +package com.kamco.cd.training.postgres.repository.schedule; + +import static com.kamco.cd.training.postgres.entity.QModelMasterEntity.modelMasterEntity; + +import com.kamco.cd.training.common.enums.TrainStatusType; +import com.kamco.cd.training.postgres.entity.ModelMetricsTestEntity; +import com.querydsl.jpa.impl.JPAQueryFactory; +import java.util.List; +import org.springframework.data.jpa.repository.support.QuerydslRepositorySupport; +import org.springframework.jdbc.core.JdbcTemplate; + +public class ModelTestMetricsJobRepositoryImpl extends QuerydslRepositorySupport + implements ModelTestMetricsJobRepositoryCustom { + + private final JPAQueryFactory queryFactory; + private final JdbcTemplate jdbcTemplate; + + public ModelTestMetricsJobRepositoryImpl( + JPAQueryFactory queryFactory, JdbcTemplate jdbcTemplate) { + super(ModelMetricsTestEntity.class); + this.queryFactory = queryFactory; + this.jdbcTemplate = jdbcTemplate; + } + + @Override + public List getTrainMetricSaveNotYetModelIds() { + return queryFactory + .select(modelMasterEntity.id) + .from(modelMasterEntity) + .where( + modelMasterEntity.step1EndDttm.isNotNull(), + modelMasterEntity.step1State.eq(TrainStatusType.COMPLETED.getId()), + modelMasterEntity + .step1MetricSaveYn + .isNull() + .or(modelMasterEntity.step1MetricSaveYn.isFalse())) + .fetch(); + } + + @Override + public void insertModelMetricsTrain(List batchArgs) { + String sql = + """ + insert into tb_model_matrics_train + (model_id, epoch, iteration, loss, lr, duration_time) + values (?, ?, ?, ?, ?, ?) + """; + + jdbcTemplate.batchUpdate(sql, batchArgs); + } + + @Override + public void updateModelMetricsTrainSaveYn(Long modelId, String stepNo) { + queryFactory + .update(modelMasterEntity) + .set( + stepNo.equals("step1") + ? modelMasterEntity.step1MetricSaveYn + : modelMasterEntity.step2MetricSaveYn, + true) + .where(modelMasterEntity.id.eq(modelId)) + .execute(); + } + + @Override + public void insertModelMetricsValidation(List batchArgs) { + String sql = + """ + insert into tb_model_matrics_validation + (model_id, epoch, a_acc, m_fscore, m_precision, m_recall, m_iou, m_acc, changed_fscore, changed_precision, changed_recall, + unchanged_fscore, unchanged_precision, unchanged_recall + ) + values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """; + + jdbcTemplate.batchUpdate(sql, batchArgs); + } +} diff --git a/src/main/java/com/kamco/cd/training/postgres/repository/schedule/ModelTrainMetricsJobRepository.java b/src/main/java/com/kamco/cd/training/postgres/repository/schedule/ModelTrainMetricsJobRepository.java new file mode 100644 index 0000000..9397e15 --- /dev/null +++ b/src/main/java/com/kamco/cd/training/postgres/repository/schedule/ModelTrainMetricsJobRepository.java @@ -0,0 +1,9 @@ +package com.kamco.cd.training.postgres.repository.schedule; + +import com.kamco.cd.training.postgres.entity.ModelMetricsTrainEntity; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; + +@Repository +public interface ModelTrainMetricsJobRepository + extends JpaRepository, ModelTrainMetricsJobRepositoryCustom {} diff --git a/src/main/java/com/kamco/cd/training/postgres/repository/schedule/ModelTrainMetricsJobRepositoryCustom.java b/src/main/java/com/kamco/cd/training/postgres/repository/schedule/ModelTrainMetricsJobRepositoryCustom.java new file mode 100644 index 0000000..7a8c681 --- /dev/null +++ b/src/main/java/com/kamco/cd/training/postgres/repository/schedule/ModelTrainMetricsJobRepositoryCustom.java @@ -0,0 +1,14 @@ +package com.kamco.cd.training.postgres.repository.schedule; + +import java.util.List; + +public interface ModelTrainMetricsJobRepositoryCustom { + + List getTrainMetricSaveNotYetModelIds(); + + void insertModelMetricsTrain(List batchArgs); + + void updateModelMetricsTrainSaveYn(Long modelId, String stepNo); + + void insertModelMetricsValidation(List batchArgs); +} diff --git a/src/main/java/com/kamco/cd/training/postgres/repository/schedule/ModelTrainMetricsJobRepositoryImpl.java b/src/main/java/com/kamco/cd/training/postgres/repository/schedule/ModelTrainMetricsJobRepositoryImpl.java new file mode 100644 index 0000000..78bbda6 --- /dev/null +++ b/src/main/java/com/kamco/cd/training/postgres/repository/schedule/ModelTrainMetricsJobRepositoryImpl.java @@ -0,0 +1,78 @@ +package com.kamco.cd.training.postgres.repository.schedule; + +import static com.kamco.cd.training.postgres.entity.QModelMasterEntity.modelMasterEntity; + +import com.kamco.cd.training.common.enums.TrainStatusType; +import com.kamco.cd.training.postgres.entity.ModelMetricsTrainEntity; +import com.querydsl.jpa.impl.JPAQueryFactory; +import java.util.List; +import org.springframework.data.jpa.repository.support.QuerydslRepositorySupport; +import org.springframework.jdbc.core.JdbcTemplate; + +public class ModelTrainMetricsJobRepositoryImpl extends QuerydslRepositorySupport + implements ModelTrainMetricsJobRepositoryCustom { + + private final JPAQueryFactory queryFactory; + private final JdbcTemplate jdbcTemplate; + + public ModelTrainMetricsJobRepositoryImpl( + JPAQueryFactory queryFactory, JdbcTemplate jdbcTemplate) { + super(ModelMetricsTrainEntity.class); + this.queryFactory = queryFactory; + this.jdbcTemplate = jdbcTemplate; + } + + @Override + public List getTrainMetricSaveNotYetModelIds() { + return queryFactory + .select(modelMasterEntity.id) + .from(modelMasterEntity) + .where( + modelMasterEntity.step1EndDttm.isNotNull(), + modelMasterEntity.step1State.eq(TrainStatusType.COMPLETED.getId()), + modelMasterEntity + .step1MetricSaveYn + .isNull() + .or(modelMasterEntity.step1MetricSaveYn.isFalse())) + .fetch(); + } + + @Override + public void insertModelMetricsTrain(List batchArgs) { + String sql = + """ + insert into tb_model_matrics_train + (model_id, epoch, iteration, loss, lr, duration_time) + values (?, ?, ?, ?, ?, ?) + """; + + jdbcTemplate.batchUpdate(sql, batchArgs); + } + + @Override + public void updateModelMetricsTrainSaveYn(Long modelId, String stepNo) { + queryFactory + .update(modelMasterEntity) + .set( + stepNo.equals("step1") + ? modelMasterEntity.step1MetricSaveYn + : modelMasterEntity.step2MetricSaveYn, + true) + .where(modelMasterEntity.id.eq(modelId)) + .execute(); + } + + @Override + public void insertModelMetricsValidation(List batchArgs) { + String sql = + """ + insert into tb_model_matrics_validation + (model_id, epoch, a_acc, m_fscore, m_precision, m_recall, m_iou, m_acc, changed_fscore, changed_precision, changed_recall, + unchanged_fscore, unchanged_precision, unchanged_recall + ) + values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """; + + jdbcTemplate.batchUpdate(sql, batchArgs); + } +} diff --git a/src/main/java/com/kamco/cd/training/schedule/service/ModelTestMetricsJobService.java b/src/main/java/com/kamco/cd/training/schedule/service/ModelTestMetricsJobService.java new file mode 100644 index 0000000..75ef117 --- /dev/null +++ b/src/main/java/com/kamco/cd/training/schedule/service/ModelTestMetricsJobService.java @@ -0,0 +1,157 @@ +package com.kamco.cd.training.schedule.service; + +import com.jcraft.jsch.ChannelSftp; +import com.jcraft.jsch.JSch; +import com.jcraft.jsch.Session; +import com.kamco.cd.training.postgres.core.ModelTrainMetricsJobCoreService; +import java.io.BufferedReader; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVRecord; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; + +@Slf4j +@Service +@RequiredArgsConstructor +public class ModelTestMetricsJobService { + + private final ModelTrainMetricsJobCoreService modelTrainMetricsJobCoreService; + + @Value("${spring.profiles.active}") + private String profile; + + /** + * 실행중인 profile + * + * @return + */ + private boolean isLocalProfile() { + return "local".equalsIgnoreCase(profile); + } + + @Scheduled(cron = "0 * * * * *") + public void findTestValidMetricCsvFiles() { + // if (isLocalProfile()) { + // return; + // } + + List modelIds = + modelTrainMetricsJobCoreService + .getTrainMetricSaveNotYetModelIds(); // TODO: uid, uuid ? 가져오기로 해야함 + + if (modelIds.isEmpty()) { + return; + } + + Session session = null; + ChannelSftp sftp = null; + + JSch jsch = new JSch(); + + // try { + // session = jsch.getSession("kcomu", "192.168.2.86", 22); + // session.setPassword("Kamco2025!"); + // + // Properties config = new Properties(); + // config.put("StrictHostKeyChecking", "no"); + // session.setConfig(config); + // + // session.connect(); + // + // sftp = (ChannelSftp) session.openChannel("sftp"); + // sftp.connect(); + + // InputStream csvInputStream = + // sftp.get("/home/kcomu/data/response/test/metrics/train.csv"); + + String localPath = "C:\\data\\upload\\train.csv"; + try (BufferedReader reader = + Files.newBufferedReader(Paths.get(localPath), StandardCharsets.UTF_8); ) { + + log.info("### localPath={}", localPath); + CSVParser parser = CSVFormat.DEFAULT.withFirstRecordAsHeader().parse(reader); + + List batchArgs = new ArrayList<>(); + + for (CSVRecord record : parser) { + + int epoch = Integer.parseInt(record.get("Epoch")); + long iteration = Long.parseLong(record.get("Iteration")); + double Loss = Double.parseDouble(record.get("Loss")); + double LR = Double.parseDouble(record.get("LR")); + float time = Float.parseFloat(record.get("Time")); + + batchArgs.add(new Object[] {modelIds.getFirst(), epoch, iteration, Loss, LR, time}); + } + + modelTrainMetricsJobCoreService.insertModelMetricsTrain(batchArgs); + + } catch (IOException e) { + throw new RuntimeException(e); + } + + String validationPath = "C:\\data\\upload\\val.csv"; + try (BufferedReader reader = + Files.newBufferedReader(Paths.get(validationPath), StandardCharsets.UTF_8); ) { + + log.info("### validationPath={}", validationPath); + CSVParser parser = CSVFormat.DEFAULT.withFirstRecordAsHeader().parse(reader); + + List batchArgs = new ArrayList<>(); + + for (CSVRecord record : parser) { + + int epoch = Integer.parseInt(record.get("Epoch")); + float aAcc = Float.parseFloat(record.get("aAcc")); + float mFscore = Float.parseFloat(record.get("mFscore")); + float mPrecision = Float.parseFloat(record.get("mPrecision")); + float mRecall = Float.parseFloat(record.get("mRecall")); + float mIoU = Float.parseFloat(record.get("mIoU")); + float mAcc = Float.parseFloat(record.get("mAcc")); + float changed_fscore = Float.parseFloat(record.get("changed_fscore")); + float changed_precision = Float.parseFloat(record.get("changed_precision")); + float changed_recall = Float.parseFloat(record.get("changed_recall")); + float unchanged_fscore = Float.parseFloat(record.get("unchanged_fscore")); + float unchanged_precision = Float.parseFloat(record.get("unchanged_precision")); + float unchanged_recall = Float.parseFloat(record.get("unchanged_recall")); + + batchArgs.add( + new Object[] { + modelIds.getFirst(), + epoch, + aAcc, + mFscore, + mPrecision, + mRecall, + mIoU, + mAcc, + changed_fscore, + changed_precision, + changed_recall, + unchanged_fscore, + unchanged_precision, + unchanged_recall + }); + } + + modelTrainMetricsJobCoreService.insertModelMetricsValidation(batchArgs); + + } catch (IOException e) { + throw new RuntimeException(e); + } + // } catch (JSchException | SftpException e) { + // throw new RuntimeException(e); + // } + modelTrainMetricsJobCoreService.updateModelMetricsTrainSaveYn(modelIds.getFirst(), "step1"); + } +} diff --git a/src/main/java/com/kamco/cd/training/schedule/service/ModelTrainMetricsJobService.java b/src/main/java/com/kamco/cd/training/schedule/service/ModelTrainMetricsJobService.java new file mode 100644 index 0000000..65072cc --- /dev/null +++ b/src/main/java/com/kamco/cd/training/schedule/service/ModelTrainMetricsJobService.java @@ -0,0 +1,157 @@ +package com.kamco.cd.training.schedule.service; + +import com.jcraft.jsch.ChannelSftp; +import com.jcraft.jsch.JSch; +import com.jcraft.jsch.Session; +import com.kamco.cd.training.postgres.core.ModelTrainMetricsJobCoreService; +import java.io.BufferedReader; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVRecord; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; + +@Slf4j +@Service +@RequiredArgsConstructor +public class ModelTrainMetricsJobService { + + private final ModelTrainMetricsJobCoreService modelTrainMetricsJobCoreService; + + @Value("${spring.profiles.active}") + private String profile; + + /** + * 실행중인 profile + * + * @return + */ + private boolean isLocalProfile() { + return "local".equalsIgnoreCase(profile); + } + + @Scheduled(cron = "0 0/5 * * * *") + public void findTrainValidMetricCsvFiles() { + if (isLocalProfile()) { + return; + } + + List modelIds = + modelTrainMetricsJobCoreService + .getTrainMetricSaveNotYetModelIds(); // TODO: uid, uuid ? 가져오기로 해야함 + + if (modelIds.isEmpty()) { + return; + } + + Session session = null; + ChannelSftp sftp = null; + + JSch jsch = new JSch(); + + // try { + // session = jsch.getSession("kcomu", "192.168.2.86", 22); + // session.setPassword("Kamco2025!"); + // + // Properties config = new Properties(); + // config.put("StrictHostKeyChecking", "no"); + // session.setConfig(config); + // + // session.connect(); + // + // sftp = (ChannelSftp) session.openChannel("sftp"); + // sftp.connect(); + + // InputStream csvInputStream = + // sftp.get("/home/kcomu/data/response/test/metrics/train.csv"); + + String localPath = "C:\\data\\upload\\train.csv"; + try (BufferedReader reader = + Files.newBufferedReader(Paths.get(localPath), StandardCharsets.UTF_8); ) { + + log.info("### localPath={}", localPath); + CSVParser parser = CSVFormat.DEFAULT.withFirstRecordAsHeader().parse(reader); + + List batchArgs = new ArrayList<>(); + + for (CSVRecord record : parser) { + + int epoch = Integer.parseInt(record.get("Epoch")); + long iteration = Long.parseLong(record.get("Iteration")); + double Loss = Double.parseDouble(record.get("Loss")); + double LR = Double.parseDouble(record.get("LR")); + float time = Float.parseFloat(record.get("Time")); + + batchArgs.add(new Object[] {modelIds.getFirst(), epoch, iteration, Loss, LR, time}); + } + + modelTrainMetricsJobCoreService.insertModelMetricsTrain(batchArgs); + + } catch (IOException e) { + throw new RuntimeException(e); + } + + String validationPath = "C:\\data\\upload\\val.csv"; + try (BufferedReader reader = + Files.newBufferedReader(Paths.get(validationPath), StandardCharsets.UTF_8); ) { + + log.info("### validationPath={}", validationPath); + CSVParser parser = CSVFormat.DEFAULT.withFirstRecordAsHeader().parse(reader); + + List batchArgs = new ArrayList<>(); + + for (CSVRecord record : parser) { + + int epoch = Integer.parseInt(record.get("Epoch")); + float aAcc = Float.parseFloat(record.get("aAcc")); + float mFscore = Float.parseFloat(record.get("mFscore")); + float mPrecision = Float.parseFloat(record.get("mPrecision")); + float mRecall = Float.parseFloat(record.get("mRecall")); + float mIoU = Float.parseFloat(record.get("mIoU")); + float mAcc = Float.parseFloat(record.get("mAcc")); + float changed_fscore = Float.parseFloat(record.get("changed_fscore")); + float changed_precision = Float.parseFloat(record.get("changed_precision")); + float changed_recall = Float.parseFloat(record.get("changed_recall")); + float unchanged_fscore = Float.parseFloat(record.get("unchanged_fscore")); + float unchanged_precision = Float.parseFloat(record.get("unchanged_precision")); + float unchanged_recall = Float.parseFloat(record.get("unchanged_recall")); + + batchArgs.add( + new Object[] { + modelIds.getFirst(), + epoch, + aAcc, + mFscore, + mPrecision, + mRecall, + mIoU, + mAcc, + changed_fscore, + changed_precision, + changed_recall, + unchanged_fscore, + unchanged_precision, + unchanged_recall + }); + } + + modelTrainMetricsJobCoreService.insertModelMetricsValidation(batchArgs); + + } catch (IOException e) { + throw new RuntimeException(e); + } + // } catch (JSchException | SftpException e) { + // throw new RuntimeException(e); + // } + modelTrainMetricsJobCoreService.updateModelMetricsTrainSaveYn(modelIds.getFirst(), "step1"); + } +} From 207cc47f1b3f84f91268edfd3199940d30f51416 Mon Sep 17 00:00:00 2001 From: "gayoun.park" Date: Wed, 11 Feb 2026 18:50:43 +0900 Subject: [PATCH 4/4] =?UTF-8?q?=EC=8A=A4=EC=BC=80=EC=A4=84=20=EC=A3=BC?= =?UTF-8?q?=EC=84=9D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../training/schedule/service/ModelTestMetricsJobService.java | 3 +-- .../training/schedule/service/ModelTrainMetricsJobService.java | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/kamco/cd/training/schedule/service/ModelTestMetricsJobService.java b/src/main/java/com/kamco/cd/training/schedule/service/ModelTestMetricsJobService.java index 75ef117..e0ae083 100644 --- a/src/main/java/com/kamco/cd/training/schedule/service/ModelTestMetricsJobService.java +++ b/src/main/java/com/kamco/cd/training/schedule/service/ModelTestMetricsJobService.java @@ -17,7 +17,6 @@ import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVParser; import org.apache.commons.csv.CSVRecord; import org.springframework.beans.factory.annotation.Value; -import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; @Slf4j @@ -39,7 +38,7 @@ public class ModelTestMetricsJobService { return "local".equalsIgnoreCase(profile); } - @Scheduled(cron = "0 * * * * *") + // @Scheduled(cron = "0 * * * * *") public void findTestValidMetricCsvFiles() { // if (isLocalProfile()) { // return; diff --git a/src/main/java/com/kamco/cd/training/schedule/service/ModelTrainMetricsJobService.java b/src/main/java/com/kamco/cd/training/schedule/service/ModelTrainMetricsJobService.java index 65072cc..a1a9485 100644 --- a/src/main/java/com/kamco/cd/training/schedule/service/ModelTrainMetricsJobService.java +++ b/src/main/java/com/kamco/cd/training/schedule/service/ModelTrainMetricsJobService.java @@ -17,7 +17,6 @@ import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVParser; import org.apache.commons.csv.CSVRecord; import org.springframework.beans.factory.annotation.Value; -import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; @Slf4j @@ -39,7 +38,7 @@ public class ModelTrainMetricsJobService { return "local".equalsIgnoreCase(profile); } - @Scheduled(cron = "0 0/5 * * * *") + // @Scheduled(cron = "0 0/5 * * * *") public void findTrainValidMetricCsvFiles() { if (isLocalProfile()) { return;