MVPTeam/kamco-train-api
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

테이블 구조 변경

Browse Source
This commit is contained in:
Gina(박가연)
2026-02-11 18:49:59 +09:00
parent 2cfa2adcf5
commit b6338bce8e
13 changed files with 559 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
build.gradle
View File

@@ -84,7 +84,7 @@ dependencies {
implementation 'org.reflections:reflections:0.10.2'
implementation 'com.jcraft:jsch:0.1.55'
implementation 'org.apache.commons:commons-csv:1.10.0'
}
configurations.configureEach {

31
src/main/java/com/kamco/cd/training/postgres/core/ModelTrainMetricsJobCoreService.java Normal file
View File

@@ -0,0 +1,31 @@
package com.kamco.cd.training.postgres.core;
import com.kamco.cd.training.postgres.repository.schedule.ModelTrainMetricsJobRepository;
import java.util.List;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
@Service
@RequiredArgsConstructor
public class ModelTrainMetricsJobCoreService {
private final ModelTrainMetricsJobRepository modelTrainMetricsJobRepository;
public List<Long> getTrainMetricSaveNotYetModelIds() {
return modelTrainMetricsJobRepository.getTrainMetricSaveNotYetModelIds();
}
public void insertModelMetricsTrain(List<Object[]> batchArgs) {
modelTrainMetricsJobRepository.insertModelMetricsTrain(batchArgs);
}
@Transactional
public void updateModelMetricsTrainSaveYn(Long modelId, String stepNo) {
modelTrainMetricsJobRepository.updateModelMetricsTrainSaveYn(modelId, stepNo);
}
public void insertModelMetricsValidation(List<Object[]> batchArgs) {
modelTrainMetricsJobRepository.insertModelMetricsValidation(batchArgs);
}
}

14
src/main/java/com/kamco/cd/training/postgres/entity/ModelMatricsTestEntity.java → src/main/java/com/kamco/cd/training/postgres/entity/ModelMetricsTestEntity.java
View File

@@ -19,8 +19,8 @@ import org.hibernate.annotations.ColumnDefault;
@Getter
@Setter
@Entity
@Table(name = "tb_model_matrics_test")
public class ModelMatricsTestEntity {
@Table(name = "tb_model_metrics_test")
public class ModelMetricsTestEntity {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
@@ -45,9 +45,6 @@ public class ModelMatricsTestEntity {
@Column(name = "fn")
private Long fn;
@Column(name = "tn")
private Long tn;
@Column(name = "precisions")
private Float precisions;
@@ -63,8 +60,11 @@ public class ModelMatricsTestEntity {
@Column(name = "iou")
private Float iou;
@Column(name = "processed_images")
private Long processedImages;
@Column(name = "detection_count")
private Long detectionCount;
@Column(name = "gt_count")
private Long gtCount;
@ColumnDefault("now()")
@Column(name = "created_dttm")

4
src/main/java/com/kamco/cd/training/postgres/entity/ModelMatricsTrainEntity.java → src/main/java/com/kamco/cd/training/postgres/entity/ModelMetricsTrainEntity.java
View File

@@ -18,8 +18,8 @@ import org.hibernate.annotations.ColumnDefault;
@Getter
@Setter
@Entity
@Table(name = "tb_model_matrics_train")
public class ModelMatricsTrainEntity {
@Table(name = "tb_model_metrics_train")
public class ModelMetricsTrainEntity {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)

4
src/main/java/com/kamco/cd/training/postgres/entity/ModelMatricsValidationEntity.java → src/main/java/com/kamco/cd/training/postgres/entity/ModelMetricsValidationEntity.java
View File

@@ -18,8 +18,8 @@ import org.hibernate.annotations.ColumnDefault;
@Getter
@Setter
@Entity
@Table(name = "tb_model_matrics_validation")
public class ModelMatricsValidationEntity {
@Table(name = "tb_model_metrics_validation")
public class ModelMetricsValidationEntity {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)

9
src/main/java/com/kamco/cd/training/postgres/repository/schedule/ModelTestMetricsJobRepository.java Normal file
View File

@@ -0,0 +1,9 @@
package com.kamco.cd.training.postgres.repository.schedule;
import com.kamco.cd.training.postgres.entity.ModelMetricsTestEntity;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;
@Repository
public interface ModelTestMetricsJobRepository
extends JpaRepository<ModelMetricsTestEntity, Long>, ModelTestMetricsJobRepositoryCustom {}

14
src/main/java/com/kamco/cd/training/postgres/repository/schedule/ModelTestMetricsJobRepositoryCustom.java Normal file
View File

@@ -0,0 +1,14 @@
package com.kamco.cd.training.postgres.repository.schedule;
import java.util.List;
public interface ModelTestMetricsJobRepositoryCustom {
List<Long> getTrainMetricSaveNotYetModelIds();
void insertModelMetricsTrain(List<Object[]> batchArgs);
void updateModelMetricsTrainSaveYn(Long modelId, String stepNo);
void insertModelMetricsValidation(List<Object[]> batchArgs);
}

78
src/main/java/com/kamco/cd/training/postgres/repository/schedule/ModelTestMetricsJobRepositoryImpl.java Normal file
View File

@@ -0,0 +1,78 @@
package com.kamco.cd.training.postgres.repository.schedule;
import static com.kamco.cd.training.postgres.entity.QModelMasterEntity.modelMasterEntity;
import com.kamco.cd.training.common.enums.TrainStatusType;
import com.kamco.cd.training.postgres.entity.ModelMetricsTestEntity;
import com.querydsl.jpa.impl.JPAQueryFactory;
import java.util.List;
import org.springframework.data.jpa.repository.support.QuerydslRepositorySupport;
import org.springframework.jdbc.core.JdbcTemplate;
public class ModelTestMetricsJobRepositoryImpl extends QuerydslRepositorySupport
implements ModelTestMetricsJobRepositoryCustom {
private final JPAQueryFactory queryFactory;
private final JdbcTemplate jdbcTemplate;
public ModelTestMetricsJobRepositoryImpl(
JPAQueryFactory queryFactory, JdbcTemplate jdbcTemplate) {
super(ModelMetricsTestEntity.class);
this.queryFactory = queryFactory;
this.jdbcTemplate = jdbcTemplate;
}
@Override
public List<Long> getTrainMetricSaveNotYetModelIds() {
return queryFactory
.select(modelMasterEntity.id)
.from(modelMasterEntity)
.where(
modelMasterEntity.step1EndDttm.isNotNull(),
modelMasterEntity.step1State.eq(TrainStatusType.COMPLETED.getId()),
modelMasterEntity
.step1MetricSaveYn
.isNull()
.or(modelMasterEntity.step1MetricSaveYn.isFalse()))
.fetch();
}
@Override
public void insertModelMetricsTrain(List<Object[]> batchArgs) {
String sql =
"""
insert into tb_model_matrics_train
(model_id, epoch, iteration, loss, lr, duration_time)
values (?, ?, ?, ?, ?, ?)
""";
jdbcTemplate.batchUpdate(sql, batchArgs);
}
@Override
public void updateModelMetricsTrainSaveYn(Long modelId, String stepNo) {
queryFactory
.update(modelMasterEntity)
.set(
stepNo.equals("step1")
? modelMasterEntity.step1MetricSaveYn
: modelMasterEntity.step2MetricSaveYn,
true)
.where(modelMasterEntity.id.eq(modelId))
.execute();
}
@Override
public void insertModelMetricsValidation(List<Object[]> batchArgs) {
String sql =
"""
insert into tb_model_matrics_validation
(model_id, epoch, a_acc, m_fscore, m_precision, m_recall, m_iou, m_acc, changed_fscore, changed_precision, changed_recall,
unchanged_fscore, unchanged_precision, unchanged_recall
)
values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""";
jdbcTemplate.batchUpdate(sql, batchArgs);
}
}

9
src/main/java/com/kamco/cd/training/postgres/repository/schedule/ModelTrainMetricsJobRepository.java Normal file
View File

@@ -0,0 +1,9 @@
package com.kamco.cd.training.postgres.repository.schedule;
import com.kamco.cd.training.postgres.entity.ModelMetricsTrainEntity;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;
@Repository
public interface ModelTrainMetricsJobRepository
extends JpaRepository<ModelMetricsTrainEntity, Long>, ModelTrainMetricsJobRepositoryCustom {}

14
src/main/java/com/kamco/cd/training/postgres/repository/schedule/ModelTrainMetricsJobRepositoryCustom.java Normal file
View File

@@ -0,0 +1,14 @@
package com.kamco.cd.training.postgres.repository.schedule;
import java.util.List;
public interface ModelTrainMetricsJobRepositoryCustom {
List<Long> getTrainMetricSaveNotYetModelIds();
void insertModelMetricsTrain(List<Object[]> batchArgs);
void updateModelMetricsTrainSaveYn(Long modelId, String stepNo);
void insertModelMetricsValidation(List<Object[]> batchArgs);
}

78
src/main/java/com/kamco/cd/training/postgres/repository/schedule/ModelTrainMetricsJobRepositoryImpl.java Normal file
View File

@@ -0,0 +1,78 @@
package com.kamco.cd.training.postgres.repository.schedule;
import static com.kamco.cd.training.postgres.entity.QModelMasterEntity.modelMasterEntity;
import com.kamco.cd.training.common.enums.TrainStatusType;
import com.kamco.cd.training.postgres.entity.ModelMetricsTrainEntity;
import com.querydsl.jpa.impl.JPAQueryFactory;
import java.util.List;
import org.springframework.data.jpa.repository.support.QuerydslRepositorySupport;
import org.springframework.jdbc.core.JdbcTemplate;
public class ModelTrainMetricsJobRepositoryImpl extends QuerydslRepositorySupport
implements ModelTrainMetricsJobRepositoryCustom {
private final JPAQueryFactory queryFactory;
private final JdbcTemplate jdbcTemplate;
public ModelTrainMetricsJobRepositoryImpl(
JPAQueryFactory queryFactory, JdbcTemplate jdbcTemplate) {
super(ModelMetricsTrainEntity.class);
this.queryFactory = queryFactory;
this.jdbcTemplate = jdbcTemplate;
}
@Override
public List<Long> getTrainMetricSaveNotYetModelIds() {
return queryFactory
.select(modelMasterEntity.id)
.from(modelMasterEntity)
.where(
modelMasterEntity.step1EndDttm.isNotNull(),
modelMasterEntity.step1State.eq(TrainStatusType.COMPLETED.getId()),
modelMasterEntity
.step1MetricSaveYn
.isNull()
.or(modelMasterEntity.step1MetricSaveYn.isFalse()))
.fetch();
}
@Override
public void insertModelMetricsTrain(List<Object[]> batchArgs) {
String sql =
"""
insert into tb_model_matrics_train
(model_id, epoch, iteration, loss, lr, duration_time)
values (?, ?, ?, ?, ?, ?)
""";
jdbcTemplate.batchUpdate(sql, batchArgs);
}
@Override
public void updateModelMetricsTrainSaveYn(Long modelId, String stepNo) {
queryFactory
.update(modelMasterEntity)
.set(
stepNo.equals("step1")
? modelMasterEntity.step1MetricSaveYn
: modelMasterEntity.step2MetricSaveYn,
true)
.where(modelMasterEntity.id.eq(modelId))
.execute();
}
@Override
public void insertModelMetricsValidation(List<Object[]> batchArgs) {
String sql =
"""
insert into tb_model_matrics_validation
(model_id, epoch, a_acc, m_fscore, m_precision, m_recall, m_iou, m_acc, changed_fscore, changed_precision, changed_recall,
unchanged_fscore, unchanged_precision, unchanged_recall
)
values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""";
jdbcTemplate.batchUpdate(sql, batchArgs);
}
}

157
src/main/java/com/kamco/cd/training/schedule/service/ModelTestMetricsJobService.java Normal file
View File

@@ -0,0 +1,157 @@
package com.kamco.cd.training.schedule.service;
import com.jcraft.jsch.ChannelSftp;
import com.jcraft.jsch.JSch;
import com.jcraft.jsch.Session;
import com.kamco.cd.training.postgres.core.ModelTrainMetricsJobCoreService;
import java.io.BufferedReader;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
@Slf4j
@Service
@RequiredArgsConstructor
public class ModelTestMetricsJobService {
private final ModelTrainMetricsJobCoreService modelTrainMetricsJobCoreService;
@Value("${spring.profiles.active}")
private String profile;
/**
* 실행중인 profile
*
* @return
*/
private boolean isLocalProfile() {
return "local".equalsIgnoreCase(profile);
}
@Scheduled(cron = "0 * * * * *")
public void findTestValidMetricCsvFiles() {
// if (isLocalProfile()) {
// return;
// }
List<Long> modelIds =
modelTrainMetricsJobCoreService
.getTrainMetricSaveNotYetModelIds(); // TODO: uid, uuid ? 가져오기로 해야함
if (modelIds.isEmpty()) {
return;
}
Session session = null;
ChannelSftp sftp = null;
JSch jsch = new JSch();
// try {
// session = jsch.getSession("kcomu", "192.168.2.86", 22);
// session.setPassword("Kamco2025!");
//
// Properties config = new Properties();
// config.put("StrictHostKeyChecking", "no");
// session.setConfig(config);
//
// session.connect();
//
// sftp = (ChannelSftp) session.openChannel("sftp");
// sftp.connect();
// InputStream csvInputStream =
// sftp.get("/home/kcomu/data/response/test/metrics/train.csv");
String localPath = "C:\\data\\upload\\train.csv";
try (BufferedReader reader =
Files.newBufferedReader(Paths.get(localPath), StandardCharsets.UTF_8); ) {
log.info("### localPath={}", localPath);
CSVParser parser = CSVFormat.DEFAULT.withFirstRecordAsHeader().parse(reader);
List<Object[]> batchArgs = new ArrayList<>();
for (CSVRecord record : parser) {
int epoch = Integer.parseInt(record.get("Epoch"));
long iteration = Long.parseLong(record.get("Iteration"));
double Loss = Double.parseDouble(record.get("Loss"));
double LR = Double.parseDouble(record.get("LR"));
float time = Float.parseFloat(record.get("Time"));
batchArgs.add(new Object[] {modelIds.getFirst(), epoch, iteration, Loss, LR, time});
}
modelTrainMetricsJobCoreService.insertModelMetricsTrain(batchArgs);
} catch (IOException e) {
throw new RuntimeException(e);
}
String validationPath = "C:\\data\\upload\\val.csv";
try (BufferedReader reader =
Files.newBufferedReader(Paths.get(validationPath), StandardCharsets.UTF_8); ) {
log.info("### validationPath={}", validationPath);
CSVParser parser = CSVFormat.DEFAULT.withFirstRecordAsHeader().parse(reader);
List<Object[]> batchArgs = new ArrayList<>();
for (CSVRecord record : parser) {
int epoch = Integer.parseInt(record.get("Epoch"));
float aAcc = Float.parseFloat(record.get("aAcc"));
float mFscore = Float.parseFloat(record.get("mFscore"));
float mPrecision = Float.parseFloat(record.get("mPrecision"));
float mRecall = Float.parseFloat(record.get("mRecall"));
float mIoU = Float.parseFloat(record.get("mIoU"));
float mAcc = Float.parseFloat(record.get("mAcc"));
float changed_fscore = Float.parseFloat(record.get("changed_fscore"));
float changed_precision = Float.parseFloat(record.get("changed_precision"));
float changed_recall = Float.parseFloat(record.get("changed_recall"));
float unchanged_fscore = Float.parseFloat(record.get("unchanged_fscore"));
float unchanged_precision = Float.parseFloat(record.get("unchanged_precision"));
float unchanged_recall = Float.parseFloat(record.get("unchanged_recall"));
batchArgs.add(
new Object[] {
modelIds.getFirst(),
epoch,
aAcc,
mFscore,
mPrecision,
mRecall,
mIoU,
mAcc,
changed_fscore,
changed_precision,
changed_recall,
unchanged_fscore,
unchanged_precision,
unchanged_recall
});
}
modelTrainMetricsJobCoreService.insertModelMetricsValidation(batchArgs);
} catch (IOException e) {
throw new RuntimeException(e);
}
// } catch (JSchException | SftpException e) {
// throw new RuntimeException(e);
// }
modelTrainMetricsJobCoreService.updateModelMetricsTrainSaveYn(modelIds.getFirst(), "step1");
}
}

157
src/main/java/com/kamco/cd/training/schedule/service/ModelTrainMetricsJobService.java Normal file
View File

@@ -0,0 +1,157 @@
package com.kamco.cd.training.schedule.service;
import com.jcraft.jsch.ChannelSftp;
import com.jcraft.jsch.JSch;
import com.jcraft.jsch.Session;
import com.kamco.cd.training.postgres.core.ModelTrainMetricsJobCoreService;
import java.io.BufferedReader;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
@Slf4j
@Service
@RequiredArgsConstructor
public class ModelTrainMetricsJobService {
private final ModelTrainMetricsJobCoreService modelTrainMetricsJobCoreService;
@Value("${spring.profiles.active}")
private String profile;
/**
* 실행중인 profile
*
* @return
*/
private boolean isLocalProfile() {
return "local".equalsIgnoreCase(profile);
}
@Scheduled(cron = "0 0/5 * * * *")
public void findTrainValidMetricCsvFiles() {
if (isLocalProfile()) {
return;
}
List<Long> modelIds =
modelTrainMetricsJobCoreService
.getTrainMetricSaveNotYetModelIds(); // TODO: uid, uuid ? 가져오기로 해야함
if (modelIds.isEmpty()) {
return;
}
Session session = null;
ChannelSftp sftp = null;
JSch jsch = new JSch();
// try {
// session = jsch.getSession("kcomu", "192.168.2.86", 22);
// session.setPassword("Kamco2025!");
//
// Properties config = new Properties();
// config.put("StrictHostKeyChecking", "no");
// session.setConfig(config);
//
// session.connect();
//
// sftp = (ChannelSftp) session.openChannel("sftp");
// sftp.connect();
// InputStream csvInputStream =
// sftp.get("/home/kcomu/data/response/test/metrics/train.csv");
String localPath = "C:\\data\\upload\\train.csv";
try (BufferedReader reader =
Files.newBufferedReader(Paths.get(localPath), StandardCharsets.UTF_8); ) {
log.info("### localPath={}", localPath);
CSVParser parser = CSVFormat.DEFAULT.withFirstRecordAsHeader().parse(reader);
List<Object[]> batchArgs = new ArrayList<>();
for (CSVRecord record : parser) {
int epoch = Integer.parseInt(record.get("Epoch"));
long iteration = Long.parseLong(record.get("Iteration"));
double Loss = Double.parseDouble(record.get("Loss"));
double LR = Double.parseDouble(record.get("LR"));
float time = Float.parseFloat(record.get("Time"));
batchArgs.add(new Object[] {modelIds.getFirst(), epoch, iteration, Loss, LR, time});
}
modelTrainMetricsJobCoreService.insertModelMetricsTrain(batchArgs);
} catch (IOException e) {
throw new RuntimeException(e);
}
String validationPath = "C:\\data\\upload\\val.csv";
try (BufferedReader reader =
Files.newBufferedReader(Paths.get(validationPath), StandardCharsets.UTF_8); ) {
log.info("### validationPath={}", validationPath);
CSVParser parser = CSVFormat.DEFAULT.withFirstRecordAsHeader().parse(reader);
List<Object[]> batchArgs = new ArrayList<>();
for (CSVRecord record : parser) {
int epoch = Integer.parseInt(record.get("Epoch"));
float aAcc = Float.parseFloat(record.get("aAcc"));
float mFscore = Float.parseFloat(record.get("mFscore"));
float mPrecision = Float.parseFloat(record.get("mPrecision"));
float mRecall = Float.parseFloat(record.get("mRecall"));
float mIoU = Float.parseFloat(record.get("mIoU"));
float mAcc = Float.parseFloat(record.get("mAcc"));
float changed_fscore = Float.parseFloat(record.get("changed_fscore"));
float changed_precision = Float.parseFloat(record.get("changed_precision"));
float changed_recall = Float.parseFloat(record.get("changed_recall"));
float unchanged_fscore = Float.parseFloat(record.get("unchanged_fscore"));
float unchanged_precision = Float.parseFloat(record.get("unchanged_precision"));
float unchanged_recall = Float.parseFloat(record.get("unchanged_recall"));
batchArgs.add(
new Object[] {
modelIds.getFirst(),
epoch,
aAcc,
mFscore,
mPrecision,
mRecall,
mIoU,
mAcc,
changed_fscore,
changed_precision,
changed_recall,
unchanged_fscore,
unchanged_precision,
unchanged_recall
});
}
modelTrainMetricsJobCoreService.insertModelMetricsValidation(batchArgs);
} catch (IOException e) {
throw new RuntimeException(e);
}
// } catch (JSchException | SftpException e) {
// throw new RuntimeException(e);
// }
modelTrainMetricsJobCoreService.updateModelMetricsTrainSaveYn(modelIds.getFirst(), "step1");
}
}