MVPTeam/kamco-train-api
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

csv 파일 읽는 경로 읽어서 수정, train은 epoch + 1 해서 저장

Browse Source
This commit is contained in:
Gina(박가연)
2026-02-12 15:24:30 +09:00
parent b451f697bc
commit 2df4a7a80b
9 changed files with 160 additions and 123 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
src/main/java/com/kamco/cd/training/train/dto/ModelTrainMetricsDto.java Normal file
View File

@@ -0,0 +1,21 @@
package com.kamco.cd.training.train.dto;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
public class ModelTrainMetricsDto {
@Schema(name = "ResponsePathDto", description = "AI 결과 저장된 path 경로 정보")
@Getter
@Setter
@NoArgsConstructor
@AllArgsConstructor
public static class ResponsePathDto {
private Long modelId;
private String responsePath;
}
}

94
src/main/java/com/kamco/cd/training/train/service/ModelTestMetricsJobService.java
View File

@@ -1,6 +1,7 @@
package com.kamco.cd.training.train.service;
import com.kamco.cd.training.postgres.core.ModelTestMetricsJobCoreService;
import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.ResponsePathDto;
import java.io.BufferedReader;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
@@ -35,66 +36,67 @@ public class ModelTestMetricsJobService {
return "local".equalsIgnoreCase(profile);
}
// @Scheduled(cron = "0 0/10 * * * *")
// @Scheduled(cron = "0 * * * * *")
public void findTestValidMetricCsvFiles() {
if (isLocalProfile()) {
return;
}
// if (isLocalProfile()) {
// return;
// }
List<Long> modelIds =
modelTestMetricsJobCoreService
.getTestMetricSaveNotYetModelIds(); // TODO: uid, uuid ? 가져오기로 해야함
List<ResponsePathDto> modelIds =
modelTestMetricsJobCoreService.getTestMetricSaveNotYetModelIds();
if (modelIds.isEmpty()) {
return;
}
String localPath = "C:\\data\\upload\\test.csv";
try (BufferedReader reader =
Files.newBufferedReader(Paths.get(localPath), StandardCharsets.UTF_8); ) {
for (ResponsePathDto modelInfo : modelIds) {
log.info("### localPath={}", localPath);
CSVParser parser = CSVFormat.DEFAULT.withFirstRecordAsHeader().parse(reader);
String testPath = modelInfo.getResponsePath() + "/metrics/test.csv";
try (BufferedReader reader =
Files.newBufferedReader(Paths.get(testPath), StandardCharsets.UTF_8); ) {
List<Object[]> batchArgs = new ArrayList<>();
CSVParser parser = CSVFormat.DEFAULT.withFirstRecordAsHeader().parse(reader);
for (CSVRecord record : parser) {
List<Object[]> batchArgs = new ArrayList<>();
String model = record.get("model");
long TP = Long.parseLong(record.get("TP"));
long FP = Long.parseLong(record.get("FP"));
long FN = Long.parseLong(record.get("FN"));
float precision = Float.parseFloat(record.get("precision"));
float recall = Float.parseFloat(record.get("recall"));
float f1_score = Float.parseFloat(record.get("f1_score"));
float accuracy = Float.parseFloat(record.get("accuracy"));
float iou = Float.parseFloat(record.get("iou"));
long detection_count = Long.parseLong(record.get("detection_count"));
long gt_count = Long.parseLong(record.get("gt_count"));
for (CSVRecord record : parser) {
batchArgs.add(
new Object[] {
modelIds.getFirst(),
model,
TP,
FP,
FN,
precision,
recall,
f1_score,
accuracy,
iou,
detection_count,
gt_count
});
String model = record.get("model");
long TP = Long.parseLong(record.get("TP"));
long FP = Long.parseLong(record.get("FP"));
long FN = Long.parseLong(record.get("FN"));
float precision = Float.parseFloat(record.get("precision"));
float recall = Float.parseFloat(record.get("recall"));
float f1_score = Float.parseFloat(record.get("f1_score"));
float accuracy = Float.parseFloat(record.get("accuracy"));
float iou = Float.parseFloat(record.get("iou"));
long detection_count = Long.parseLong(record.get("detection_count"));
long gt_count = Long.parseLong(record.get("gt_count"));
batchArgs.add(
new Object[] {
modelInfo.getModelId(),
model,
TP,
FP,
FN,
precision,
recall,
f1_score,
accuracy,
iou,
detection_count,
gt_count
});
}
modelTestMetricsJobCoreService.insertModelMetricsTest(batchArgs);
} catch (IOException e) {
throw new RuntimeException(e);
}
modelTestMetricsJobCoreService.insertModelMetricsTest(batchArgs);
} catch (IOException e) {
throw new RuntimeException(e);
modelTestMetricsJobCoreService.updateModelMetricsTrainSaveYn(modelInfo.getModelId(), "step2");
}
modelTestMetricsJobCoreService.updateModelMetricsTrainSaveYn(modelIds.getFirst(), "step2");
}
}

136
src/main/java/com/kamco/cd/training/train/service/ModelTrainMetricsJobService.java
View File

@@ -1,6 +1,7 @@
package com.kamco.cd.training.train.service;
import com.kamco.cd.training.postgres.core.ModelTrainMetricsJobCoreService;
import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.ResponsePathDto;
import java.io.BufferedReader;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
@@ -35,96 +36,97 @@ public class ModelTrainMetricsJobService {
return "local".equalsIgnoreCase(profile);
}
// @Scheduled(cron = "0 0/10 * * * *")
// @Scheduled(cron = "0 * * * * *")
public void findTrainValidMetricCsvFiles() {
if (isLocalProfile()) {
return;
}
// if (isLocalProfile()) {
// return;
// }
List<Long> modelIds =
modelTrainMetricsJobCoreService
.getTrainMetricSaveNotYetModelIds(); // TODO: uid, uuid ? 가져오기로 해야함
List<ResponsePathDto> modelIds =
modelTrainMetricsJobCoreService.getTrainMetricSaveNotYetModelIds();
if (modelIds.isEmpty()) {
return;
}
String localPath = "C:\\data\\upload\\train.csv";
try (BufferedReader reader =
Files.newBufferedReader(Paths.get(localPath), StandardCharsets.UTF_8); ) {
for (ResponsePathDto modelInfo : modelIds) {
log.info("### localPath={}", localPath);
CSVParser parser = CSVFormat.DEFAULT.withFirstRecordAsHeader().parse(reader);
String trainPath = modelInfo.getResponsePath() + "/metrics/train.csv";
try (BufferedReader reader =
Files.newBufferedReader(Paths.get(trainPath), StandardCharsets.UTF_8); ) {
List<Object[]> batchArgs = new ArrayList<>();
CSVParser parser = CSVFormat.DEFAULT.withFirstRecordAsHeader().parse(reader);
for (CSVRecord record : parser) {
List<Object[]> batchArgs = new ArrayList<>();
int epoch = Integer.parseInt(record.get("Epoch"));
long iteration = Long.parseLong(record.get("Iteration"));
double Loss = Double.parseDouble(record.get("Loss"));
double LR = Double.parseDouble(record.get("LR"));
float time = Float.parseFloat(record.get("Time"));
for (CSVRecord record : parser) {
batchArgs.add(new Object[] {modelIds.getFirst(), epoch, iteration, Loss, LR, time});
int epoch = Integer.parseInt(record.get("Epoch")) + 1; // TODO : 나중에 AI 개발 완료되면 -1 하기
long iteration = Long.parseLong(record.get("Iteration"));
double Loss = Double.parseDouble(record.get("Loss"));
double LR = Double.parseDouble(record.get("LR"));
float time = Float.parseFloat(record.get("Time"));
batchArgs.add(new Object[] {modelInfo.getModelId(), epoch, iteration, Loss, LR, time});
}
modelTrainMetricsJobCoreService.insertModelMetricsTrain(batchArgs);
} catch (IOException e) {
throw new RuntimeException(e);
}
modelTrainMetricsJobCoreService.insertModelMetricsTrain(batchArgs);
String validationPath = modelInfo.getResponsePath() + "/metrics/val.csv";
try (BufferedReader reader =
Files.newBufferedReader(Paths.get(validationPath), StandardCharsets.UTF_8); ) {
} catch (IOException e) {
throw new RuntimeException(e);
}
CSVParser parser = CSVFormat.DEFAULT.withFirstRecordAsHeader().parse(reader);
String validationPath = "C:\\data\\upload\\val.csv";
try (BufferedReader reader =
Files.newBufferedReader(Paths.get(validationPath), StandardCharsets.UTF_8); ) {
List<Object[]> batchArgs = new ArrayList<>();
log.info("### validationPath={}", validationPath);
CSVParser parser = CSVFormat.DEFAULT.withFirstRecordAsHeader().parse(reader);
for (CSVRecord record : parser) {
List<Object[]> batchArgs = new ArrayList<>();
int epoch = Integer.parseInt(record.get("Epoch"));
float aAcc = Float.parseFloat(record.get("aAcc"));
float mFscore = Float.parseFloat(record.get("mFscore"));
float mPrecision = Float.parseFloat(record.get("mPrecision"));
float mRecall = Float.parseFloat(record.get("mRecall"));
float mIoU = Float.parseFloat(record.get("mIoU"));
float mAcc = Float.parseFloat(record.get("mAcc"));
float changed_fscore = Float.parseFloat(record.get("changed_fscore"));
float changed_precision = Float.parseFloat(record.get("changed_precision"));
float changed_recall = Float.parseFloat(record.get("changed_recall"));
float unchanged_fscore = Float.parseFloat(record.get("unchanged_fscore"));
float unchanged_precision = Float.parseFloat(record.get("unchanged_precision"));
float unchanged_recall = Float.parseFloat(record.get("unchanged_recall"));
for (CSVRecord record : parser) {
batchArgs.add(
new Object[] {
modelInfo.getModelId(),
epoch,
aAcc,
mFscore,
mPrecision,
mRecall,
mIoU,
mAcc,
changed_fscore,
changed_precision,
changed_recall,
unchanged_fscore,
unchanged_precision,
unchanged_recall
});
}
int epoch = Integer.parseInt(record.get("Epoch"));
float aAcc = Float.parseFloat(record.get("aAcc"));
float mFscore = Float.parseFloat(record.get("mFscore"));
float mPrecision = Float.parseFloat(record.get("mPrecision"));
float mRecall = Float.parseFloat(record.get("mRecall"));
float mIoU = Float.parseFloat(record.get("mIoU"));
float mAcc = Float.parseFloat(record.get("mAcc"));
float changed_fscore = Float.parseFloat(record.get("changed_fscore"));
float changed_precision = Float.parseFloat(record.get("changed_precision"));
float changed_recall = Float.parseFloat(record.get("changed_recall"));
float unchanged_fscore = Float.parseFloat(record.get("unchanged_fscore"));
float unchanged_precision = Float.parseFloat(record.get("unchanged_precision"));
float unchanged_recall = Float.parseFloat(record.get("unchanged_recall"));
modelTrainMetricsJobCoreService.insertModelMetricsValidation(batchArgs);
batchArgs.add(
new Object[] {
modelIds.getFirst(),
epoch,
aAcc,
mFscore,
mPrecision,
mRecall,
mIoU,
mAcc,
changed_fscore,
changed_precision,
changed_recall,
unchanged_fscore,
unchanged_precision,
unchanged_recall
});
} catch (IOException e) {
throw new RuntimeException(e);
}
modelTrainMetricsJobCoreService.insertModelMetricsValidation(batchArgs);
} catch (IOException e) {
throw new RuntimeException(e);
modelTrainMetricsJobCoreService.updateModelMetricsTrainSaveYn(
modelInfo.getModelId(), "step1");
}
modelTrainMetricsJobCoreService.updateModelMetricsTrainSaveYn(modelIds.getFirst(), "step1");
}
}