패키징 zip파일 만들기 커밋
This commit is contained in:
@@ -1,6 +1,8 @@
|
||||
package com.kamco.cd.training.train.dto;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import java.util.Properties;
|
||||
import java.util.UUID;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Getter;
|
||||
@@ -20,4 +22,38 @@ public class ModelTrainMetricsDto {
|
||||
private String responsePath;
|
||||
private UUID uuid;
|
||||
}
|
||||
|
||||
@Getter
|
||||
@AllArgsConstructor
|
||||
public static class ModelMetricJsonDto {
|
||||
|
||||
@JsonProperty("cd_model_type")
|
||||
private String cdModelType;
|
||||
|
||||
@JsonProperty("model_version")
|
||||
private String modelVersion;
|
||||
|
||||
private Properties properties;
|
||||
}
|
||||
|
||||
@Getter
|
||||
@AllArgsConstructor
|
||||
public static class Properties {
|
||||
|
||||
@JsonProperty("f1_score")
|
||||
private Float f1Score;
|
||||
|
||||
private Float precision;
|
||||
private Float recall;
|
||||
private Float loss;
|
||||
private Double iou;
|
||||
}
|
||||
|
||||
@Getter
|
||||
@AllArgsConstructor
|
||||
public static class ModelTestFileName {
|
||||
|
||||
private String bestEpochFileName;
|
||||
private String modelVersion;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,14 +1,26 @@
|
||||
package com.kamco.cd.training.train.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.SerializationFeature;
|
||||
import com.kamco.cd.training.postgres.core.ModelTestMetricsJobCoreService;
|
||||
import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.ModelMetricJsonDto;
|
||||
import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.ModelTestFileName;
|
||||
import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.ResponsePathDto;
|
||||
import java.io.BufferedReader;
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.nio.file.StandardOpenOption;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipOutputStream;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.csv.CSVFormat;
|
||||
@@ -31,6 +43,9 @@ public class ModelTestMetricsJobService {
|
||||
@Value("${train.docker.responseDir}")
|
||||
private String responseDir;
|
||||
|
||||
@Value("${file.pt-path}")
|
||||
private String ptPathDir;
|
||||
|
||||
/**
|
||||
* 실행중인 profile
|
||||
*
|
||||
@@ -40,8 +55,8 @@ public class ModelTestMetricsJobService {
|
||||
return "local".equalsIgnoreCase(profile);
|
||||
}
|
||||
|
||||
// @Scheduled(cron = "0 * * * * *")
|
||||
public void findTestValidMetricCsvFiles() {
|
||||
// @Scheduled(cron = "0 * * * * *")
|
||||
public void findTestValidMetricCsvFiles() throws IOException {
|
||||
// if (isLocalProfile()) {
|
||||
// return;
|
||||
// }
|
||||
@@ -100,7 +115,97 @@ public class ModelTestMetricsJobService {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
|
||||
// 패키징할 파일 만들기
|
||||
ModelMetricJsonDto jsonDto =
|
||||
modelTestMetricsJobCoreService.getTestMetricPackingInfo(modelInfo.getModelId());
|
||||
try {
|
||||
writeJsonFile(
|
||||
jsonDto,
|
||||
Paths.get(
|
||||
responseDir
|
||||
+ "/"
|
||||
+ modelInfo.getUuid()
|
||||
+ "/"
|
||||
+ jsonDto.getModelVersion()
|
||||
+ ".json"));
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
|
||||
Path responsePath = Paths.get(responseDir + "/" + modelInfo.getUuid());
|
||||
|
||||
ModelTestFileName fileInfo =
|
||||
modelTestMetricsJobCoreService.findModelTestFileNames(modelInfo.getModelId());
|
||||
|
||||
Path zipPath =
|
||||
Paths.get(
|
||||
responseDir + "/" + modelInfo.getUuid() + "/" + fileInfo.getModelVersion() + ".zip");
|
||||
Set<String> targetNames =
|
||||
Set.of(
|
||||
"model_config.py",
|
||||
fileInfo.getBestEpochFileName() + ".pth",
|
||||
fileInfo.getModelVersion() + ".json");
|
||||
|
||||
List<Path> files = new ArrayList<>();
|
||||
try (Stream<Path> s = Files.list(responsePath)) {
|
||||
files.addAll(
|
||||
s.filter(Files::isRegularFile)
|
||||
.filter(p -> targetNames.contains(p.getFileName().toString()))
|
||||
.collect(Collectors.toList()));
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
|
||||
try (Stream<Path> s = Files.list(Path.of(ptPathDir))) {
|
||||
files.addAll(
|
||||
s.filter(Files::isRegularFile)
|
||||
.limit(1) // yolov8_6th-6m.pt 파일 1개만
|
||||
.collect(Collectors.toList()));
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
|
||||
zipFiles(files, zipPath);
|
||||
modelTestMetricsJobCoreService.updateModelMetricsTrainSaveYn(modelInfo.getModelId(), "step2");
|
||||
}
|
||||
}
|
||||
|
||||
private void writeJsonFile(Object data, Path outputPath) throws IOException {
|
||||
|
||||
Path parent = outputPath.getParent();
|
||||
|
||||
if (parent != null) {
|
||||
Files.createDirectories(parent);
|
||||
}
|
||||
|
||||
ObjectMapper objectMapper = new ObjectMapper();
|
||||
objectMapper.enable(SerializationFeature.INDENT_OUTPUT);
|
||||
|
||||
try (OutputStream os =
|
||||
Files.newOutputStream(
|
||||
outputPath, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING)) {
|
||||
objectMapper.writeValue(os, data);
|
||||
}
|
||||
}
|
||||
|
||||
private void zipFiles(List<Path> files, Path zipPath) throws IOException {
|
||||
|
||||
Path parent = zipPath.getParent();
|
||||
if (parent != null) {
|
||||
Files.createDirectories(parent);
|
||||
}
|
||||
|
||||
try (ZipOutputStream zos = new ZipOutputStream(Files.newOutputStream(zipPath))) {
|
||||
|
||||
for (Path file : files) {
|
||||
|
||||
ZipEntry entry = new ZipEntry(file.getFileName().toString());
|
||||
zos.putNextEntry(entry);
|
||||
|
||||
Files.copy(file, zos);
|
||||
|
||||
zos.closeEntry();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user