패키징 zip파일 만들기 커밋

src/main/java/com/kamco/cd/training/postgres/core/ModelTestMetricsJobCoreService.java

@@ -1,6 +1,8 @@
 package com.kamco.cd.training.postgres.core;
 
 import com.kamco.cd.training.postgres.repository.train.ModelTestMetricsJobRepository;
+import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.ModelMetricJsonDto;
+import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.ModelTestFileName;
 import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.ResponsePathDto;
 import java.util.List;
 import lombok.RequiredArgsConstructor;
@@ -26,4 +28,12 @@ public class ModelTestMetricsJobCoreService {
   public void insertModelMetricsTest(List<Object[]> batchArgs) {
     modelTestMetricsJobRepository.insertModelMetricsTest(batchArgs);
   }
+
+  public ModelMetricJsonDto getTestMetricPackingInfo(Long modelId) {
+    return modelTestMetricsJobRepository.getTestMetricPackingInfo(modelId);
+  }
+
+  public ModelTestFileName findModelTestFileNames(Long modelId) {
+    return modelTestMetricsJobRepository.findModelTestFileNames(modelId);
+  }
 }

src/main/java/com/kamco/cd/training/postgres/repository/train/ModelTestMetricsJobRepositoryCustom.java

@@ -1,5 +1,7 @@
 package com.kamco.cd.training.postgres.repository.train;
 
+import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.ModelMetricJsonDto;
+import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.ModelTestFileName;
 import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.ResponsePathDto;
 import java.util.List;
 
@@ -10,4 +12,8 @@ public interface ModelTestMetricsJobRepositoryCustom {
   List<ResponsePathDto> getTestMetricSaveNotYetModelIds();
 
   void insertModelMetricsTest(List<Object[]> batchArgs);
+
+  ModelMetricJsonDto getTestMetricPackingInfo(Long modelId);
+
+  ModelTestFileName findModelTestFileNames(Long modelId);
 }

src/main/java/com/kamco/cd/training/postgres/repository/train/ModelTestMetricsJobRepositoryImpl.java

@@ -1,9 +1,14 @@
 package com.kamco.cd.training.postgres.repository.train;
 
 import static com.kamco.cd.training.postgres.entity.QModelMasterEntity.modelMasterEntity;
+import static com.kamco.cd.training.postgres.entity.QModelMetricsTestEntity.modelMetricsTestEntity;
+import static com.kamco.cd.training.postgres.entity.QModelMetricsTrainEntity.modelMetricsTrainEntity;
 
 import com.kamco.cd.training.common.enums.TrainStatusType;
 import com.kamco.cd.training.postgres.entity.ModelMetricsTestEntity;
+import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.ModelMetricJsonDto;
+import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.ModelTestFileName;
+import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.Properties;
 import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.ResponsePathDto;
 import com.querydsl.core.types.Projections;
 import com.querydsl.jpa.impl.JPAQueryFactory;
@@ -70,4 +75,43 @@ public class ModelTestMetricsJobRepositoryImpl extends QuerydslRepositorySupport
 
     jdbcTemplate.batchUpdate(sql, batchArgs);
   }
+
+  @Override
+  public ModelMetricJsonDto getTestMetricPackingInfo(Long modelId) {
+    return queryFactory
+        .select(
+            Projections.constructor(
+                ModelMetricJsonDto.class,
+                modelMasterEntity.modelNo,
+                modelMasterEntity.modelVer,
+                Projections.constructor(
+                    Properties.class,
+                    modelMetricsTestEntity.f1Score,
+                    modelMetricsTestEntity.precisions,
+                    modelMetricsTestEntity.recall,
+                    modelMetricsTestEntity.iou,
+                    modelMetricsTrainEntity.loss)))
+        .from(modelMetricsTestEntity)
+        .innerJoin(modelMasterEntity)
+        .on(modelMetricsTestEntity.model.id.eq(modelMasterEntity.id))
+        .innerJoin(modelMetricsTrainEntity)
+        .on(
+            modelMetricsTestEntity.model.eq(modelMetricsTrainEntity.model),
+            modelMasterEntity.bestEpoch.eq(modelMetricsTrainEntity.epoch))
+        .where(modelMetricsTestEntity.model.id.eq(modelId))
+        .fetchOne();
+  }
+
+  @Override
+  public ModelTestFileName findModelTestFileNames(Long modelId) {
+    return queryFactory
+        .select(
+            Projections.constructor(
+                ModelTestFileName.class, modelMetricsTestEntity.model1, modelMasterEntity.modelVer))
+        .from(modelMetricsTestEntity)
+        .innerJoin(modelMasterEntity)
+        .on(modelMetricsTestEntity.model.id.eq(modelMasterEntity.id))
+        .where(modelMetricsTestEntity.model.id.eq(modelId))
+        .fetchOne();
+  }
 }

src/main/java/com/kamco/cd/training/train/dto/ModelTrainMetricsDto.java

@@ -1,6 +1,8 @@
 package com.kamco.cd.training.train.dto;
 
+import com.fasterxml.jackson.annotation.JsonProperty;
 import io.swagger.v3.oas.annotations.media.Schema;
+import java.util.Properties;
 import java.util.UUID;
 import lombok.AllArgsConstructor;
 import lombok.Getter;
@@ -20,4 +22,38 @@ public class ModelTrainMetricsDto {
     private String responsePath;
     private UUID uuid;
   }
+
+  @Getter
+  @AllArgsConstructor
+  public static class ModelMetricJsonDto {
+
+    @JsonProperty("cd_model_type")
+    private String cdModelType;
+
+    @JsonProperty("model_version")
+    private String modelVersion;
+
+    private Properties properties;
+  }
+
+  @Getter
+  @AllArgsConstructor
+  public static class Properties {
+
+    @JsonProperty("f1_score")
+    private Float f1Score;
+
+    private Float precision;
+    private Float recall;
+    private Float loss;
+    private Double iou;
+  }
+
+  @Getter
+  @AllArgsConstructor
+  public static class ModelTestFileName {
+
+    private String bestEpochFileName;
+    private String modelVersion;
+  }
 }

src/main/java/com/kamco/cd/training/train/service/ModelTestMetricsJobService.java

@@ -1,14 +1,26 @@
 package com.kamco.cd.training.train.service;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
 import com.kamco.cd.training.postgres.core.ModelTestMetricsJobCoreService;
+import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.ModelMetricJsonDto;
+import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.ModelTestFileName;
 import com.kamco.cd.training.train.dto.ModelTrainMetricsDto.ResponsePathDto;
 import java.io.BufferedReader;
 import java.io.IOException;
+import java.io.OutputStream;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
+import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.csv.CSVFormat;
@@ -31,6 +43,9 @@ public class ModelTestMetricsJobService {
   @Value("${train.docker.responseDir}")
   private String responseDir;
 
+  @Value("${file.pt-path}")
+  private String ptPathDir;
+
   /**
    * 실행중인 profile
    *
@@ -40,8 +55,8 @@ public class ModelTestMetricsJobService {
     return "local".equalsIgnoreCase(profile);
   }
 
-  //    @Scheduled(cron = "0 * * * * *")
+  //  @Scheduled(cron = "0 * * * * *")
-  public void findTestValidMetricCsvFiles() {
+  public void findTestValidMetricCsvFiles() throws IOException {
     //    if (isLocalProfile()) {
     //      return;
     //    }
@@ -100,7 +115,97 @@ public class ModelTestMetricsJobService {
         throw new RuntimeException(e);
       }
 
+      // 패키징할 파일 만들기
+      ModelMetricJsonDto jsonDto =
+          modelTestMetricsJobCoreService.getTestMetricPackingInfo(modelInfo.getModelId());
+      try {
+        writeJsonFile(
+            jsonDto,
+            Paths.get(
+                responseDir
+                    + "/"
+                    + modelInfo.getUuid()
+                    + "/"
+                    + jsonDto.getModelVersion()
+                    + ".json"));
+      } catch (IOException e) {
+        throw new RuntimeException(e);
+      }
+
+      Path responsePath = Paths.get(responseDir + "/" + modelInfo.getUuid());
+
+      ModelTestFileName fileInfo =
+          modelTestMetricsJobCoreService.findModelTestFileNames(modelInfo.getModelId());
+
+      Path zipPath =
+          Paths.get(
+              responseDir + "/" + modelInfo.getUuid() + "/" + fileInfo.getModelVersion() + ".zip");
+      Set<String> targetNames =
+          Set.of(
+              "model_config.py",
+              fileInfo.getBestEpochFileName() + ".pth",
+              fileInfo.getModelVersion() + ".json");
+
+      List<Path> files = new ArrayList<>();
+      try (Stream<Path> s = Files.list(responsePath)) {
+        files.addAll(
+            s.filter(Files::isRegularFile)
+                .filter(p -> targetNames.contains(p.getFileName().toString()))
+                .collect(Collectors.toList()));
+      } catch (IOException e) {
+        throw new RuntimeException(e);
+      }
+
+      try (Stream<Path> s = Files.list(Path.of(ptPathDir))) {
+        files.addAll(
+            s.filter(Files::isRegularFile)
+                .limit(1) // yolov8_6th-6m.pt 파일 1개만
+                .collect(Collectors.toList()));
+      } catch (IOException e) {
+        throw new RuntimeException(e);
+      }
+
+      zipFiles(files, zipPath);
       modelTestMetricsJobCoreService.updateModelMetricsTrainSaveYn(modelInfo.getModelId(), "step2");
     }
   }
+
+  private void writeJsonFile(Object data, Path outputPath) throws IOException {
+
+    Path parent = outputPath.getParent();
+
+    if (parent != null) {
+      Files.createDirectories(parent);
+    }
+
+    ObjectMapper objectMapper = new ObjectMapper();
+    objectMapper.enable(SerializationFeature.INDENT_OUTPUT);
+
+    try (OutputStream os =
+        Files.newOutputStream(
+            outputPath, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING)) {
+      objectMapper.writeValue(os, data);
+    }
+  }
+
+  private void zipFiles(List<Path> files, Path zipPath) throws IOException {
+
+    Path parent = zipPath.getParent();
+    if (parent != null) {
+      Files.createDirectories(parent);
+    }
+
+    try (ZipOutputStream zos = new ZipOutputStream(Files.newOutputStream(zipPath))) {
+
+      for (Path file : files) {
+
+        ZipEntry entry = new ZipEntry(file.getFileName().toString());
+        zos.putNextEntry(entry);
+
+        Files.copy(file, zos);
+
+        zos.closeEntry();
+      }
+    }
+  }
 }

src/main/resources/application-dev.yml

@@ -58,6 +58,9 @@ file:
   dataset-dir: /home/kcomu/data/request/
   dataset-tmp-dir: ${file.dataset-dir}tmp/
 
+  pt-path: /home/kcomu/data/response/v6-cls-checkpoints/
+  pt-FileName: yolov8_6th-6m.pt
+
 train:
   docker:
     image: "kamco-cd-train:latest"

src/main/resources/application-prod.yml

@@ -44,6 +44,9 @@ file:
   dataset-dir: /home/kcomu/data/request/
   dataset-tmp-dir: ${file.dataset-dir}tmp/
 
+  pt-path: /home/kcomu/data/response/v6-cls-checkpoints/
+  pt-FileName: yolov8_6th-6m.pt
+
 train:
   docker:
     image: "kamco-cd-train:latest"