MVPTeam/kamco-train-api
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

학습데이터 등록 로직 커밋

Browse Source
This commit is contained in:
Gina(박가연)
2026-02-10 16:15:21 +09:00
parent 89744d2aa1
commit 0e9fa80092
9 changed files with 331 additions and 148 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
src/main/java/com/kamco/cd/training/dataset/dto/DatasetDto.java
View File

@@ -313,24 +313,12 @@ public class DatasetDto {
@NoArgsConstructor
@AllArgsConstructor
public static class AddReq {
private Long stage;
private String title;
private String fileName;
private String filePath;
private Long fileSize;
private String memo;
}
@Getter
@Setter
@NoArgsConstructor
@AllArgsConstructor
@Builder
public static class DatasetFileDto {
private String fileName;
private String filePath;
}
@Getter
@Setter
@NoArgsConstructor

1
src/main/java/com/kamco/cd/training/dataset/dto/DatasetObjDto.java
View File

@@ -145,6 +145,7 @@ public class DatasetObjDto {
private String comparePath;
private String targetPath;
private String labelPath;
private String geojsonPath;
private String mapSheetNum;
private JsonNode geojson;
private String fileName;

193
src/main/java/com/kamco/cd/training/dataset/service/DatasetService.java
View File

@@ -2,6 +2,8 @@ package com.kamco.cd.training.dataset.service;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.kamco.cd.training.common.enums.LearnDataType;
import com.kamco.cd.training.common.exception.CustomApiException;
import com.kamco.cd.training.common.service.FormatStorage;
@@ -156,9 +158,10 @@ public class DatasetService {
}
}
@Transactional
public ResponseObj insertDataset(@Valid AddReq addReq) {
Long datasetUid = 0L; // master id 값, 등록하면서 가져올 예정
Long datasetUid = null; // master id 값, 등록하면서 가져올 예정
try {
// 압축 해제
@@ -166,74 +169,27 @@ public class DatasetService {
// 해제한 폴더 읽어서 데이터 저장
List<Map<String, Object>> list =
getUnzipDatasetFiles(addReq.getFilePath() + addReq.getFileName().replace(".zip", ""));
getUnzipDatasetFiles(
addReq.getFilePath() + addReq.getFileName().replace(".zip", ""), "train");
int idx = 0;
for (Map<String, Object> map : list) {
String comparePath = (String) map.get("input1");
String targetPath = (String) map.get("input2");
String labelPath = (String) map.get("label");
JsonNode json = (JsonNode) map.get("label-json");
String fileName = Paths.get(comparePath).getFileName().toString();
String[] fileNameStr = fileName.split("_");
String compareYyyy = fileNameStr[1];
String targetYyyy = fileNameStr[2];
String mapSheetNum = fileNameStr[3];
if (idx == 0) {
String title = compareYyyy + "-" + targetYyyy;
String dataType = LearnDataType.PRODUCTION.getId(); // 만들어 넣는 건 다 제작
Long stage =
datasetCoreService.getDatasetMaxStage(
Integer.parseInt(compareYyyy), Integer.parseInt(targetYyyy))
+ 1;
String uid = Paths.get(addReq.getFilePath()).getParent().getFileName().toString();
DatasetMngRegDto mngRegDto =
DatasetMngRegDto.builder()
.uid(uid)
.dataType(dataType)
.compareYyyy(Integer.parseInt(compareYyyy))
.targetYyyy(Integer.parseInt(targetYyyy))
.title(title)
.memo(addReq.getMemo())
.roundNo(stage)
.totalSize(addReq.getFileSize())
.datasetPath(addReq.getFilePath())
.build();
datasetUid = datasetCoreService.insertDatasetMngData(mngRegDto); // tb_dataset 에 insert
}
// datasetUid 로 obj 도 등록하기
// Json 갯수만큼 for문 돌려서 insert 해야 함
for (JsonNode feature : json.path("features")) {
JsonNode prop = feature.path("properties");
String compareClassCd = prop.path("before").asText(null);
String targetClassCd = prop.path("after").asText(null);
DatasetObjRegDto objRegDto =
DatasetObjRegDto.builder()
.datasetUid(datasetUid)
.compareYyyy(Integer.parseInt(compareYyyy))
.compareClassCd(compareClassCd)
.targetYyyy(Integer.parseInt(targetYyyy))
.targetClassCd(targetClassCd)
.comparePath(comparePath)
.targetPath(targetPath)
.labelPath(labelPath)
.mapSheetNum(mapSheetNum)
.geojson(feature)
.fileName(fileName)
.build();
datasetCoreService.insertDatasetObj(objRegDto);
}
datasetUid =
this.insertTrainTestData(map, addReq, idx, datasetUid, "train"); // train 데이터 insert
idx++;
}
List<Map<String, Object>> testList =
getUnzipDatasetFiles(
addReq.getFilePath() + addReq.getFileName().replace(".zip", ""), "test");
int testIdx = 0;
for (Map<String, Object> test : testList) {
datasetUid =
this.insertTrainTestData(test, addReq, testIdx, datasetUid, "test"); // test 데이터 insert
testIdx++;
}
} catch (IOException e) {
log.error(e.getMessage());
return new ResponseObj(ApiResponseCode.INTERNAL_SERVER_ERROR, e.getMessage());
@@ -241,11 +197,105 @@ public class DatasetService {
return new ResponseObj(ApiResponseCode.OK, "업로드 성공하였습니다.");
}
private List<Map<String, Object>> getUnzipDatasetFiles(String unzipRootPath) {
@Transactional
public Long insertTrainTestData(
Map<String, Object> map, AddReq addReq, int idx, Long datasetUid, String subDir) {
ObjectMapper mapper = new ObjectMapper();
String comparePath = (String) map.get("input1");
String targetPath = (String) map.get("input2");
String labelPath = (String) map.get("label");
String geojsonPath = (String) map.get("geojson_path");
Object labelJson = map.get("label-json");
JsonNode json;
Path root = Paths.get(unzipRootPath).resolve("train");
if (labelJson instanceof JsonNode jn) {
json = jn;
} else {
try {
json = mapper.readTree(labelJson.toString());
} catch (IOException e) {
throw new RuntimeException("label_json parse error", e);
}
}
String fileName = Paths.get(comparePath).getFileName().toString();
String[] fileNameStr = fileName.split("_");
String compareYyyy = fileNameStr[1];
String targetYyyy = fileNameStr[2];
String mapSheetNum = fileNameStr[3];
if (idx == 0 && subDir.equals("train")) {
String title = compareYyyy + "-" + targetYyyy; // 제목 : 비교년도-기준년도
String dataType = LearnDataType.PRODUCTION.getId(); // 만들어 넣는 건 다 제작
Long stage =
datasetCoreService.getDatasetMaxStage(
Integer.parseInt(compareYyyy), Integer.parseInt(targetYyyy))
+ 1;
String uid = addReq.getFileName().replace(".zip", "");
DatasetMngRegDto mngRegDto =
DatasetMngRegDto.builder()
.uid(uid)
.dataType(dataType)
.compareYyyy(Integer.parseInt(compareYyyy))
.targetYyyy(Integer.parseInt(targetYyyy))
.title(title)
.memo(addReq.getMemo())
.roundNo(stage)
.totalSize(addReq.getFileSize())
.datasetPath(addReq.getFilePath())
.build();
datasetUid = datasetCoreService.insertDatasetMngData(mngRegDto); // tb_dataset 에 insert
}
// datasetUid 로 obj 도 등록하기
// Json 갯수만큼 for문 돌려서 insert 해야 함, features에 빈값
if (json != null && json.path("features") != null && !json.path("features").isEmpty()) {
for (JsonNode feature : json.path("features")) {
JsonNode prop = feature.path("properties");
String compareClassCd = prop.path("before").asText(null);
String targetClassCd = prop.path("after").asText(null);
// 한 개씩 자른 geojson을 FeatureCollection 으로 만들어서 넣기
ObjectNode root = mapper.createObjectNode();
root.put("type", "FeatureCollection");
ArrayNode features = mapper.createArrayNode();
features.add(feature);
root.set("features", features);
DatasetObjRegDto objRegDto =
DatasetObjRegDto.builder()
.datasetUid(datasetUid)
.compareYyyy(Integer.parseInt(compareYyyy))
.compareClassCd(compareClassCd)
.targetYyyy(Integer.parseInt(targetYyyy))
.targetClassCd(targetClassCd)
.comparePath(comparePath)
.targetPath(targetPath)
.labelPath(labelPath)
.mapSheetNum(mapSheetNum)
.geojson(root)
.geojsonPath(geojsonPath)
.fileName(fileName)
.build();
if (subDir.equals("train")) {
datasetCoreService.insertDatasetObj(objRegDto);
} else {
datasetCoreService.insertDatasetTestObj(objRegDto);
}
}
}
return datasetUid;
}
private List<Map<String, Object>> getUnzipDatasetFiles(String unzipRootPath, String subDir) {
Path root = Paths.get(unzipRootPath).resolve(subDir);
Map<String, Map<String, Object>> grouped = new HashMap<>();
long total_file_cnt = 0;
for (String dirName : LABEL_DIRS) {
Path dir = root.resolve(dirName);
@@ -254,15 +304,6 @@ public class DatasetService {
throw new IllegalStateException("폴더가 존재하지 않습니다 : " + dir);
}
// 파일 갯수는 같아서 비교년도일 때만 file_cnt 를 가지고 오기
// if(dirName.equals("input1")){
// try (Stream<Path> stream = Files.list(dir)) {
// total_file_cnt = stream.filter(Files::isRegularFile).count();
// } catch (IOException e) {
// throw new RuntimeException(e);
// }
// }
try (Stream<Path> stream = Files.list(dir)) {
stream
.filter(Files::isRegularFile)
@@ -282,8 +323,8 @@ public class DatasetService {
if ("label-json".equals(dirName)) {
// json 파일이면 파싱
data.put("label-json", readJson(path));
data.put("geojson_path", path.toAbsolutePath().toString());
} else {
// 나머지는 경로 or 파일명
data.put(dirName, path.toAbsolutePath().toString());
}
});