460 lines
16 KiB
Java
460 lines
16 KiB
Java
package com.interplug.qcast.batch;
|
|
|
|
import java.time.Duration;
|
|
import java.util.*;
|
|
import java.util.concurrent.ConcurrentHashMap;
|
|
|
|
import lombok.RequiredArgsConstructor;
|
|
import lombok.extern.slf4j.Slf4j;
|
|
import org.springframework.batch.core.*;
|
|
import org.springframework.batch.core.explore.JobExplorer;
|
|
import org.springframework.batch.core.launch.JobLauncher;
|
|
import org.springframework.batch.core.repository.JobExecutionAlreadyRunningException;
|
|
import org.springframework.batch.core.repository.JobInstanceAlreadyCompleteException;
|
|
import org.springframework.batch.core.repository.JobRepository;
|
|
import org.springframework.batch.core.repository.JobRestartException;
|
|
import org.springframework.beans.factory.annotation.Value;
|
|
import org.springframework.boot.context.event.ApplicationReadyEvent;
|
|
import org.springframework.scheduling.annotation.Scheduled;
|
|
import org.springframework.web.bind.annotation.GetMapping;
|
|
import org.springframework.web.bind.annotation.PathVariable;
|
|
import org.springframework.web.bind.annotation.RestController;
|
|
import org.springframework.context.event.EventListener;
|
|
import java.time.LocalDateTime;
|
|
import java.sql.SQLException;
|
|
|
|
@RestController
|
|
@RequiredArgsConstructor
|
|
@Slf4j
|
|
public class JobLauncherController {
|
|
private final Map<String, Job> jobs; // 여러 Job을 주입받도록 변경
|
|
|
|
private final JobLauncher jobLauncher;
|
|
private final JobExplorer jobExplorer;
|
|
private final JobRepository jobRepository; // 생성자 주입을 위해 필드 추가
|
|
|
|
@Value("${qsp.master-admin-user-batch-url}")
|
|
private String qspInterfaceUrl;
|
|
|
|
@Value("${spring.profiles.scheduler}")
|
|
private String scheduler;
|
|
|
|
/**
|
|
* 서버 시작 시 실행 중(STARTED)인 상태로 남은 Job들을 FAILED 처리
|
|
*/
|
|
@EventListener(ApplicationReadyEvent.class)
|
|
public void resetFailedJobs() {
|
|
log.info("Checking for 'STARTED' jobs to reset after reboot...");
|
|
for (String jobName : jobs.keySet()) {
|
|
Set<JobExecution> runningExecutions = jobExplorer.findRunningJobExecutions(jobName);
|
|
for (JobExecution execution : runningExecutions) {
|
|
execution.setStatus(BatchStatus.FAILED);
|
|
execution.setExitStatus(ExitStatus.FAILED.addExitDescription("Reset on application startup"));
|
|
execution.setEndTime(LocalDateTime.now()); // new Date() 대신 LocalDateTime.now() 사용
|
|
jobRepository.update(execution);
|
|
log.info("Reset job execution {} for job {}", execution.getId(), jobName);
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
* 특정 Job을 매핑으로 실행하는 메소드
|
|
*
|
|
* @param jobName
|
|
* @return
|
|
* @throws JobInstanceAlreadyCompleteException
|
|
* @throws JobExecutionAlreadyRunningException
|
|
* @throws JobParametersInvalidException
|
|
* @throws JobRestartException
|
|
*/
|
|
@GetMapping("/batch/job/{jobName}") // Path Variable로 jobName을 받음
|
|
public Map<String, Object> launchJob(@PathVariable String jobName)
|
|
throws JobInstanceAlreadyCompleteException, JobExecutionAlreadyRunningException,
|
|
JobParametersInvalidException, JobRestartException {
|
|
|
|
|
|
log.info("Manual launch requested for job: {}", jobName);
|
|
Job job = jobs.get(jobName);
|
|
Map<String, Object> resultMap = new HashMap<String, Object>();
|
|
if (job == null) {
|
|
// return "Job " + jobName + " not found";
|
|
resultMap.put("code", "FAILED");
|
|
resultMap.put("message", "Job" + jobName + " not found");
|
|
return resultMap;
|
|
}
|
|
|
|
// 실행 중인 Job 확인 (데이터베이스 기반)
|
|
if (isJobRunning(jobName)) {
|
|
log.warn("Job {} is already running, skipping execution", jobName);
|
|
resultMap.put("code", "FAILED");
|
|
resultMap.put("message", "Job "+ jobName +" is already running, skipping execution");
|
|
return resultMap;
|
|
}
|
|
|
|
try {
|
|
log.info("Starting job: {}", jobName);
|
|
|
|
JobParameters jobParameters = new JobParametersBuilder().addString("jobName", jobName)
|
|
.addDate("time", new Date()).toJobParameters();
|
|
|
|
JobExecution jobExecution = jobLauncher.run(job, jobParameters);
|
|
log.info("Job {} started with execution id {}", jobName, jobExecution.getId());
|
|
|
|
BatchStatus status = jobExecution.getStatus();
|
|
ExitStatus exitStatus = jobExecution.getExitStatus();
|
|
|
|
resultMap.put("code", status.toString());
|
|
resultMap.put("message", exitStatus.getExitDescription());
|
|
|
|
// return "Job " + jobName + " started";
|
|
return resultMap;
|
|
} catch (JobExecutionAlreadyRunningException e) {
|
|
log.warn("Job {} 이 다른 인스턴스에서 이미 실행 중입니다.", jobName);
|
|
resultMap.put("code", "ALREADY_RUNNING");
|
|
resultMap.put("message", "Job " + jobName + " is already running on another instance.");
|
|
return resultMap;
|
|
} catch (Exception e) {
|
|
logExceptionDetails("Manual launch failed", jobName, e);
|
|
if (isDeadlockException(e)) {
|
|
logDeadlockDetails(jobName, e);
|
|
resultMap.put("code", "DEADLOCK");
|
|
resultMap.put("message", "Deadlock detected while executing job " + jobName + ". Please retry.");
|
|
return resultMap;
|
|
}
|
|
resultMap.put("code", "FAILED");
|
|
resultMap.put("message", e.getMessage());
|
|
return resultMap;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Q.CAST 판매점 / 사용자 / 즐겨찾기 / 노출 아이템 동기화 배치
|
|
*
|
|
|
|
*/
|
|
// @Scheduled(cron = "*/5 * * * * *")
|
|
@Scheduled(cron = "0 50 23 * * *")
|
|
public String storeAdditionalJob() throws JobInstanceAlreadyCompleteException, JobExecutionAlreadyRunningException, JobParametersInvalidException, JobRestartException {
|
|
|
|
return executeScheduledJob("storeAdditionalJob");
|
|
|
|
}
|
|
|
|
/**
|
|
* 아이템 동기화 배치
|
|
*
|
|
|
|
*/
|
|
@Scheduled(cron = "0 30 02 * * *")
|
|
public String materialJob() throws JobInstanceAlreadyCompleteException, JobExecutionAlreadyRunningException, JobParametersInvalidException, JobRestartException {
|
|
|
|
return executeScheduledJob("materialJob");
|
|
|
|
}
|
|
|
|
/**
|
|
* BOM 아이템 동기화 배치
|
|
*
|
|
|
|
*/
|
|
@Scheduled(cron = "0 40 02 * * *")
|
|
public String bomJob() throws JobInstanceAlreadyCompleteException, JobExecutionAlreadyRunningException, JobParametersInvalidException, JobRestartException {
|
|
|
|
return executeScheduledJob("bomJob");
|
|
|
|
}
|
|
|
|
/**
|
|
* 영업사원 동기화 배치
|
|
*
|
|
|
|
*/
|
|
@Scheduled(cron = "0 40 03 * * *")
|
|
public String businessChargerJob() throws JobInstanceAlreadyCompleteException, JobExecutionAlreadyRunningException, JobParametersInvalidException, JobRestartException {
|
|
|
|
return executeScheduledJob("businessChargerJob");
|
|
|
|
}
|
|
|
|
/**
|
|
* 관리자 유저 동기화 배치
|
|
*
|
|
|
|
*/
|
|
@Scheduled(cron = "0 30 01 * * *")
|
|
public String adminUserJob() throws JobInstanceAlreadyCompleteException, JobExecutionAlreadyRunningException, JobParametersInvalidException, JobRestartException {
|
|
|
|
return executeScheduledJob("adminUserJob");
|
|
|
|
}
|
|
|
|
/**
|
|
* 가격 동기화 배치
|
|
*
|
|
|
|
*/
|
|
@Scheduled(cron = "0 20 00 * * *")
|
|
public String priceJob() throws JobInstanceAlreadyCompleteException, JobExecutionAlreadyRunningException, JobParametersInvalidException, JobRestartException {
|
|
|
|
return executeScheduledJob("priceJob");
|
|
|
|
}
|
|
|
|
/**
|
|
* 공통코드 M_COMM_H, M_COMM_L 동기화 배치
|
|
*
|
|
|
|
*/
|
|
@Scheduled(cron = "0 10 03 * * *")
|
|
public String commonCodeJob() throws JobInstanceAlreadyCompleteException, JobExecutionAlreadyRunningException, JobParametersInvalidException, JobRestartException {
|
|
|
|
return executeScheduledJob("commonCodeJob");
|
|
|
|
}
|
|
|
|
/**
|
|
* Q.CAST 견적특이사항 / 아이템 표시, 미표시 동기화 배치
|
|
*
|
|
|
|
*/
|
|
@Scheduled(cron = "0 30 23 * * *")
|
|
public String specialNoteDispItemAdditionalInfoJob() throws JobInstanceAlreadyCompleteException, JobExecutionAlreadyRunningException, JobParametersInvalidException, JobRestartException {
|
|
return executeScheduledJob("specialNoteDispItemAdditionalJob");
|
|
|
|
}
|
|
|
|
/**
|
|
* Plan Confrim 동기화 배치
|
|
*
|
|
|
|
*/
|
|
@Scheduled(cron = "0 05 04 * * *")
|
|
public String planConfirmJob() throws JobInstanceAlreadyCompleteException, JobExecutionAlreadyRunningException, JobParametersInvalidException, JobRestartException {
|
|
|
|
return executeScheduledJob("planConfirmJob");
|
|
|
|
}
|
|
|
|
/**
|
|
* 견적서 전송 동기화 배치
|
|
*
|
|
|
|
*/
|
|
@Scheduled(cron = "0 20 04 * * *")
|
|
public String estimateSyncJob() throws JobInstanceAlreadyCompleteException, JobExecutionAlreadyRunningException, JobParametersInvalidException, JobRestartException {
|
|
|
|
return executeScheduledJob("estimateSyncJob");
|
|
|
|
}
|
|
|
|
/**
|
|
* 공통 스케줄러 실행 메소드
|
|
*/
|
|
private String executeScheduledJob(String jobName) throws JobInstanceAlreadyCompleteException, JobExecutionAlreadyRunningException, JobParametersInvalidException, JobRestartException {
|
|
Job job = jobs.get(jobName);
|
|
if (job == null) {
|
|
log.error("Job {} not found", jobName);
|
|
return "Job " + jobName + " not found";
|
|
}
|
|
|
|
// if (!"Y".equals(scheduler) &&
|
|
// !"materialJob".equals(jobName) &&
|
|
// !"commonCodeJob".equals(jobName) &&
|
|
// !"specialNoteDispItemAdditionalJob".equals(jobName) &&
|
|
// !"planConfirmJob".equals(jobName) &&
|
|
// !"estimateSyncJob".equals(jobName) &&
|
|
// !"bomJob".equals(jobName)){
|
|
// log.info("Scheduler disabled, skipping job {}", jobName);
|
|
// return "Scheduler disabled";
|
|
// }
|
|
|
|
// 허용된 작업 목록 정의
|
|
Set<String> allowedJobs = new HashSet<>(Arrays.asList(
|
|
"materialJob",
|
|
"commonCodeJob",
|
|
"specialNoteDispItemAdditionalJob",
|
|
"planConfirmJob",
|
|
"estimateSyncJob",
|
|
"bomJob",
|
|
"storeAdditionalJob",
|
|
"businessChargerJob",
|
|
"adminUserJob"
|
|
));
|
|
|
|
// 스케줄러가 비활성화되어 있고, 허용된 작업이 아닌 경우
|
|
if (!"Y".equals(scheduler) && !allowedJobs.contains(jobName)) {
|
|
log.info("스케줄러가 비활성화되어 작업을 건너뜁니다: {}", jobName);
|
|
return "Scheduler disabled";
|
|
}
|
|
|
|
// 실행 중인 Job 확인 (데이터베이스 기반)
|
|
if (isJobRunning(jobName)) {
|
|
log.warn("Job {} is already running, skipping execution", jobName);
|
|
return "Job already running";
|
|
}
|
|
|
|
// JobParameters jobParameters =
|
|
// new JobParametersBuilder().addDate("time", new Date()).toJobParameters();
|
|
//
|
|
// jobLauncher.run(job, jobParameters);
|
|
//
|
|
// return jobName+ " executed successfully";
|
|
|
|
JobParameters jobParameters = new JobParametersBuilder()
|
|
.addString("jobName", jobName)
|
|
.addDate("time", new Date())
|
|
.toJobParameters();
|
|
|
|
try {
|
|
log.info("Job {} 실행 시작", jobName);
|
|
JobExecution jobExecution = jobLauncher.run(job, jobParameters);
|
|
log.info("Job {} completed with execution id {} and status {}",
|
|
jobName, jobExecution.getId(), jobExecution.getExitStatus().getExitCode());
|
|
return jobName + " executed successfully";
|
|
} catch (JobExecutionAlreadyRunningException e) {
|
|
log.warn("Job {} 이 다른 인스턴스에서 이미 실행 중입니다. 스킵합니다.", jobName);
|
|
return "Job " + jobName + " is already running on another instance. Skipped.";
|
|
} catch (Exception e) {
|
|
logExceptionDetails("Scheduled launch failed", jobName, e);
|
|
if (isDeadlockException(e)) {
|
|
logDeadlockDetails(jobName, e);
|
|
return "Deadlock detected while executing job " + jobName + ". Manual retry required.";
|
|
}
|
|
log.error("Job {} 실행 실패", jobName, e);
|
|
return "Error executing job " + jobName + ": " + e.getMessage();
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Job 실행 상태 확인
|
|
*/
|
|
private boolean isJobRunning(String jobName) {
|
|
try {
|
|
log.debug("Checking running executions for job: {}", jobName);
|
|
Set<JobExecution> runningExecutions = jobExplorer.findRunningJobExecutions(jobName);
|
|
if (runningExecutions.isEmpty()) {
|
|
return false;
|
|
}
|
|
|
|
boolean isActuallyBlocked = false;
|
|
for (JobExecution execution : runningExecutions) {
|
|
LocalDateTime startTime = execution.getStartTime();
|
|
if (startTime == null) continue;
|
|
|
|
// 현재 시간과 시작 시간의 차이 계산
|
|
Duration duration = Duration.between(startTime, LocalDateTime.now());
|
|
long hours = duration.toHours();
|
|
|
|
if (hours >= 1) {
|
|
// 1시간 이상 경과한 경우 로그 출력 및 DB 강제 업데이트
|
|
log.warn("Job {} (Execution ID: {}) 가 실행된 지 {}시간이 지났습니다. 상태를 FAILED로 초기화하고 재실행을 시도합니다.",
|
|
jobName, execution.getId(), hours);
|
|
|
|
execution.setStatus(BatchStatus.FAILED);
|
|
execution.setExitStatus(ExitStatus.FAILED.addExitDescription("Forcefully reset: Running for more than 1 hour"));
|
|
execution.setEndTime(LocalDateTime.now());
|
|
jobRepository.update(execution);
|
|
|
|
log.info("Job {} 의 이전 실행 상태가 성공적으로 초기화되었습니다.", jobName);
|
|
} else {
|
|
// 1시간 미만으로 실행 중인 잡이 있다면 진짜 실행 중인 것으로 간주
|
|
log.info("Job {} 가 현재 실행 중입니다. (시작 시간: {}, 경과 시간: {}분)",
|
|
jobName, startTime, duration.toMinutes());
|
|
isActuallyBlocked = true;
|
|
}
|
|
}
|
|
|
|
return isActuallyBlocked;
|
|
} catch (Exception e) {
|
|
log.error("Job 상태 확인 중 오류 발생: {}", e.getMessage());
|
|
return false;
|
|
}
|
|
}
|
|
|
|
private void logExceptionDetails(String context, String jobName, Exception e) {
|
|
Throwable root = e;
|
|
while (root.getCause() != null) {
|
|
root = root.getCause();
|
|
}
|
|
if (root instanceof SQLException) {
|
|
SQLException sqlEx = (SQLException) root;
|
|
log.error("{} for job {}: SQLState={}, errorCode={}, message={}, sqlChain={}",
|
|
context, jobName, sqlEx.getSQLState(), sqlEx.getErrorCode(), sqlEx.getMessage(),
|
|
buildSqlExceptionChain(sqlEx), e);
|
|
} else {
|
|
log.error("{} for job {}: {}", context, jobName, root.getMessage(), e);
|
|
}
|
|
}
|
|
|
|
private void logDeadlockDetails(String jobName, Exception e) {
|
|
log.error("===== [DEADLOCK] Job: {} 데드락 상세 정보 =====", jobName);
|
|
|
|
Throwable current = e;
|
|
int depth = 0;
|
|
while (current != null && depth < 10) {
|
|
log.error("[DEADLOCK] Exception chain [{}]: {} - {}",
|
|
depth, current.getClass().getName(), current.getMessage());
|
|
|
|
if (current instanceof SQLException) {
|
|
SQLException sqlEx = (SQLException) current;
|
|
log.error("[DEADLOCK] SQLState: {}, ErrorCode: {}, Message: {}",
|
|
sqlEx.getSQLState(), sqlEx.getErrorCode(), sqlEx.getMessage());
|
|
|
|
// SQLException 체인의 next exception도 확인
|
|
SQLException nextEx = sqlEx.getNextException();
|
|
int sqlDepth = 0;
|
|
while (nextEx != null && sqlDepth < 10) {
|
|
log.error("[DEADLOCK] SQL NextException [{}]: SQLState={}, ErrorCode={}, Message={}",
|
|
sqlDepth, nextEx.getSQLState(), nextEx.getErrorCode(), nextEx.getMessage());
|
|
nextEx = nextEx.getNextException();
|
|
sqlDepth++;
|
|
}
|
|
}
|
|
current = current.getCause();
|
|
depth++;
|
|
}
|
|
|
|
log.error("[DEADLOCK] Full stack trace:", e);
|
|
log.error("===== [DEADLOCK] Job: {} 데드락 상세 정보 끝 =====", jobName);
|
|
}
|
|
|
|
private boolean isDeadlockException(Throwable throwable) {
|
|
Throwable root = throwable;
|
|
while (root.getCause() != null) {
|
|
root = root.getCause();
|
|
}
|
|
if (root instanceof SQLException) {
|
|
SQLException sqlEx = (SQLException) root;
|
|
if ("40001".equals(sqlEx.getSQLState()) || sqlEx.getErrorCode() == 1205) {
|
|
return true;
|
|
}
|
|
}
|
|
String message = root.getMessage();
|
|
if (message == null) return false;
|
|
String lowerMessage = message.toLowerCase();
|
|
return lowerMessage.contains("deadlock")
|
|
|| message.contains("デッドロック")
|
|
|| message.contains("교착 상태");
|
|
}
|
|
|
|
private String buildSqlExceptionChain(SQLException sqlEx) {
|
|
StringBuilder sb = new StringBuilder();
|
|
SQLException current = sqlEx;
|
|
int depth = 0;
|
|
while (current != null && depth < 10) {
|
|
if (depth > 0) {
|
|
sb.append(" | ");
|
|
}
|
|
sb.append("[").append(current.getClass().getSimpleName())
|
|
.append(" sqlState=").append(current.getSQLState())
|
|
.append(" errorCode=").append(current.getErrorCode())
|
|
.append(" message=").append(current.getMessage()).append("]");
|
|
current = current.getNextException();
|
|
depth++;
|
|
}
|
|
return sb.toString();
|
|
}
|
|
}
|