#!/bin/bash # ============================================================ # GUARDiA SM | sm_full_check.sh # 대상: 전체 SM 점검 오케스트레이터 # 파라미터: SERVER_TYPE=WEB|WAS|DB|ESB|SEARCH|AGENT|ALL # SCRIPT_BASE=/opt/guardia/itsm/scripts/sm # TIMEOUT=300 (스크립트당 타임아웃 초) # OUTPUT_FORMAT=text|json # ============================================================ set -euo pipefail SERVER_TYPE=${SERVER_TYPE:-ALL} SCRIPT_BASE=${SCRIPT_BASE:-$(dirname "$0")/..} TIMEOUT_SEC=${TIMEOUT:-300} OUTPUT_FORMAT=${OUTPUT_FORMAT:-text} OK="[OK]"; WARN="[WARN]"; CRIT="[CRIT]" SEP="═══════════════════════════════════════════" RESULT=0 START_EPOCH=$(date +%s) echo "======================================================" echo " GUARDiA SM 전체 점검 오케스트레이터" echo " 서버: $(hostname -s) 타입: ${SERVER_TYPE}" echo " 점검 시작: $(date '+%Y-%m-%d %H:%M:%S %Z')" echo "======================================================" # 스크립트 경로 정규화 SCRIPT_BASE=$(realpath "$SCRIPT_BASE" 2>/dev/null || echo "$SCRIPT_BASE") # ── 점검 결과 수집 배열 ─────────────────────────────────── declare -a RESULTS=() declare -A SCRIPT_MAP=( ["system"]="${SCRIPT_BASE}/common/system_health.sh" ["apache"]="${SCRIPT_BASE}/web/web_apache_sm.sh" ["nginx"]="${SCRIPT_BASE}/web/web_nginx_sm.sh" ["webtob"]="${SCRIPT_BASE}/web/web_webtob_sm.sh" ["tomcat"]="${SCRIPT_BASE}/was/was_tomcat_sm.sh" ["jboss"]="${SCRIPT_BASE}/was/was_jboss_sm.sh" ["jeus"]="${SCRIPT_BASE}/was/was_jeus_sm.sh" ["weblogic"]="${SCRIPT_BASE}/was/was_weblogic_sm.sh" ["postgresql"]="${SCRIPT_BASE}/db/db_postgresql_sm.sh" ["oracle"]="${SCRIPT_BASE}/db/db_oracle_sm.sh" ["mysql"]="${SCRIPT_BASE}/db/db_mysql_sm.sh" ["tibero"]="${SCRIPT_BASE}/db/db_tibero_sm.sh" ["esb"]="${SCRIPT_BASE}/esb/esb_check.sh" ["elasticsearch"]="${SCRIPT_BASE}/search/search_elasticsearch_sm.sh" ["solr"]="${SCRIPT_BASE}/search/search_solr_sm.sh" ["pinpoint"]="${SCRIPT_BASE}/agent/agent_pinpoint_sm.sh" ["scouter"]="${SCRIPT_BASE}/agent/agent_scouter_sm.sh" ["crontab"]="${SCRIPT_BASE}/crontab/crontab_sm.sh" ["ping"]="${SCRIPT_BASE}/ping/ping_test.sh" ["log"]="${SCRIPT_BASE}/log/log_analysis.sh" ) # ── 서버 타입별 실행 스크립트 결정 ─────────────────────── # 환경변수로 개별 스크립트 선택 # 예: WEB_ENGINE=nginx DB_ENGINE=postgresql WEB_ENGINE=${WEB_ENGINE:-apache} WAS_ENGINE=${WAS_ENGINE:-tomcat} DB_ENGINE=${DB_ENGINE:-postgresql} ESB_ENGINE=${ESB_ENGINE:-activemq} SEARCH_ENGINE=${SEARCH_ENGINE:-elasticsearch} APM_ENGINE=${APM_ENGINE:-scouter} # 타입별 스크립트 목록 case "${SERVER_TYPE^^}" in WEB) SCRIPTS=("system" "$WEB_ENGINE" "ping" "log") ;; WAS) SCRIPTS=("system" "$WAS_ENGINE" "ping" "log") ;; DB) SCRIPTS=("system" "$DB_ENGINE" "ping" "log") ;; ESB) SCRIPTS=("system" "esb" "ping" "log") ;; SEARCH) SCRIPTS=("system" "$SEARCH_ENGINE" "ping" "log") ;; AGENT) SCRIPTS=("system" "$APM_ENGINE" "ping" "log") ;; ALL) SCRIPTS=("system" "$WEB_ENGINE" "$WAS_ENGINE" "$DB_ENGINE" "esb" \ "$SEARCH_ENGINE" "$APM_ENGINE" "crontab" "ping" "log") ;; CUSTOM) # 개별 지정: CUSTOM_SCRIPTS="system tomcat postgresql ping" IFS=' ' read -ra SCRIPTS <<< "${CUSTOM_SCRIPTS:-system}" ;; *) echo " ${WARN} 알 수 없는 SERVER_TYPE: ${SERVER_TYPE}" echo " 지원: WEB, WAS, DB, ESB, SEARCH, AGENT, ALL, CUSTOM" exit 1 ;; esac echo " 실행할 점검 항목: ${SCRIPTS[*]}" echo "======================================================" # ── 스크립트 순차 실행 + 결과 집계 ────────────────────── RUN_OK=0; RUN_WARN=0; RUN_CRIT=0; RUN_SKIP=0 for SCRIPT_KEY in "${SCRIPTS[@]}"; do SCRIPT_PATH="${SCRIPT_MAP[$SCRIPT_KEY]:-}" echo echo "${SEP}" echo " 점검: ${SCRIPT_KEY} (${SCRIPT_PATH})" echo "${SEP}" if [ -z "$SCRIPT_PATH" ]; then echo " ${WARN} 알 수 없는 스크립트 키: ${SCRIPT_KEY}" RUN_SKIP=$(( RUN_SKIP + 1 )) RESULTS+=("${SCRIPT_KEY}:SKIP") continue fi if [ ! -f "$SCRIPT_PATH" ]; then echo " ${WARN} 스크립트 없음: ${SCRIPT_PATH}" RUN_SKIP=$(( RUN_SKIP + 1 )) RESULTS+=("${SCRIPT_KEY}:SKIP") continue fi chmod +x "$SCRIPT_PATH" 2>/dev/null || true # 타임아웃 실행 EXIT_CODE=0 timeout "$TIMEOUT_SEC" bash "$SCRIPT_PATH" 2>&1 || EXIT_CODE=$? case $EXIT_CODE in 0) echo " → 결과: ${OK}" RUN_OK=$(( RUN_OK + 1 )) RESULTS+=("${SCRIPT_KEY}:OK") ;; 1) echo " → 결과: ${WARN}" RUN_WARN=$(( RUN_WARN + 1 )) RESULTS+=("${SCRIPT_KEY}:WARN") [ $RESULT -lt 1 ] && RESULT=1 ;; 2) echo " → 결과: ${CRIT}" RUN_CRIT=$(( RUN_CRIT + 1 )) RESULTS+=("${SCRIPT_KEY}:CRIT") RESULT=2 ;; 124) echo " → 결과: ${WARN} (타임아웃 ${TIMEOUT_SEC}s)" RUN_WARN=$(( RUN_WARN + 1 )) RESULTS+=("${SCRIPT_KEY}:TIMEOUT") [ $RESULT -lt 1 ] && RESULT=1 ;; *) echo " → 결과: ${WARN} (종료코드: ${EXIT_CODE})" RUN_WARN=$(( RUN_WARN + 1 )) RESULTS+=("${SCRIPT_KEY}:EXIT${EXIT_CODE}") [ $RESULT -lt 1 ] && RESULT=1 ;; esac done END_EPOCH=$(date +%s) ELAPSED=$(( END_EPOCH - START_EPOCH )) # ── 최종 요약 ───────────────────────────────────────────── echo echo "======================================================" echo " GUARDiA SM 전체 점검 완료 요약" echo "======================================================" echo " 서버 : $(hostname -s)" echo " 서버 타입 : ${SERVER_TYPE}" echo " 점검 소요 : ${ELAPSED}초" echo " 점검 항목 : $(( RUN_OK + RUN_WARN + RUN_CRIT + RUN_SKIP ))개" echo " ─────────────────────────────────────────" echo " ${OK} 정상 : ${RUN_OK}개" echo " ${WARN} 주의 : ${RUN_WARN}개" echo " ${CRIT} 긴급 : ${RUN_CRIT}개" echo " 건너뜀 : ${RUN_SKIP}개" echo " ─────────────────────────────────────────" echo " 항목별 결과:" for R in "${RESULTS[@]}"; do KEY=$(echo "$R" | cut -d: -f1) STATUS=$(echo "$R" | cut -d: -f2) case "$STATUS" in OK) printf " %-20s %s\n" "$KEY" "${OK}" ;; WARN) printf " %-20s %s\n" "$KEY" "${WARN}" ;; CRIT) printf " %-20s %s\n" "$KEY" "${CRIT}" ;; *) printf " %-20s %s (%s)\n" "$KEY" "${WARN}" "$STATUS" ;; esac done echo " ─────────────────────────────────────────" case $RESULT in 0) echo " 최종 결과: ${OK} 시스템 전체 정상" ;; 1) echo " 최종 결과: ${WARN} 주의 항목 있음 — 모니터링 강화 권고" ;; 2) echo " 최종 결과: ${CRIT} 즉시 조치 필요" ;; esac echo " 완료 시각: $(date '+%Y-%m-%d %H:%M:%S')" echo "======================================================" # JSON 출력 (GUARDiA ITSM AI 파싱용) if [ "$OUTPUT_FORMAT" = "json" ]; then echo "---JSON_RESULT---" python3 -c " import json, sys results = {} for r in '''${RESULTS[*]}'''.split(): k, v = r.split(':', 1) results[k] = v print(json.dumps({ 'hostname': '$(hostname -s)', 'server_type': '${SERVER_TYPE}', 'overall': ['OK','WARN','CRIT'][${RESULT}], 'ok': ${RUN_OK}, 'warn': ${RUN_WARN}, 'crit': ${RUN_CRIT}, 'skip': ${RUN_SKIP}, 'elapsed_sec': ${ELAPSED}, 'results': results }, ensure_ascii=False, indent=2)) " 2>/dev/null || echo "{\"error\": \"json 생성 실패\"}" fi exit $RESULT