zioinfo-mail/itsm/scripts/sm/common/sm_full_check.sh

#!/bin/bash
# ============================================================
# GUARDiA SM | sm_full_check.sh
# 대상: 전체 SM 점검 오케스트레이터
# 파라미터: SERVER_TYPE=WEB|WAS|DB|ESB|SEARCH|AGENT|ALL
#           SCRIPT_BASE=/opt/guardia/itsm/scripts/sm
#           TIMEOUT=300 (스크립트당 타임아웃 초)
#           OUTPUT_FORMAT=text|json
# ============================================================
set -euo pipefail
SERVER_TYPE=${SERVER_TYPE:-ALL}
SCRIPT_BASE=${SCRIPT_BASE:-$(dirname "$0")/..}
TIMEOUT_SEC=${TIMEOUT:-300}
OUTPUT_FORMAT=${OUTPUT_FORMAT:-text}
OK="[OK]"; WARN="[WARN]"; CRIT="[CRIT]"
SEP="═══════════════════════════════════════════"
RESULT=0

START_EPOCH=$(date +%s)

echo "======================================================"
echo " GUARDiA SM 전체 점검 오케스트레이터"
echo " 서버: $(hostname -s)  타입: ${SERVER_TYPE}"
echo " 점검 시작: $(date '+%Y-%m-%d %H:%M:%S %Z')"
echo "======================================================"

# 스크립트 경로 정규화
SCRIPT_BASE=$(realpath "$SCRIPT_BASE" 2>/dev/null || echo "$SCRIPT_BASE")

# ── 점검 결과 수집 배열 ───────────────────────────────────
declare -a RESULTS=()
declare -A SCRIPT_MAP=(
    ["system"]="${SCRIPT_BASE}/common/system_health.sh"
    ["apache"]="${SCRIPT_BASE}/web/web_apache_sm.sh"
    ["nginx"]="${SCRIPT_BASE}/web/web_nginx_sm.sh"
    ["webtob"]="${SCRIPT_BASE}/web/web_webtob_sm.sh"
    ["tomcat"]="${SCRIPT_BASE}/was/was_tomcat_sm.sh"
    ["jboss"]="${SCRIPT_BASE}/was/was_jboss_sm.sh"
    ["jeus"]="${SCRIPT_BASE}/was/was_jeus_sm.sh"
    ["weblogic"]="${SCRIPT_BASE}/was/was_weblogic_sm.sh"
    ["postgresql"]="${SCRIPT_BASE}/db/db_postgresql_sm.sh"
    ["oracle"]="${SCRIPT_BASE}/db/db_oracle_sm.sh"
    ["mysql"]="${SCRIPT_BASE}/db/db_mysql_sm.sh"
    ["tibero"]="${SCRIPT_BASE}/db/db_tibero_sm.sh"
    ["esb"]="${SCRIPT_BASE}/esb/esb_check.sh"
    ["elasticsearch"]="${SCRIPT_BASE}/search/search_elasticsearch_sm.sh"
    ["solr"]="${SCRIPT_BASE}/search/search_solr_sm.sh"
    ["pinpoint"]="${SCRIPT_BASE}/agent/agent_pinpoint_sm.sh"
    ["scouter"]="${SCRIPT_BASE}/agent/agent_scouter_sm.sh"
    ["crontab"]="${SCRIPT_BASE}/crontab/crontab_sm.sh"
    ["ping"]="${SCRIPT_BASE}/ping/ping_test.sh"
    ["log"]="${SCRIPT_BASE}/log/log_analysis.sh"
)

# ── 서버 타입별 실행 스크립트 결정 ───────────────────────
# 환경변수로 개별 스크립트 선택
# 예: WEB_ENGINE=nginx DB_ENGINE=postgresql
WEB_ENGINE=${WEB_ENGINE:-apache}
WAS_ENGINE=${WAS_ENGINE:-tomcat}
DB_ENGINE=${DB_ENGINE:-postgresql}
ESB_ENGINE=${ESB_ENGINE:-activemq}
SEARCH_ENGINE=${SEARCH_ENGINE:-elasticsearch}
APM_ENGINE=${APM_ENGINE:-scouter}

# 타입별 스크립트 목록
case "${SERVER_TYPE^^}" in
    WEB)
        SCRIPTS=("system" "$WEB_ENGINE" "ping" "log")
        ;;
    WAS)
        SCRIPTS=("system" "$WAS_ENGINE" "ping" "log")
        ;;
    DB)
        SCRIPTS=("system" "$DB_ENGINE" "ping" "log")
        ;;
    ESB)
        SCRIPTS=("system" "esb" "ping" "log")
        ;;
    SEARCH)
        SCRIPTS=("system" "$SEARCH_ENGINE" "ping" "log")
        ;;
    AGENT)
        SCRIPTS=("system" "$APM_ENGINE" "ping" "log")
        ;;
    ALL)
        SCRIPTS=("system" "$WEB_ENGINE" "$WAS_ENGINE" "$DB_ENGINE" "esb" \
                 "$SEARCH_ENGINE" "$APM_ENGINE" "crontab" "ping" "log")
        ;;
    CUSTOM)
        # 개별 지정: CUSTOM_SCRIPTS="system tomcat postgresql ping"
        IFS=' ' read -ra SCRIPTS <<< "${CUSTOM_SCRIPTS:-system}"
        ;;
    *)
        echo " ${WARN} 알 수 없는 SERVER_TYPE: ${SERVER_TYPE}"
        echo " 지원: WEB, WAS, DB, ESB, SEARCH, AGENT, ALL, CUSTOM"
        exit 1
        ;;
esac

echo " 실행할 점검 항목: ${SCRIPTS[*]}"
echo "======================================================"

# ── 스크립트 순차 실행 + 결과 집계 ──────────────────────
RUN_OK=0; RUN_WARN=0; RUN_CRIT=0; RUN_SKIP=0

for SCRIPT_KEY in "${SCRIPTS[@]}"; do
    SCRIPT_PATH="${SCRIPT_MAP[$SCRIPT_KEY]:-}"
    echo
    echo "${SEP}"
    echo " 점검: ${SCRIPT_KEY} (${SCRIPT_PATH})"
    echo "${SEP}"

    if [ -z "$SCRIPT_PATH" ]; then
        echo " ${WARN} 알 수 없는 스크립트 키: ${SCRIPT_KEY}"
        RUN_SKIP=$(( RUN_SKIP + 1 ))
        RESULTS+=("${SCRIPT_KEY}:SKIP")
        continue
    fi

    if [ ! -f "$SCRIPT_PATH" ]; then
        echo " ${WARN} 스크립트 없음: ${SCRIPT_PATH}"
        RUN_SKIP=$(( RUN_SKIP + 1 ))
        RESULTS+=("${SCRIPT_KEY}:SKIP")
        continue
    fi

    chmod +x "$SCRIPT_PATH" 2>/dev/null || true

    # 타임아웃 실행
    EXIT_CODE=0
    timeout "$TIMEOUT_SEC" bash "$SCRIPT_PATH" 2>&1 || EXIT_CODE=$?

    case $EXIT_CODE in
        0)
            echo " → 결과: ${OK}"
            RUN_OK=$(( RUN_OK + 1 ))
            RESULTS+=("${SCRIPT_KEY}:OK")
            ;;
        1)
            echo " → 결과: ${WARN}"
            RUN_WARN=$(( RUN_WARN + 1 ))
            RESULTS+=("${SCRIPT_KEY}:WARN")
            [ $RESULT -lt 1 ] && RESULT=1
            ;;
        2)
            echo " → 결과: ${CRIT}"
            RUN_CRIT=$(( RUN_CRIT + 1 ))
            RESULTS+=("${SCRIPT_KEY}:CRIT")
            RESULT=2
            ;;
        124)
            echo " → 결과: ${WARN} (타임아웃 ${TIMEOUT_SEC}s)"
            RUN_WARN=$(( RUN_WARN + 1 ))
            RESULTS+=("${SCRIPT_KEY}:TIMEOUT")
            [ $RESULT -lt 1 ] && RESULT=1
            ;;
        *)
            echo " → 결과: ${WARN} (종료코드: ${EXIT_CODE})"
            RUN_WARN=$(( RUN_WARN + 1 ))
            RESULTS+=("${SCRIPT_KEY}:EXIT${EXIT_CODE}")
            [ $RESULT -lt 1 ] && RESULT=1
            ;;
    esac
done

END_EPOCH=$(date +%s)
ELAPSED=$(( END_EPOCH - START_EPOCH ))

# ── 최종 요약 ─────────────────────────────────────────────
echo
echo "======================================================"
echo " GUARDiA SM 전체 점검 완료 요약"
echo "======================================================"
echo " 서버         : $(hostname -s)"
echo " 서버 타입    : ${SERVER_TYPE}"
echo " 점검 소요    : ${ELAPSED}초"
echo " 점검 항목    : $(( RUN_OK + RUN_WARN + RUN_CRIT + RUN_SKIP ))개"
echo " ─────────────────────────────────────────"
echo "  ${OK}   정상     : ${RUN_OK}개"
echo "  ${WARN} 주의     : ${RUN_WARN}개"
echo "  ${CRIT} 긴급     : ${RUN_CRIT}개"
echo "  건너뜀          : ${RUN_SKIP}개"
echo " ─────────────────────────────────────────"
echo " 항목별 결과:"
for R in "${RESULTS[@]}"; do
    KEY=$(echo "$R" | cut -d: -f1)
    STATUS=$(echo "$R" | cut -d: -f2)
    case "$STATUS" in
        OK)   printf "   %-20s %s\n" "$KEY" "${OK}" ;;
        WARN) printf "   %-20s %s\n" "$KEY" "${WARN}" ;;
        CRIT) printf "   %-20s %s\n" "$KEY" "${CRIT}" ;;
        *)    printf "   %-20s %s (%s)\n" "$KEY" "${WARN}" "$STATUS" ;;
    esac
done
echo " ─────────────────────────────────────────"
case $RESULT in
    0) echo " 최종 결과: ${OK}   시스템 전체 정상" ;;
    1) echo " 최종 결과: ${WARN} 주의 항목 있음 — 모니터링 강화 권고" ;;
    2) echo " 최종 결과: ${CRIT} 즉시 조치 필요" ;;
esac
echo " 완료 시각: $(date '+%Y-%m-%d %H:%M:%S')"
echo "======================================================"

# JSON 출력 (GUARDiA ITSM AI 파싱용)
if [ "$OUTPUT_FORMAT" = "json" ]; then
    echo "---JSON_RESULT---"
    python3 -c "
import json, sys
results = {}
for r in '''${RESULTS[*]}'''.split():
    k, v = r.split(':', 1)
    results[k] = v
print(json.dumps({
    'hostname': '$(hostname -s)',
    'server_type': '${SERVER_TYPE}',
    'overall': ['OK','WARN','CRIT'][${RESULT}],
    'ok': ${RUN_OK},
    'warn': ${RUN_WARN},
    'crit': ${RUN_CRIT},
    'skip': ${RUN_SKIP},
    'elapsed_sec': ${ELAPSED},
    'results': results
}, ensure_ascii=False, indent=2))
" 2>/dev/null || echo "{\"error\": \"json 생성 실패\"}"
fi

exit $RESULT