diff --git a/playbooks/statistica.yml b/playbooks/statistica.yml new file mode 100644 index 0000000..5d5f6cf --- /dev/null +++ b/playbooks/statistica.yml @@ -0,0 +1,557 @@ +--- +- name: Weekly Debian server report (with trends) → Telegram via SOCKS5 + hosts: all + gather_facts: yes + become: true + + vars: + telegram_token: "8820292312:AAECFO1AEWOJTh30zNuhyjC5C81o_qJHQcM" + telegram_chat_id: "407968638" + telegram_proxy: "socks5h://administrator:60NthRegT@192.168.0.218:1080" + + # Снапшоты живут НА ЦЕЛЕВОМ ХОСТЕ — переживают перезапуск пода EE + snapshot_dir: "/var/lib/ansible-metrics" + + metrics_packages: + - curl + - bc + - procps + - sysstat + - lm-sensors + - smartmontools + - gawk + - coreutils + + tasks: + # ================================================================== + # 0. PREFLIGHT + # ================================================================== + - name: Ensure required packages are installed (Debian) + ansible.builtin.apt: + name: "{{ metrics_packages }}" + state: present + update_cache: yes + cache_valid_time: 3600 + when: ansible_os_family == "Debian" + + - name: Enable and start sysstat (needed for sar) + ansible.builtin.service: + name: sysstat + state: started + enabled: yes + when: ansible_os_family == "Debian" + failed_when: false + changed_when: false + + - name: Warn if host is not Debian family + ansible.builtin.debug: + msg: "WARNING: {{ inventory_hostname }} is {{ ansible_os_family }}, не Debian — часть метрик может быть недоступна" + when: ansible_os_family != "Debian" + + # ================================================================== + # 1. ЧИТАЕМ СНАПШОТ ПРОШЛОЙ НЕДЕЛИ (на целевом хосте) + # ================================================================== + - name: Ensure snapshot dir exists on target host + ansible.builtin.file: + path: "{{ snapshot_dir }}" + state: directory + mode: "0755" + become: true + + - name: Read previous week snapshot + ansible.builtin.slurp: + src: "{{ snapshot_dir }}/{{ inventory_hostname }}.json" + register: prev_snapshot_raw + failed_when: false + become: true + + - name: Parse previous snapshot (or empty dict on first run) + ansible.builtin.set_fact: + prev: "{{ (prev_snapshot_raw.content | b64decode | from_json) if prev_snapshot_raw.content is defined else {} }}" + + # ================================================================== + # 2. БАЗОВЫЕ МЕТРИКИ + # ================================================================== + - name: Gather uptime + ansible.builtin.command: uptime -p + register: uptime_output + changed_when: false + + - name: Get load average + ansible.builtin.command: uptime + register: load_output + changed_when: false + + - name: Extract load averages + ansible.builtin.set_fact: + load1: "{{ (load_output.stdout.split('load average:')[1].split(',')[0] | trim) if 'load average:' in load_output.stdout else 'N/A' }}" + load5: "{{ (load_output.stdout.split('load average:')[1].split(',')[1] | trim) if 'load average:' in load_output.stdout else 'N/A' }}" + load15: "{{ (load_output.stdout.split('load average:')[1].split(',')[2] | trim) if 'load average:' in load_output.stdout else 'N/A' }}" + + # CPU idle (vmstat -> mpstat -> top -> /proc/stat) + - name: Get CPU idle via vmstat + ansible.builtin.shell: | + if command -v vmstat >/dev/null 2>&1; then + vmstat 1 2 | tail -1 | awk '{print $15}' + else + echo "" + fi + register: cpu_idle_vmstat + changed_when: false + failed_when: false + + - name: Get CPU idle via mpstat + ansible.builtin.shell: | + if command -v mpstat >/dev/null 2>&1; then + mpstat 1 1 | awk '/Average/ {print $NF}' + else + echo "" + fi + register: cpu_idle_mpstat + changed_when: false + failed_when: false + when: cpu_idle_vmstat.stdout == "" + + - name: Get CPU idle via top + ansible.builtin.shell: | + top -bn2 | grep "Cpu(s)" | tail -1 | sed 's/.*, *\([0-9.]*\)%* id.*/\1/' + register: cpu_idle_top + changed_when: false + failed_when: false + when: cpu_idle_vmstat.stdout == "" and cpu_idle_mpstat.stdout == "" + + - name: Get CPU idle via /proc/stat + ansible.builtin.shell: | + prev_idle=$(awk '/^cpu / {print $5}' /proc/stat) + prev_total=$(awk '/^cpu / {print $2+$3+$4+$5+$6+$7+$8}' /proc/stat) + sleep 1 + curr_idle=$(awk '/^cpu / {print $5}' /proc/stat) + curr_total=$(awk '/^cpu / {print $2+$3+$4+$5+$6+$7+$8}' /proc/stat) + diff_idle=$((curr_idle - prev_idle)) + diff_total=$((curr_total - prev_total)) + awk -v i="$diff_idle" -v t="$diff_total" 'BEGIN {printf "%.1f", (i*100)/t}' + register: cpu_idle_proc + changed_when: false + failed_when: false + when: cpu_idle_vmstat.stdout == "" and cpu_idle_mpstat.stdout == "" and cpu_idle_top.stdout == "" + + - name: Set final idle with validation + ansible.builtin.set_fact: + cpu_idle_final: >- + {%- set idle = (cpu_idle_vmstat.stdout | float) if cpu_idle_vmstat.stdout != "" else + (cpu_idle_mpstat.stdout | float) if cpu_idle_mpstat.stdout != "" else + (cpu_idle_top.stdout | float) if cpu_idle_top.stdout != "" else + (cpu_idle_proc.stdout | float) if cpu_idle_proc.stdout != "" else 90 -%} + {%- if idle > 100 or idle < 0 -%}{{ 90 }}{%- else -%}{{ idle }}{%- endif -%} + + # Память + - name: Get memory usage human + ansible.builtin.shell: | + free -h | awk '/Mem:/ {printf "Занято: %s, Всего: %s, Доступно: %s", $3, $2, $6}' + register: memory_usage_human + changed_when: false + + - name: Get memory percentage + ansible.builtin.shell: | + free -m | awk 'NR==2{printf "%.0f", $3/$2*100}' + register: mem_percent + changed_when: false + + # Диск + - name: Get disk usage for root + ansible.builtin.shell: | + df -h / | awk 'NR==2{printf "Занято: %s, Всего: %s, Свободно: %s, Использовано: %s", $3, $2, $4, $5}' + register: disk_usage + changed_when: false + + # Частота CPU + - name: Get CPU frequency + ansible.builtin.shell: | + if [ -f /sys/devices/system/cpu/cpu0/cpufreq/scaling_cur_freq ]; then + awk '{printf "%.2f GHz", $1/1000000}' /sys/devices/system/cpu/cpu0/cpufreq/scaling_cur_freq + elif command -v lscpu >/dev/null 2>&1; then + lscpu | grep "MHz" | awk '{print $3}' | head -1 | awk '{printf "%.2f GHz", $1/1000}' + else + echo "" + fi + register: cpu_freq + changed_when: false + failed_when: false + + - name: Set CPU frequency text + ansible.builtin.set_fact: + cpu_freq_text: "{{ cpu_freq.stdout | default('') }}" + + # Температура CPU + - name: Get CPU temperature from thermal zone + ansible.builtin.shell: | + cat /sys/class/thermal/thermal_zone0/temp 2>/dev/null | awk '{printf "%.1f°C", $1/1000}' + register: cpu_temp + changed_when: false + failed_when: false + + - name: Fallback to sensors + ansible.builtin.shell: | + sensors 2>/dev/null | grep -i "Package id 0" | awk '{print $4}' | sed 's/+//g' | sed 's/°C//g' | awk '{printf "%.1f°C", $1}' + register: cpu_temp_sensors + changed_when: false + failed_when: false + when: cpu_temp.stdout == "" + + - name: Set final CPU temperature + ansible.builtin.set_fact: + cpu_temp_final: "{{ cpu_temp.stdout if cpu_temp.stdout != '' else (cpu_temp_sensors.stdout | default('')) }}" + + # Температура и здоровье диска + - name: Check if smartctl exists + ansible.builtin.command: which smartctl + register: smartctl_check + changed_when: false + failed_when: false + + - name: Get first block device + ansible.builtin.shell: | + lsblk -dn -o NAME | grep -E '^(sd|hd|nvme|vd|xvd)' | head -1 + register: disk_device + changed_when: false + failed_when: false + + - name: Get disk temperature + ansible.builtin.shell: | + smartctl -A /dev/{{ disk_device.stdout }} 2>/dev/null \ + | grep -i "Temperature" | awk '{print $10}' \ + | sed 's/°C//g' | awk '{printf "%.1f°C", $1}' + register: disk_temp + changed_when: false + failed_when: false + when: smartctl_check.rc == 0 and disk_device.stdout != "" + + - name: Get disk SMART health + ansible.builtin.shell: | + smartctl -H /dev/{{ disk_device.stdout }} 2>/dev/null \ + | grep -i "SMART overall-health" | awk -F: '{print $2}' | xargs + register: smart_health + changed_when: false + failed_when: false + when: smartctl_check.rc == 0 and disk_device.stdout != "" + + - name: Get reallocated sectors + ansible.builtin.shell: | + smartctl -A /dev/{{ disk_device.stdout }} 2>/dev/null \ + | awk '/Reallocated_Sector_Ct/ {print $NF}' | head -1 + register: reallocated + changed_when: false + failed_when: false + when: smartctl_check.rc == 0 and disk_device.stdout != "" + + - name: Set disk-related facts (safe) + ansible.builtin.set_fact: + disk_temp_text: "{{ (disk_temp | default({})).stdout | default('') }}" + disk_device_name: "{{ (disk_device | default({})).stdout | default('') }}" + smart_health_text: "{{ (smart_health | default({})).stdout | default('N/A') }}" + reallocated_text: "{{ (reallocated | default({})).stdout | default('0') }}" + + # ================================================================== + # 3. НЕДЕЛЬНЫЕ АГРЕГАТЫ через sar + # ================================================================== + - name: Weekly CPU avg/peak from sar + ansible.builtin.shell: | + { + for i in $(seq 0 6); do + d=$(date -d "$i days ago" +%d) + f="/var/log/sysstat/sa$d" + [ -f "$f" ] && sar -u -f "$f" 2>/dev/null + done + } | awk '/^[0-9]/ { + cpu=100-$NF + sum+=cpu; n++ + if (cpu>max) max=cpu + } END { + if (n>0) printf "%.1f %.1f %d", sum/n, max, n; + else print "N/A N/A 0" + }' + register: cpu_week + changed_when: false + failed_when: false + + - name: Weekly memory peak from sar + ansible.builtin.shell: | + { + for i in $(seq 0 6); do + d=$(date -d "$i days ago" +%d) + f="/var/log/sysstat/sa$d" + [ -f "$f" ] && sar -r -f "$f" 2>/dev/null + done + } | awk '/^[0-9]/ { + used=$4+$5 + if (used>max) max=used + } END { + if (max>0) printf "%.1f", max; + else print "N/A" + }' + register: mem_week_peak + changed_when: false + failed_when: false + + - name: Parse weekly CPU values + ansible.builtin.set_fact: + cpu_week_avg: "{{ (cpu_week.stdout.split()[0]) if cpu_week.stdout.split() | length >= 1 else 'N/A' }}" + cpu_week_peak: "{{ (cpu_week.stdout.split()[1]) if cpu_week.stdout.split() | length >= 2 else 'N/A' }}" + cpu_week_samples: "{{ (cpu_week.stdout.split()[2]) if cpu_week.stdout.split() | length >= 3 else '0' }}" + + # ================================================================== + # 4. СОСТОЯНИЕ СИСТЕМЫ + # ================================================================== + - name: Count upgradable packages + ansible.builtin.shell: | + apt list --upgradable 2>/dev/null | tail -n +2 | grep -c upgradable || true + register: upgradable_count + changed_when: false + failed_when: false + + - name: Count security updates + ansible.builtin.shell: | + apt list --upgradable 2>/dev/null | grep -ci security || true + register: security_count + changed_when: false + failed_when: false + + - name: Check if reboot required + ansible.builtin.stat: + path: /var/run/reboot-required + register: reboot_required + + - name: Count failed systemd units + ansible.builtin.shell: | + systemctl --failed --no-legend 2>/dev/null | wc -l + register: failed_units + changed_when: false + failed_when: false + + - name: Count reboots in last 7 days + ansible.builtin.shell: | + last -x reboot --since "7 days ago" 2>/dev/null | grep -c "^reboot" || true + register: reboot_count + changed_when: false + failed_when: false + + - name: Count failed SSH logins in last 7 days + ansible.builtin.shell: | + journalctl --since "7 days ago" -u ssh --no-pager 2>/dev/null \ + | grep -c "Failed password" || true + register: failed_ssh + changed_when: false + failed_when: false + + - name: Count OOM kills in last 7 days + ansible.builtin.shell: | + journalctl --since "7 days ago" -k --no-pager 2>/dev/null \ + | grep -ci "out of memory" || true + register: oom_kills + changed_when: false + failed_when: false + + - name: Count I/O errors in last 7 days + ansible.builtin.shell: | + journalctl --since "7 days ago" -k --no-pager 2>/dev/null \ + | grep -ci "I/O error" || true + register: io_errors + changed_when: false + failed_when: false + + - name: Top 5 by memory now + ansible.builtin.shell: | + ps aux --sort=-%mem | head -6 | tail -5 | awk '{print $11 " (" $4 "%)"}' + register: top_procs + changed_when: false + + - name: Top 5 long-running processes + ansible.builtin.shell: | + ps -eo etimes,comm --sort=-etimes | awk 'NR>1 && $1>604800 {print $2 " (uptime " int($1/86400) "d)"}' | head -5 + register: top_long + changed_when: false + failed_when: false + + - name: Join top lists + ansible.builtin.set_fact: + top_procs_joined: "{{ top_procs.stdout_lines | join('\n') }}" + top_long_joined: "{{ top_long.stdout_lines | join('\n') if top_long.stdout_lines | length > 0 else '(нет процессов с аптаймом > 7 дней)' }}" + + # ================================================================== + # 5. РАСЧЁТЫ, ДЕЛЬТЫ, HEALTH-SCORE + # ================================================================== + - name: Calculate CPU usage % + ansible.builtin.set_fact: + cpu_usage: "{{ 100 - (cpu_idle_final | float) }}" + + - name: Extract disk used % + ansible.builtin.set_fact: + disk_used_percent: "{{ disk_usage.stdout.split('Использовано:')[1].strip().replace('%', '') | float }}" + + - name: Compute deltas vs previous week + ansible.builtin.set_fact: + disk_delta: "{{ ((disk_used_percent | float) - (prev.disk_used_pct | float)) | round(1) }}" + mem_delta: "{{ ((mem_percent.stdout | float) - (prev.mem_pct | float)) | round(1) }}" + when: + - prev.disk_used_pct is defined + - prev.mem_pct is defined + + - name: Format delta strings with arrows + ansible.builtin.set_fact: + disk_delta_str: >- + {%- if disk_delta is defined -%} + {%- set d = disk_delta | float -%} + {%- if d > 1 %} (+{{ d }}% ⬆️) + {%- elif d < -1 %} ({{ d }}% ⬇️) + {%- else %} (≈ без изменений){%- endif -%} + {%- else -%} (нет данных за прошлую неделю){%- endif -%} + mem_delta_str: >- + {%- if mem_delta is defined -%} + {%- set d = mem_delta | float -%} + {%- if d > 3 %} (+{{ d }}% ⬆️) + {%- elif d < -3 %} ({{ d }}% ⬇️) + {%- else %} (≈ без изменений){%- endif -%} + {%- else -%} (нет данных){%- endif -%} + + - name: Set component health + ansible.builtin.set_fact: + cpu_health: "{% if (cpu_usage | float) < 50 %}🟢{% elif (cpu_usage | float) < 80 %}🟡{% else %}🔴{% endif %}" + mem_health: "{% if (mem_percent.stdout | float) < 60 %}🟢{% elif (mem_percent.stdout | float) < 85 %}🟡{% else %}🔴{% endif %}" + disk_health: "{% if (disk_used_percent | float) < 70 %}🟢{% elif (disk_used_percent | float) < 85 %}🟡{% else %}🔴{% endif %}" + + - name: Compute warnings list + ansible.builtin.set_fact: + warnings_list: >- + {{ + ( + (['Диск > 80%'] if (disk_used_percent | float) > 80 else []) + + (['Диск вырос > 5% за неделю'] if (disk_delta is defined and (disk_delta | float) > 5) else []) + + (['Требуется перезагрузка'] if reboot_required.stat.exists else []) + + (['Security-обновления не установлены'] if (security_count.stdout | int) > 0 else []) + + (['Failed systemd units: ' ~ failed_units.stdout] if (failed_units.stdout | int) > 0 else []) + + (['OOM-kills за неделю'] if (oom_kills.stdout | int) > 0 else []) + + (['I/O errors в ядре'] if (io_errors.stdout | int) > 0 else []) + + (['SMART не PASSED'] if (smart_health_text != 'PASSED' and smart_health_text != 'N/A' and smart_health_text != '') else []) + ) | list + }} + + - name: Compute overall status emoji + ansible.builtin.set_fact: + overall_status: >- + {%- set n = warnings_list | length -%} + {{ '🔴 Требует внимания' if n >= 3 else ('🟡 Есть предупреждения' if n > 0 else '🟢 Всё в порядке') }} + + # ================================================================== + # 6. ФОРМИРОВАНИЕ СООБЩЕНИЯ + # ================================================================== + - name: Build weekly report message (HTML) + ansible.builtin.set_fact: + message_html: | + 📅 Еженедельный отчёт: {{ ansible_hostname }} + Статус: {{ overall_status }} {% if warnings_list | length > 0 %}({{ warnings_list | length }} ⚠️){% endif %} + {{ ansible_date_time.date }} {{ ansible_date_time.time }} · {{ ansible_default_ipv4.address | default('N/A') }} + + {% if warnings_list | length > 0 %} + ⚠️ Предупреждения: + {% for w in warnings_list %}• {{ w }} + {% endfor %} + {% endif %} + + ─────────────── + 📊 Средние и пиковые за неделю + 💻 CPU: avg {{ cpu_week_avg }}% · peak {{ cpu_week_peak }}% ({{ cpu_week_samples }} замеров) + 🧠 RAM: peak {{ mem_week_peak }}% использовано + ─────────────── + 📈 Текущее состояние + ⏱️ Аптайм: {{ uptime_output.stdout }} + 📉 Load avg (1,5,15): {{ load1 }} , {{ load5 }} , {{ load15 }} + + 💻 CPU сейчас: {{ (cpu_usage | float) | round(1) }}% {{ cpu_health }}{% if cpu_freq_text != '' %} ({{ cpu_freq_text }}){% endif %} + 🧠 RAM сейчас: {{ mem_percent.stdout }}% {{ mem_health }} ({{ memory_usage_human.stdout }}){{ mem_delta_str }} + 💾 Диск /: {{ (disk_used_percent | float) | round(1) }}% {{ disk_health }} ({{ disk_usage.stdout.split('Использовано:')[0].strip() }}){{ disk_delta_str }} + ─────────────── + 🌡️ Температуры + CPU: {{ cpu_temp_final | default('недоступно') }} + {% if disk_temp_text != "" %}Диск {{ disk_device_name }}: {{ disk_temp_text }} · SMART: {{ smart_health_text }} · realloc: {{ reallocated_text }}{% endif %} + ─────────────── + 🛠️ Обслуживание + 📦 Обновлений: {{ upgradable_count.stdout }} (🔴 security: {{ security_count.stdout }}) + ♻️ Требуется reboot: {{ 'ДА ⚠️' if reboot_required.stat.exists else 'нет' }} + 🔁 Перезагрузок за неделю: {{ reboot_count.stdout }} + 🚨 Failed units: {{ failed_units.stdout }} + ─────────────── + 🔐 Безопасность и ошибки за неделю + 🔓 Неудачных SSH-логинов: {{ failed_ssh.stdout }} + 💥 OOM-kills: {{ oom_kills.stdout }} + 🗄️ I/O errors: {{ io_errors.stdout }} + ─────────────── + 🔥 Топ-5 процессов по памяти (сейчас): +
{{ top_procs_joined }}
+ 🕰️ Топ-5 долгоживущих процессов (> 7 дней): +
{{ top_long_joined }}
+ + # ================================================================== + # 7. СОХРАНЯЕМ СНАПШОТ (на целевом хосте) + # ================================================================== + - name: Save snapshot for next week + ansible.builtin.copy: + content: | + {{ + { + 'ts': ansible_date_time.iso8601, + 'disk_used_pct': disk_used_percent | float, + 'mem_pct': mem_percent.stdout | float, + 'cpu_now': cpu_usage | float, + 'cpu_week_avg': cpu_week_avg, + 'cpu_week_peak': cpu_week_peak + } | to_nice_json + }} + dest: "{{ snapshot_dir }}/{{ inventory_hostname }}.json" + mode: "0644" + become: true + + # ================================================================== + # 8. ОТПРАВКА В TELEGRAM через SOCKS5 (с EE-контейнера AWX) + # ================================================================== + - name: Check Telegram API reachability via SOCKS5 proxy + ansible.builtin.shell: | + curl -s -o /dev/null -w "%{http_code}" --max-time 15 \ + --proxy "{{ telegram_proxy }}" \ + "https://api.telegram.org/bot{{ telegram_token }}/getMe" + register: tg_check + changed_when: false + failed_when: false + delegate_to: localhost + become: false + + - name: Fail if Telegram API unreachable + ansible.builtin.fail: + msg: > + Telegram API недоступен через прокси {{ telegram_proxy }}. + HTTP-код: {{ tg_check.stdout }}. + when: tg_check.stdout != "200" + delegate_to: localhost + become: false + + - name: Send report to Telegram + ansible.builtin.shell: | + set -e + response=$(curl -s -w "\n%{http_code}" -X POST \ + --proxy "{{ telegram_proxy }}" \ + -H "Content-Type: application/json" \ + -d '{"chat_id": "{{ telegram_chat_id }}", "text": {{ message_html | to_json }}, "parse_mode": "HTML", "disable_web_page_preview": true}' \ + "https://api.telegram.org/bot{{ telegram_token }}/sendMessage" 2>&1) + http_code=$(echo "$response" | tail -n1) + body=$(echo "$response" | sed '$d') + echo "HTTP code: $http_code" + if [ "$http_code" -eq 200 ] && echo "$body" | grep -q '"ok":true'; then + echo "OK" + exit 0 + fi + echo "ERROR: $body" >&2 + exit 1 + changed_when: false + register: tg_response + failed_when: tg_response.rc != 0 + delegate_to: localhost + become: false \ No newline at end of file