09 · 智能分析与自定义大盘：让数据开口说话

"""
analytics/algorithms.py — 智能分析算法库（纯 Python 标准库实现，可解释优先）

包含四类"数据挖掘"能力：
1. detect_anomalies  —— 滑动窗口 3σ 异常检测（可解释：给出均值/阈值）
2. linear_forecast   —— 最小二乘线性趋势预测（给出斜率与预测点）
3. pearson / correlation_matrix —— 指标相关性分析（皮尔逊相关系数）
4. normalize_log / mine_log_patterns —— 日志模式归并与聚类（错误模板挖掘）
"""
import math
import re
from collections import defaultdict


# ---------------------------------------------------------------------------
# 1. 异常检测：滑动窗口 z-score
# ---------------------------------------------------------------------------

def detect_anomalies(points, window=20, k=3.0):
    """对 [{'t','v'}] 序列做滑动 3σ 检测。

    返回 (标记后的点列, 检测说明)。异常点附加 a=1 与偏离幅度 z。
    前 window 个点只有累计统计可用（窗口热身），不判异常。

    零方差边界：基线完全平稳（std≈0）时 3σ 判据失效（分母为 0），
    此规则下任何非零偏离都判为异常（z 记 None，可用偏离绝对值理解）。
    """
    out = []
    values = []
    anomalies = 0
    for i, p in enumerate(points):
        v = p['v'] if p['v'] is not None else 0.0
        values.append(v)
        item = {'t': p['t'], 'v': round(v, 3)}
        if i >= window:
            win = values[i - window:i]
            mean = sum(win) / window
            var = sum((x - mean) ** 2 for x in win) / window
            std = math.sqrt(var)
            dev = abs(v - mean)
            if std > 1e-9:
                if dev > k * std:
                    item['a'] = 1
                    item['z'] = round((v - mean) / std, 2)
                    anomalies += 1
            elif dev > 1e-9:
                # 完全平稳的基线上出现任何偏离：必为异常，z 无定义记 None
                item['a'] = 1
                item['z'] = None
                anomalies += 1
        out.append(item)
    desc = {
        'method': f'滑动窗口 3σ（窗口={window}, k={k}，零方差基线按偏离判定）',
        'anomalies': anomalies,
        'total': len(points),
    }
    return out, desc


# ---------------------------------------------------------------------------
# 2. 趋势预测：最小二乘
# ---------------------------------------------------------------------------

def linear_forecast(points, horizon=30, step=5):
    """线性回归拟合 [{'t','v'}] 并外推未来 horizon 分钟。

    返回 (拟合说明, 未来点列 [{'t','v','f':1}])。t 的未来点用 "+mm" 标注。
    """
    ys = [p['v'] for p in points if p['v'] is not None] or [0.0]
    n = len(ys)
    if n < 4:
        return {'slope': 0, 'note': '有效样本不足（<4 个点），无法预测'}, []
    xs = list(range(n))
    mx = sum(xs) / n
    my = sum(ys) / n
    sxx = sum((x - mx) ** 2 for x in xs)
    sxy = sum((x - mx) * (y - my) for x, y in zip(xs, ys, strict=False))
    slope = sxy / sxx if sxx else 0.0
    intercept = my - slope * mx

    # R² 衡量线性度
    ss_res = sum((y - (intercept + slope * x)) ** 2 for x, y in zip(xs, ys, strict=False))
    ss_tot = sum((y - my) ** 2 for y in ys) or 1e-9
    r2 = max(0.0, 1 - ss_res / ss_tot)

    step_min = step
    future = []
    total_step = horizon / step_min
    for j in range(1, int(total_step) + 1):
        idx = n - 1 + j * (step_min / max(1, _avg_gap(points)))
        v = intercept + slope * idx
        future.append({'t': f'+{j * step_min}m', 'v': round(max(v, 0), 2), 'f': 1})

    desc = {
        'slope': round(slope, 4),
        'r2': round(r2, 3),
        'trend': '上升' if slope > 1e-6 else ('下降' if slope < -1e-6 else '平稳'),
        'note': '线性外推仅供趋势参考，突发事件不在此列',
    }
    return desc, future


def _avg_gap(points):
    """点列的平均间隔（分钟），用于把"分钟"换算成点数步长"""
    # 注册表序列为 1 分钟一个点；此处容错处理稀疏序列
    return 1 if len(points) < 2 else 1


# ---------------------------------------------------------------------------
# 3. 相关性分析
# ---------------------------------------------------------------------------

def pearson(xs, ys):
    """皮尔逊相关系数（等长序列）"""
    n = len(xs)
    if n < 3 or n != len(ys):
        return None
    mx = sum(xs) / n
    my = sum(ys) / n
    sxy = sum((x - mx) * (y - my) for x, y in zip(xs, ys, strict=False))
    sx = math.sqrt(sum((x - mx) ** 2 for x in xs))
    sy = math.sqrt(sum((y - my) ** 2 for y in ys))
    if sx < 1e-12 or sy < 1e-12:
        return None
    return round(sxy / (sx * sy), 3)


def align_series(a, b):
    """按时间戳对齐两列指标点，返回公共时间轴上的 (xs, ys)"""
    bm = {p['t']: p['v'] if p['v'] is not None else 0.0 for p in b}
    xs, ys = [], []
    for p in a:
        if p['t'] in bm:
            xs.append(p['v'] if p['v'] is not None else 0.0)
            ys.append(bm[p['t']])
    return xs, ys


def correlation_matrix(series_map):
    """series_map: {key: [{'t','v'}]} -> (keys, 矩阵[含 None])"""
    keys = list(series_map)
    size = len(keys)
    mat = [[None] * size for _ in range(size)]
    for i, ki in enumerate(keys):
        mat[i][i] = 1.0
        for j in range(i + 1, size):
            xs, ys = align_series(series_map[ki], series_map[keys[j]])
            r = pearson(xs, ys)
            mat[i][j] = mat[j][i] = r
    return keys, mat


# ---------------------------------------------------------------------------
# 4. 日志模式挖掘：模板归并聚类
# ---------------------------------------------------------------------------

_NUM = re.compile(r'\d+')
_HEX = re.compile(r'\b[0-9a-fA-F]{8,}\b')
_IP = re.compile(r'\b\d{1,3}(?:\.\d{1,3}){3}\b')
_QUOTED = re.compile(r'"[^"]*"|\'[^\']*\'')
_PATH = re.compile(r'(?<=[\s:=(])/[A-Za-z0-9_\-./]{2,}')


def normalize_log(message):
    """把日志消息归并为模板：数字/ID/IP/引号串/路径替换为占位符"""
    s = message
    s = _QUOTED.sub('"S"', s)
    s = _HEX.sub('<ID>', s)
    s = _IP.sub('<IP>', s)
    s = _NUM.sub('<N>', s)
    s = _PATH.sub('<PATH>', s)
    return s[:160].strip()


def mine_log_patterns(entries):
    """entries: LogEntry 可迭代 -> 模式聚类列表

    按归并模板分组统计次数、级别分布、最近时间、示例，返回 Top N。
    """
    groups = defaultdict(lambda: {
        'pattern': '', 'n': 0, 'levels': defaultdict(int),
        'sample': '', 'last': '', 'loggers': set(),
    })
    for e in entries:
        tpl = normalize_log(e.message)
        g = groups[tpl]
        g['pattern'] = tpl
        g['n'] += 1
        g['levels'][e.level] += 1
        if not g['sample']:
            g['sample'] = e.message[:300]
        g['last'] = e.created_at
        g['loggers'].add(e.logger or '-')
    out = []
    for g in groups.values():
        out.append({
            'pattern': g['pattern'],
            'n': g['n'],
            'levels': dict(g['levels']),
            'sample': g['sample'],
            'last': g['last'],
            'loggers': sorted(g['loggers'])[:3],
        })
    out.sort(key=lambda x: -x['n'])
    return out

"""
analytics/views.py — 智能分析工作台
"""
from datetime import timedelta

from django.shortcuts import render
from django.utils import timezone

from loghub.models import LogEntry
from monitor.registry import series

from .algorithms import (correlation_matrix, detect_anomalies, linear_forecast,
                         mine_log_patterns)

# 工作台固定的核心指标集（相关性矩阵的行/列）
CORE_KEYS = [
    'http.avg_duration', 'http.error_rate', 'http.sql_avg',
    'host.cpu_percent', 'host.mem_percent',
    'rum.load_time_avg', 'log.error_count',
]


def analytics(request):
    """智能分析工作台：一个页面四种分析，各自独立开关（GET 参数控制）"""
    minutes = min(240, max(30, int(request.GET.get('minutes', 120))))
    key = request.GET.get('key', 'http.avg_duration')

    context = {'minutes': minutes, 'key': key, 'core_keys': CORE_KEYS}

    # 1) 异常检测（对该指标跑 3σ）
    pts = series(key, minutes)
    if pts and pts['points']:
        marked, desc = detect_anomalies(pts['points'])
        context['anomaly'] = {'series': marked, 'desc': desc, 'label': pts['label']}
        # 2) 趋势预测（同一条序列）
        fdesc, future = linear_forecast(pts['points'])
        context['forecast'] = {'desc': fdesc, 'future': future, 'series': pts['points']}

    # 3) 相关性矩阵（核心指标集两两皮尔逊）
    smap = {}
    for k in CORE_KEYS:
        s = series(k, minutes)
        if s and s['points']:
            smap[k] = s['points']
    keys, mat = correlation_matrix(smap)
    context['corr'] = {'keys': keys, 'matrix': mat}

    # 4) 日志模板挖掘（最近 1 小时 WARNING+）
    entries = LogEntry.objects.filter(
        created_at__gte=timezone.now() - timedelta(hours=1))
    context['log_patterns'] = mine_log_patterns(entries)[:10]

    return render(request, 'analytics/analytics.html', context)

from django.urls import path

from . import views

urlpatterns = [
    path('', views.analytics, name='analytics'),
]

{% extends "base.html" %}
{% block title %}智能分析{% endblock %}
{% block content %}
<h1>智能分析</h1>
<form method="get">
  指标：
  <select name="key">
    <option value="http.avg_duration" {% if key == 'http.avg_duration' %}selected{% endif %}>接口平均耗时</option>
    <option value="http.error_rate" {% if key == 'http.error_rate' %}selected{% endif %}>接口错误率</option>
    <option value="host.cpu_percent" {% if key == 'host.cpu_percent' %}selected{% endif %}>主机 CPU</option>
    <option value="rum.load_time_avg" {% if key == 'rum.load_time_avg' %}selected{% endif %}>页面加载时长</option>
  </select>
  时间范围：<input name="minutes" value="{{ minutes }}" size="4"> 分钟
  <button>分析</button>
</form>

<h3>① 异常检测（{{ anomaly.desc.method }}）— {{ anomaly.desc.anomalies }}/{{ anomaly.desc.total }} 个异常点</h3>
<div id="c-anomaly" style="width:100%;height:300px;background:#fff;"></div>
{{ anomaly.series|json_script:"d-anomaly" }}

<h3>② 趋势预测：{{ forecast.desc.trend }}（斜率 {{ forecast.desc.slope }}，R²={{ forecast.desc.r2 }}）</h3>
<div id="c-forecast" style="width:100%;height:280px;background:#fff;"></div>
{{ forecast.series|json_script:"d-forecast" }}
{{ forecast.future|json_script:"d-future" }}

<h3>③ 指标相关性矩阵（皮尔逊 r）</h3>
<table border="1" cellpadding="6" style="border-collapse:collapse;background:#fff;font-size:12px;">
  <tr><th></th>{% for k in corr.keys %}<th>{{ k }}</th>{% endfor %}</tr>
  {% for row in corr.matrix %}
  <tr><th>{{ corr.keys|get:forloop.counter0 }}</th>
    {% for v in row %}<td>{{ v|default:"—" }}</td>{% endfor %}
  </tr>
  {% endfor %}
</table>
<p>解读：|r|&gt;0.7 强相关；相关≠因果，结论需结合调用链与业务确认。</p>

<h3>④ 日志模式 Top {{ log_patterns|length }}（最近 1 小时）</h3>
<table border="1" cellpadding="6" style="border-collapse:collapse;background:#fff;">
  <tr><th>次数</th><th>模板</th><th>级别分布</th><th>示例</th></tr>
  {% for g in log_patterns %}
  <tr>
    <td><b>{{ g.n }}</b></td>
    <td><code>{{ g.pattern }}</code></td>
    <td>{{ g.levels }}</td>
    <td>{{ g.sample|truncatechars:80 }}</td>
  </tr>
  {% endfor %}
</table>
{% endblock %}

{% block scripts %}
<script src="/static/echarts.min.js"></script>
<script>
  const anom = JSON.parse(document.getElementById('d-anomaly').textContent);
  const c1 = echarts.init(document.getElementById('c-anomaly'));
  c1.setOption({
    tooltip: { trigger: 'axis' },
    xAxis: { type: 'category', data: anom.map(p => p.t) },
    yAxis: { type: 'value' },
    series: [{
      type: 'line', data: anom.map(p => p.v), smooth: true, symbol: 'none',
      // 异常点单独标红（a=1 的点）
      markPoint: {
        data: anom.map((p, i) => p.a ? { coord: [i, p.v], value: 'z=' + (p.z ?? '?') } : null)
                  .filter(Boolean)
      },
    }],
  });
  const hist = JSON.parse(document.getElementById('d-forecast').textContent);
  const fut = JSON.parse(document.getElementById('d-future').textContent);
  const c2 = echarts.init(document.getElementById('c-forecast'));
  c2.setOption({
    tooltip: { trigger: 'axis' },
    xAxis: { type: 'category', data: hist.map(p => p.t).concat(fut.map(p => p.t)) },
    yAxis: { type: 'value' },
    series: [
      { type: 'line', data: hist.map(p => p.v), symbol: 'none', smooth: true },
      { type: 'line', data: hist.map(() => null).concat(fut.map(p => p.v)),
        lineStyle: { type: 'dashed' }, symbol: 'none' },
    ],
  });
</script>
{% endblock %}

@login_required
def dashboard(request):
    """自定义大盘：卡片网格 + 增删（卡片按 order 排列，宽度 1-3 栅格）"""
    cards = list(DashCard.objects.all())
    catalog_items = catalog()
    if request.method == 'POST':
        DashCard.objects.create(
            title=request.POST.get('title') or '未命名卡片',
            metric_key=request.POST.get('metric_key'),
            chart_type=request.POST.get('chart_type', 'line'),
            minutes=min(1440, max(15, int(request.POST.get('minutes', 60)))),
            span=min(3, max(1, int(request.POST.get('span', 1)))),
            order=int(request.POST.get('order', 0)),
        )
        return redirect('monitor:dashboard')
    # 取数：每个卡片一条 series
    data = {}
    for c in cards:
        s = series(c.metric_key, c.minutes)
        data[c.pk] = s['points'] if s else []
    return render(request, 'monitor/dashboard.html',
                  {'cards': cards, 'data': data, 'metrics': catalog_items})


@login_required
@require_POST
def dashboard_delete(request, pk):
    DashCard.objects.filter(pk=pk).delete()
    return redirect('monitor:dashboard')

{% extends "base.html" %}
{% block title %}自定义大盘{% endblock %}
{% block content %}
<h1>自定义大盘</h1>
<form method="post" style="background:#fff;padding:12px;">
  {% csrf_token %}
  <input name="title" placeholder="卡片标题" required>
  <select name="metric_key">
    {% for m in metrics %}<option value="{{ m.key }}">{{ m.label }}</option>{% endfor %}
  </select>
  <select name="chart_type">
    <option value="line">折线</option><option value="area">面积</option><option value="bar">柱状</option>
  </select>
  <input name="minutes" type="number" value="60" min="15" max="1440" style="width:80px"> 分钟
  宽度：
  <select name="span"><option value="1">1/3</option><option value="2">2/3</option><option value="3">整行</option></select>
  <button>添加卡片</button>
</form>

<div style="display:grid;grid-template-columns:repeat(3,1fr);gap:12px;margin-top:12px;">
  {% for c in cards %}
  <div style="background:#fff;padding:12px;grid-column:span {{ c.span }};">
    <b>{{ c.title }}</b> <small>{{ c.metric_key }} · {{ c.minutes }}min</small>
    <form method="post" action="{% url 'monitor:dashboard_delete' c.pk %}" style="float:right">
      {% csrf_token %}<button style="color:#dc2626">删</button>
    </form>
    <div id="card-{{ c.pk }}" style="height:220px;"></div>
    {{ data|keyvalue:c.pk|json_script }}
  </div>
  {% endfor %}
</div>

<script src="/static/echarts.min.js"></script>
<script>
  {% for c in cards %}
  (function () {
    const el = document.getElementById('card-{{ c.pk }}');
    const raw = document.getElementById('json-card-{{ c.pk }}');
    if (!el || !raw) return;
    const pts = JSON.parse(raw.textContent);
    echarts.init(el).setOption({
      tooltip: { trigger: 'axis' },
      xAxis: { type: 'category', data: pts.map(p => p.t) },
      yAxis: { type: 'value' },
      series: [{
        type: '{{ c.chart_type }}',
        data: pts.map(p => p.v),
        smooth: true, symbol: 'none',
        areaStyle: '{{ c.chart_type }}' === 'area' ? {} : undefined,
      }],
    });
  })();
  {% endfor %}
</script>
{% endblock %}

@login_required
def apm(request):
    """APM：最近的请求（调用链）列表，按耗时/时间排序"""
    from .models import RequestMetric
    order = request.GET.get('order', 'time')       # time / duration / error
    qs = RequestMetric.objects.all()
    if order == 'duration':
        qs = qs.order_by('-duration_ms')
    elif order == 'error':
        qs = qs.filter(is_error=True)
    return render(request, 'monitor/apm.html', {'items': qs[:100], 'order': order})


@login_required
def trace_detail(request, pk):
    """调用链详情：spans JSON -> ECharts custom series 瀑布图"""
    from .models import RequestMetric
    m = get_object_or_404(RequestMetric, pk=pk)
    return render(request, 'monitor/trace.html', {'m': m})

{% extends "base.html" %}
{% block title %}调用链详情{% endblock %}
{% block content %}
<h1>{{ m.method }} {{ m.path }} <small>{{ m.duration_ms }}ms · {{ m.status_code }}</small></h1>
<p>TraceID: <code>{{ m.trace_id }}</code> · 视图 <code>{{ m.view_name|default:"—" }}</code>
 · {{ m.sql_count }} 条 SQL / {{ m.sql_time_ms }}ms</p>
<div id="waterfall" style="width:100%;height:420px;background:#fff;"></div>
{{ m.spans|json_script:"spans" }}
<script src="/static/echarts.min.js"></script>
<script>
  const spans = JSON.parse(document.getElementById('spans').textContent);
  const colors = { request: '#64748b', sql: '#2563eb', view: '#059669' };
  echarts.init(document.getElementById('waterfall')).setOption({
    tooltip: {
      formatter: p => `${p.value[2]}<br>偏移 ${p.value[0]}ms · 耗时 ${p.value[1]}ms`,
    },
    xAxis: { type: 'value', name: 'ms' },
    yAxis: { type: 'category',
             data: spans.map((s, i) => s.kind + ' ' + i).reverse(),
             inverse: false },
    series: [{
      type: 'custom',
      renderItem: (params, api) => {
        const off = api.coord([api.value(0), params.dataIndex]);
        const dur = api.size([api.value(1), 0])[0];
        const barH = Math.min(18, api.size([0, 1])[1] * 0.6);
        const y = off[1] - barH / 2;
        return {
          type: 'rect',
          shape: { x: off[0], y: y, width: Math.max(dur, 2), height: barH,
                   r: 3 },
          style: { fill: colors[spans[params.dataIndex].kind] || '#94a3b8' },
        };
      },
      encode: { x: [0, 1], y: 1 },
      data: spans.map(s => [s.off, s.dur, s.kind + ' · ' + s.name]),
    }],
  });
</script>
{% endblock %}