"""Service layer for M&EL indicators.

All persistent mutations should go through these helpers so that receivers,
HTMX views, and the API layer share one idempotent write path.
"""
from __future__ import annotations

import logging
from dataclasses import dataclass
from datetime import date
from decimal import Decimal
from typing import Iterable

from django.conf import settings
from django.core.exceptions import ValidationError
from django.db import transaction
from django.utils import timezone

from apps.mel.indicators.models import (
    AlertEscalation,
    AlertEscalationRule,
    DataPoint,
    EvidenceAttachment,
    ImpactDatapoint,
    ImpactEvaluation,
    Indicator,
    IndicatorSilentAlert,
    IndicatorState,
    IndicatorTarget,
    IndicatorVariance,
    LogFrame,
    LogFrameLevel,
    LogFrameRow,
    ProgrammeHealthScore,
    RagStatus,
    RagThreshold,
    ValidationRule,
)


# ---------------------------------------------------------------------------
# G9 — Custom validation rules
# ---------------------------------------------------------------------------


class DataPointValidationError(ValueError):
    """Raised when a value violates an active :class:`ValidationRule`.

    Carries enough metadata (rule_id, field, code) to render a structured
    form error pointing at the offending rule.
    """

    def __init__(self, *, rule_id: int | None, code: str, message: str, field: str = "value"):
        self.rule_id = rule_id
        self.code = code
        self.field = field
        super().__init__(message)


_DEFAULT_RULE_MESSAGES = {
    "range": "Value out of allowed range.",
    "regex": "Value does not match the required pattern.",
    "comparison": "Value fails the configured comparison.",
    "custom_expression": "Value rejected by custom validation rule.",
}


def _evaluate_custom_expression(expression: str, *, value, previous=None) -> bool:
    """Evaluate ``expression`` in a sandboxed simpleeval context.

    Names allowed: ``value`` (always), ``previous`` (when supplied).
    Functions allowed: ``min``, ``max``, ``abs``.
    All other names, attribute access, imports, and bare ``__builtins__`` are
    blocked by simpleeval's AST whitelist.
    """
    from simpleeval import EvalWithCompoundTypes

    evaluator = EvalWithCompoundTypes(
        names={"value": float(value), "previous": (float(previous) if previous is not None else None)},
        functions={"min": min, "max": max, "abs": abs},
    )
    return bool(evaluator.eval(expression))


def validate_value(
    indicator: Indicator,
    value,
    *,
    previous_value=None,
) -> None:
    """Run every active :class:`ValidationRule` on ``value`` for ``indicator``.

    Raises :class:`DataPointValidationError` on the first failure. Inactive
    rules are skipped. A rule with malformed params is logged and ignored
    (defensive — bad config should not block data entry).
    """
    import re

    rules = list(
        ValidationRule.objects.filter(indicator=indicator, is_active=True).order_by("pk")
    )
    if not rules:
        return
    try:
        value_f = float(value)
    except (TypeError, ValueError):
        value_f = None

    for rule in rules:
        params = rule.params or {}
        msg = rule.message or _DEFAULT_RULE_MESSAGES.get(rule.rule_type, "Value invalid.")
        try:
            if rule.rule_type == ValidationRule.RuleType.RANGE:
                if value_f is None:
                    raise DataPointValidationError(rule_id=rule.pk, code="range", message=msg)
                lo = params.get("min")
                hi = params.get("max")
                if lo is not None and value_f < float(lo):
                    raise DataPointValidationError(rule_id=rule.pk, code="range", message=msg)
                if hi is not None and value_f > float(hi):
                    raise DataPointValidationError(rule_id=rule.pk, code="range", message=msg)
            elif rule.rule_type == ValidationRule.RuleType.REGEX:
                pattern = params.get("pattern", "")
                if pattern and not re.search(pattern, str(value)):
                    raise DataPointValidationError(rule_id=rule.pk, code="regex", message=msg)
            elif rule.rule_type == ValidationRule.RuleType.COMPARISON:
                op = params.get("op")
                other = params.get("other")
                if value_f is None or op is None or other is None:
                    raise DataPointValidationError(rule_id=rule.pk, code="comparison", message=msg)
                other_f = float(other)
                cmp_map = {
                    ">": value_f > other_f,
                    ">=": value_f >= other_f,
                    "<": value_f < other_f,
                    "<=": value_f <= other_f,
                    "==": value_f == other_f,
                    "!=": value_f != other_f,
                }
                if not cmp_map.get(op, False):
                    raise DataPointValidationError(rule_id=rule.pk, code="comparison", message=msg)
            elif rule.rule_type == ValidationRule.RuleType.CUSTOM_EXPRESSION:
                expr = params.get("expression", "")
                if not expr:
                    continue
                # Any exception inside simpleeval (FeatureNotAvailable,
                # InvalidExpression, AttributeDoesNotExist) is reported as a
                # validation failure — better to refuse the data point than
                # silently accept it when the rule is broken.
                try:
                    ok = _evaluate_custom_expression(expr, value=value_f if value_f is not None else value, previous=previous_value)
                except DataPointValidationError:
                    raise
                except Exception as exc:
                    logger.warning(
                        "mel.validation custom_expression error indicator=%s rule=%s err=%s",
                        indicator.code, rule.pk, exc,
                    )
                    raise DataPointValidationError(
                        rule_id=rule.pk, code="custom_expression",
                        message=msg or f"Expression error: {exc}",
                    )
                if not ok:
                    raise DataPointValidationError(
                        rule_id=rule.pk, code="custom_expression", message=msg,
                    )
        except DataPointValidationError:
            raise
        except Exception as exc:  # pragma: no cover - defensive
            logger.warning(
                "mel.validation malformed-rule indicator=%s rule=%s err=%s",
                indicator.code, rule.pk, exc,
            )
            continue

logger = logging.getLogger(__name__)


# ---------------------------------------------------------------------------
# Hierarchy helpers
# ---------------------------------------------------------------------------


def validate_hierarchy(logframe: LogFrame) -> list[str]:
    """Return a list of human-readable issues or an empty list when valid.

    Enforces PRD §5.3.4:
      • every Output has ≥1 Outcome ancestor
      • every Outcome has ≥1 Impact ancestor
      • every Activity has ≥1 Output ancestor
    """
    issues: list[str] = []
    rows = list(logframe.rows.select_related("parent"))
    by_pk = {r.pk: r for r in rows}

    def ancestor_levels(row: LogFrameRow) -> set[str]:
        levels: set[str] = set()
        cur = row
        while cur.parent_id:
            parent = by_pk.get(cur.parent_id)
            if parent is None:
                break
            levels.add(parent.level)
            cur = parent
        return levels

    for row in rows:
        anc = ancestor_levels(row)
        if row.level == LogFrameLevel.OUTPUT and LogFrameLevel.OUTCOME not in anc:
            issues.append(f"Output '{row.title}' has no Outcome ancestor.")
        elif row.level == LogFrameLevel.OUTCOME and LogFrameLevel.IMPACT not in anc:
            issues.append(f"Outcome '{row.title}' has no Impact ancestor.")
        elif row.level == LogFrameLevel.ACTIVITY and LogFrameLevel.OUTPUT not in anc:
            issues.append(f"Activity '{row.title}' has no Output ancestor.")
    return issues


# ---------------------------------------------------------------------------
# Data points
# ---------------------------------------------------------------------------


@dataclass(frozen=True)
class DataPointResult:
    data_point: DataPoint
    created: bool


@transaction.atomic
def record_data_point(
    *,
    indicator: Indicator,
    value: Decimal | float | int,
    period_label: str,
    reported_at=None,
    source_module: str = "",
    source_event: str = "",
    source_object_id: str = "",
    reported_by=None,
    qualitative_note: str = "",
    auto_verify: bool = False,
    target: IndicatorTarget | None = None,
    raise_on_invalid: bool = True,
) -> DataPointResult:
    """Idempotent write of a data point.

    When ``source_module`` and ``source_object_id`` are both set, existing rows
    with the same triple (indicator, source_module, source_object_id) are
    returned unchanged — this makes receiver code safe against replayed
    signals and outbox retries.

    G9 — runs :func:`validate_value` against active validation rules. When
    ``raise_on_invalid=True`` (default) a :class:`DataPointValidationError`
    is raised. Set to ``False`` from signal-driven callers so a bad upstream
    payload doesn't crash the source transaction; the violation is logged
    and the data point is not written.
    """
    value_d = Decimal(str(value))

    if source_module and source_object_id:
        existing = DataPoint.objects.filter(
            indicator=indicator,
            source_module=source_module,
            source_object_id=source_object_id,
        ).first()
        if existing is not None:
            return DataPointResult(existing, False)

    # G9 — enforce validation rules before the write. Look up the previous
    # period's value so COMPARISON / CUSTOM_EXPRESSION rules can reference it.
    previous_value = (
        DataPoint.objects.filter(indicator=indicator)
        .exclude(period_label=period_label)
        .order_by("-reported_at")
        .values_list("value", flat=True)
        .first()
    )
    try:
        validate_value(indicator, value_d, previous_value=previous_value)
    except DataPointValidationError as exc:
        if raise_on_invalid:
            raise
        logger.warning(
            "mel.datapoint validation-skip indicator=%s rule=%s code=%s msg=%s",
            indicator.code, exc.rule_id, exc.code, exc,
        )
        # Caller asked us not to raise — return a sentinel-like result with
        # no DataPoint. Callers should check ``created`` to know if a write
        # actually happened.
        return DataPointResult(None, False)  # type: ignore[arg-type]

    if target is None:
        target = _target_for_period(indicator, period_label)

    dp = DataPoint.objects.create(
        indicator=indicator,
        target=target,
        value=value_d,
        period_label=period_label,
        reported_at=reported_at or timezone.now(),
        source_module=source_module,
        source_event=source_event,
        source_object_id=source_object_id,
        reported_by=reported_by,
        qualitative_note=qualitative_note,
        status=DataPoint.Status.VERIFIED if auto_verify else DataPoint.Status.PENDING,
        verified_at=timezone.now() if auto_verify else None,
    )
    logger.info(
        "mel.datapoint recorded indicator=%s value=%s source=%s/%s",
        indicator.code,
        value_d,
        source_module,
        source_event,
    )

    # Emit signal (fires after the transaction commits so listeners observe a
    # persisted, verified row). Listeners include tracking.receivers which
    # re-scores RAG and, on boundary crossing, fires mel_indicator_threshold_crossed.
    transaction.on_commit(
        lambda: _emit_data_point_recorded(indicator=indicator, data_point=dp, period_label=period_label)
    )

    return DataPointResult(dp, True)


def _emit_data_point_recorded(*, indicator, data_point, period_label):
    # Lazy import — signals live in tracking to keep the module graph acyclic.
    from apps.mel.tracking.signals import mel_data_point_recorded

    mel_data_point_recorded.send(
        sender=Indicator,
        indicator=indicator,
        data_point=data_point,
        period_label=period_label,
    )


def _target_for_period(indicator: Indicator, period_label: str) -> IndicatorTarget | None:
    return indicator.targets.filter(period_label=period_label).first()


# M&E SRS Table 65 — a data point may only be validated (verified/rejected)
# while it is awaiting review. A REJECTED point must be resubmitted (→ PENDING)
# before it can be verified, and a VERIFIED point that has already moved
# indicator values cannot be silently rejected.
_VALIDATABLE_STATES = frozenset(
    {DataPoint.Status.PENDING, DataPoint.Status.NEEDS_CLARIFICATION}
)


@transaction.atomic
def verify_datapoint(data_point_id: int, *, verified_by) -> DataPoint:
    """Mark a PENDING data point as VERIFIED."""
    dp = DataPoint.objects.select_for_update().get(pk=data_point_id)
    if dp.status == DataPoint.Status.VERIFIED:
        return dp
    if dp.status not in _VALIDATABLE_STATES:
        raise ValidationError(
            f"A {dp.get_status_display().lower()} data point cannot be verified — "
            "it must be resubmitted for review first."
        )
    dp.status = DataPoint.Status.VERIFIED
    dp.verified_by = verified_by
    dp.verified_at = timezone.now()
    dp.save(update_fields=["status", "verified_by", "verified_at"])
    transaction.on_commit(
        lambda: _emit_data_point_recorded(
            indicator=dp.indicator, data_point=dp, period_label=dp.period_label,
        )
    )
    return dp


@transaction.atomic
def reject_datapoint(data_point_id: int, *, rejected_by, reason: str) -> DataPoint:
    """Mark a PENDING data point as REJECTED with an auditable reason."""
    dp = DataPoint.objects.select_for_update().get(pk=data_point_id)
    if dp.status == DataPoint.Status.REJECTED:
        return dp
    if dp.status not in _VALIDATABLE_STATES:
        raise ValidationError(
            f"A {dp.get_status_display().lower()} data point cannot be rejected — "
            "its value has already updated the indicator."
        )
    dp.status = DataPoint.Status.REJECTED
    dp.verified_by = rejected_by
    dp.verified_at = timezone.now()
    if reason:
        dp.qualitative_note = (
            f"{dp.qualitative_note}\n\n[Rejected] {reason}".strip()
        )
    dp.save(update_fields=["status", "verified_by", "verified_at", "qualitative_note"])

    if rejected_by is not None and dp.reported_by_id and dp.reported_by_id != rejected_by.pk:
        transaction.on_commit(
            lambda: _notify_datapoint_rejected(dp, reason=reason)
        )
    return dp


def _notify_datapoint_rejected(dp: DataPoint, *, reason: str) -> None:
    from apps.core.notifications.models import Notification
    from apps.core.notifications.services import send_notification

    send_notification(
        dp.reported_by,
        f"Your data point for {dp.indicator.code} ({dp.period_label}) was rejected: {reason}",
        verb=Notification.Verb.MEL_DATA_POINT_REJECTED,
    )


@transaction.atomic
def request_datapoint_clarification(data_point_id: int, *, requested_by, note: str) -> DataPoint:
    """M&E SRS Table 65 — the M&E Officer asks the reporter to clarify/correct.

    The record moves to NEEDS_CLARIFICATION and remains out of verified totals
    until it is resubmitted and revalidated.
    """
    dp = DataPoint.objects.select_for_update().get(pk=data_point_id)
    dp.status = DataPoint.Status.NEEDS_CLARIFICATION
    dp.clarification_note = (note or "").strip()
    dp.save(update_fields=["status", "clarification_note"])
    if (
        requested_by is not None
        and dp.reported_by_id
        and dp.reported_by_id != requested_by.pk
    ):
        transaction.on_commit(lambda: _notify_datapoint_clarification(dp, note=note))
    return dp


def _notify_datapoint_clarification(dp: DataPoint, *, note: str) -> None:
    from django.urls import reverse

    from apps.core.notifications.models import Notification
    from apps.core.notifications.services import send_notification

    send_notification(
        dp.reported_by,
        f"Clarification requested on your data point for {dp.indicator.code} "
        f"({dp.period_label}): {note}",
        verb=Notification.Verb.MEL_DATA_POINT_CLARIFICATION,
        action_url=reverse("mel_indicators:validation_queue"),
    )


@transaction.atomic
def resubmit_datapoint(
    data_point_id: int, *, resubmitted_by, value=None, note: str = ""
) -> DataPoint:
    """M&E SRS — the reporter corrects a clarification-flagged point.

    Returns it to PENDING so the M&E Officer can revalidate.
    """
    dp = DataPoint.objects.select_for_update().get(pk=data_point_id)
    if value is not None:
        dp.value = value
    if note:
        dp.qualitative_note = f"{dp.qualitative_note}\n\n[Resubmitted] {note}".strip()
    dp.status = DataPoint.Status.PENDING
    dp.clarification_note = ""
    dp.save(update_fields=["value", "qualitative_note", "status", "clarification_note"])
    return dp


def evidence_gaps(logframe: LogFrame | None, period_label: str) -> list[DataPoint]:
    """M&E SRS Table 65 (A2) — verified data points lacking supporting evidence.

    A data point counts as evidenced when its own ``evidence`` file is set or
    at least one :class:`EvidenceAttachment` targets it. "Indicators lacking
    supporting evidence shall not be finalized for reporting."
    """
    from django.contrib.contenttypes.models import ContentType

    qs = (
        DataPoint.objects.filter(
            status=DataPoint.Status.VERIFIED,
            period_label=period_label,
            evidence="",
        )
        .select_related("indicator__responsible", "indicator__logframe_row__logframe")
        .order_by("indicator__code", "pk")
    )
    if logframe is not None:
        qs = qs.filter(indicator__logframe_row__logframe=logframe)
    ct = ContentType.objects.get_for_model(DataPoint)
    attached_ids = set(
        EvidenceAttachment.objects.filter(
            content_type=ct, object_id__in=[dp.pk for dp in qs]
        ).values_list("object_id", flat=True)
    )
    return [dp for dp in qs if dp.pk not in attached_ids]


def notify_evidence_required(gaps: list[DataPoint]) -> int:
    """Notify each responsible officer that evidence is required (one per indicator)."""
    from apps.core.notifications.models import Notification
    from apps.core.notifications.services import send_notification

    notified = 0
    seen: set[int] = set()
    for dp in gaps:
        indicator = dp.indicator
        recipient = indicator.responsible
        if recipient is None or indicator.pk in seen:
            continue
        seen.add(indicator.pk)
        send_notification(
            recipient,
            f"Supporting evidence required for {indicator.code} "
            f"({dp.period_label}) before reporting can be finalized.",
            verb=Notification.Verb.MEL_EVIDENCE_REQUIRED,
        )
        notified += 1
    return notified


# ---------------------------------------------------------------------------
# RAG thresholds
# ---------------------------------------------------------------------------


def get_rag_thresholds(logframe: LogFrame | None) -> tuple[float, float]:
    """Return ``(green_min_percent, amber_min_percent)`` for the given logframe.

    Falls back to ``settings.MEL_RAG_DEFAULT`` when no per-logframe row exists.
    """
    default = getattr(settings, "MEL_RAG_DEFAULT", {"green": 85.0, "amber": 60.0})
    if logframe is None:
        return float(default["green"]), float(default["amber"])
    row = getattr(logframe, "rag_threshold", None)
    if row is None:
        try:
            row = RagThreshold.objects.get(logframe=logframe)
        except RagThreshold.DoesNotExist:
            return float(default["green"]), float(default["amber"])
    return float(row.green_min), float(row.amber_min)


# ---------------------------------------------------------------------------
# Progress / RAG
# ---------------------------------------------------------------------------


def sum_period_value(indicator: Indicator, period_label: str) -> Decimal:
    agg = (
        DataPoint.objects.filter(
            indicator=indicator,
            period_label=period_label,
            status=DataPoint.Status.VERIFIED,
        )
        .values_list("value", flat=True)
    )
    total = Decimal("0")
    for v in agg:
        total += v
    return total


def aggregate_results_by_level(logframe: LogFrame, period_label: str) -> dict[str, dict]:
    """M&E SRS Table 65 — roll validated data up the results chain.

    Returns ``{level: {"actual": Decimal, "target": Decimal, "indicator_count": int}}``
    for the Output, Outcome (Intermediate Outcome) and Impact levels. Each level's
    ``actual`` sums the VERIFIED data-point values of every indicator attached to
    a row *at or below* that level in the causal chain (following the approved
    Results Framework hierarchy), so Outputs roll into Outcomes and Outcomes into
    Impacts.
    """
    from apps.mel.indicators.models import LogFrameLevel

    rows = list(logframe.rows.select_related("parent").prefetch_related("indicators"))
    by_pk = {r.pk: r for r in rows}

    def ancestors(row):
        cur = row
        while cur.parent_id:
            parent = by_pk.get(cur.parent_id)
            if parent is None:
                break
            yield parent
            cur = parent

    levels = (LogFrameLevel.OUTPUT, LogFrameLevel.OUTCOME, LogFrameLevel.IMPACT)
    result = {
        level: {"actual": Decimal("0"), "target": Decimal("0"), "indicator_count": 0}
        for level in levels
    }

    for row in rows:
        indicators = list(row.indicators.all())
        if not indicators:
            continue
        # The set of levels this row contributes to: its own (if aggregatable)
        # plus every ancestor level.
        contributes_to = set()
        if row.level in result:
            contributes_to.add(row.level)
        for anc in ancestors(row):
            if anc.level in result:
                contributes_to.add(anc.level)
        if not contributes_to:
            continue
        for ind in indicators:
            actual = sum_period_value(ind, period_label)
            target = (
                ind.targets.filter(period_label=period_label)
                .values_list("target_value", flat=True)
                .first()
            )
            for level in contributes_to:
                result[level]["actual"] += actual
                if target is not None:
                    result[level]["target"] += target
                result[level]["indicator_count"] += 1

    return {str(level): data for level, data in result.items()}


def _rag_for_percent(percent: float | None, *, green_min: float, amber_min: float) -> str:
    if percent is None:
        return RagStatus.UNKNOWN
    if percent >= green_min:
        return RagStatus.GREEN
    if percent >= amber_min:
        return RagStatus.AMBER
    return RagStatus.RED


def calculate_progress(indicator: Indicator, period_label: str) -> dict:
    """Return ``{actual, target, baseline, percent, rag}`` for the given period."""
    target = indicator.targets.filter(period_label=period_label).first()
    actual = sum_period_value(indicator, period_label)
    baseline = target.baseline_value if target else None
    target_value = target.target_value if target else None

    percent: float | None = None
    if target_value is not None and target_value != 0:
        percent = float(actual) / float(target_value) * 100

    lf = getattr(indicator.logframe_row, "logframe", None)
    green_min, amber_min = get_rag_thresholds(lf)
    rag = _rag_for_percent(percent, green_min=green_min, amber_min=amber_min)

    return {
        "actual": actual,
        "target": target_value,
        "baseline": baseline,
        "percent": percent,
        "rag": rag,
        "period_label": period_label,
        "green_min": green_min,
        "amber_min": amber_min,
    }


def flag_off_track(
    indicators: Iterable[Indicator] | None = None,
    *,
    period_label: str,
) -> list[dict]:
    """Return progress rows for indicators below the AMBER threshold."""
    # M&E SRS Table 64 — DRAFT indicators are not yet available for monitoring.
    qs = (
        indicators
        if indicators is not None
        else Indicator.objects.filter(is_active=True).exclude(
            workflow_status=IndicatorState.DRAFT
        )
    )
    off_track: list[dict] = []
    for indicator in qs:
        progress = calculate_progress(indicator, period_label)
        if progress["rag"] in (RagStatus.RED, RagStatus.AMBER):
            progress["indicator"] = indicator
            off_track.append(progress)
    return off_track


# ---------------------------------------------------------------------------
# Variance (FRMFL028–029)
# ---------------------------------------------------------------------------


@transaction.atomic
def compute_variance(
    indicator: Indicator,
    period_label: str,
    *,
    cause_analysis: str = "",
    adjustment: str = "",
) -> IndicatorVariance:
    """Snapshot target vs actual variance for a reporting period.

    Replaces any previous row for the same (indicator, period) pair. Returns
    the saved :class:`IndicatorVariance` row.
    """
    progress = calculate_progress(indicator, period_label)
    target_value = progress["target"]
    actual_value = progress["actual"]
    variance_pct: Decimal | None = None
    variance_abs: Decimal | None = None
    if target_value is not None:
        try:
            variance_abs = Decimal(str(actual_value)) - Decimal(str(target_value))
        except Exception:  # pragma: no cover - defensive
            variance_abs = None
    if target_value is not None and Decimal(str(target_value)) != 0:
        try:
            diff = (Decimal(str(actual_value)) - Decimal(str(target_value)))
            variance_pct = (diff / Decimal(str(target_value)) * Decimal("100")).quantize(Decimal("0.0001"))
        except Exception:  # pragma: no cover - defensive
            variance_pct = None

    row, _ = IndicatorVariance.objects.update_or_create(
        indicator=indicator,
        period_label=period_label,
        defaults={
            "target_value": target_value,
            "actual_value": actual_value,
            "variance_pct": variance_pct,
            "variance_abs": variance_abs,
            "rating": variance_rating_for_pct(variance_pct),
            "cause_analysis": cause_analysis,
            "adjustment": adjustment,
        },
    )
    return row


def variance_rating_for_pct(variance_pct) -> str:
    """M&E SRS Table 66 — map a variance percentage to a qualitative rating.

    On Track (≥ −15%), Needs Attention (−50% < pct ≤ −15%), Critical (≤ −50%).
    Bands mirror the variance-triage tiles on the indicator variance page.
    """
    from apps.mel.indicators.models import VarianceRating

    if variance_pct is None:
        return ""
    pct = float(variance_pct)
    if pct <= -50:
        return VarianceRating.CRITICAL
    if pct <= -15:
        return VarianceRating.NEEDS_ATTENTION
    return VarianceRating.ON_TRACK


# ---------------------------------------------------------------------------
# Alerts (FRMFL012, FRMFL031)
# ---------------------------------------------------------------------------


def _indicator_alert_recipients(indicator: Indicator):
    """Users that should receive alerts when ``indicator`` goes off-track.

    Includes the responsible MEL officer and the logframe owner. De-duplicates.
    """
    seen: set[int] = set()
    out = []
    if indicator.responsible_id:
        if indicator.responsible_id not in seen:
            out.append(indicator.responsible)
            seen.add(indicator.responsible_id)
    lf = getattr(indicator.logframe_row, "logframe", None)
    owner = getattr(lf, "owner", None)
    if owner and owner.pk not in seen:
        out.append(owner)
        seen.add(owner.pk)
    return out


def dispatch_indicator_alert(indicator: Indicator, progress: dict) -> int:
    """Send MEL_INDICATOR_OFF_TRACK notifications when ``progress['rag']`` is RED.

    Returns the count of recipients notified. AMBER and GREEN are no-ops —
    callers typically check the RAG themselves and only call on RED.

    G13 — also creates an :class:`AlertEscalation` row at tier 1 (idempotent
    on indicator+period) and enqueues a countdown task to advance to tier 2
    if no acknowledgement arrives within the rule's window.
    """
    if progress.get("rag") != RagStatus.RED:
        return 0
    recipients = _indicator_alert_recipients(indicator)
    if not recipients:
        logger.info("mel.alert no-recipients indicator=%s", indicator.code)
        return 0

    from apps.core.notifications.models import Notification
    from apps.core.notifications.services import send_notification

    percent = progress.get("percent")
    percent_label = f"{percent:.1f}%" if percent is not None else "n/a"
    period_label = progress.get("period_label", "this period")
    message = (
        f"Indicator {indicator.code} ({indicator.name}) is off track — "
        f"{percent_label} of target for {period_label}."
    )
    for user in recipients:
        send_notification(user, message, verb=Notification.Verb.MEL_INDICATOR_OFF_TRACK)

    _open_alert_escalation(indicator, period_label)
    return len(recipients)


def _open_alert_escalation(indicator: Indicator, period_label: str) -> AlertEscalation | None:
    """Create or refresh the tier-1 escalation row + queue the tier-2 countdown.

    Idempotent on (indicator, period_label): if an open escalation already
    exists for this period, no new countdown is enqueued (the original one
    will fire). If a previous chain was acknowledged but the indicator has
    re-flared in a new period, a fresh chain is started for that new period.
    """
    rule = (
        AlertEscalationRule.objects
        .filter(logframe=getattr(indicator.logframe_row, "logframe", None), is_active=True)
        .first()
    )
    if rule is None:
        return None
    escalation, created = AlertEscalation.objects.get_or_create(
        indicator=indicator,
        period_label=period_label,
        defaults={"tier_reached": 1, "last_notified_at": timezone.now()},
    )
    if not created:
        return escalation
    # Schedule the tier-2 escalation. Lazy import keeps the module graph clean.
    try:
        from apps.mel.indicators.tasks import escalate_alert_task

        escalate_alert_task.apply_async(
            args=[escalation.pk],
            countdown=int(rule.hours_until_tier_2) * 3600,
        )
    except Exception:  # pragma: no cover — celery may be unavailable in tests
        logger.exception("mel.escalation enqueue failed pk=%s", escalation.pk)
    return escalation


@transaction.atomic
def escalate_alert(escalation_id: int) -> str:
    """Advance the escalation tier or no-op if acknowledged.

    Idempotent: replays after acknowledgement skip cleanly. Returns a short
    status string for telemetry / Celery results inspection.
    """
    from apps.core.notifications.models import Notification
    from apps.core.notifications.services import send_notification

    try:
        esc = AlertEscalation.objects.select_for_update().get(pk=escalation_id)
    except AlertEscalation.DoesNotExist:
        return "missing"

    if esc.acknowledged_at is not None:
        return "acknowledged"

    rule = (
        AlertEscalationRule.objects
        .filter(logframe=getattr(esc.indicator.logframe_row, "logframe", None))
        .first()
    )
    if rule is None or not rule.is_active:
        return "rule-removed"

    now = timezone.now()
    if esc.tier_reached >= 3:
        return "max-tier"

    if esc.tier_reached == 1:
        recipients = list(rule.tier_2_recipients.all())
        verb = Notification.Verb.MEL_INDICATOR_ESCALATION_TIER2
        next_countdown = int(rule.hours_until_tier_3) * 3600
        new_tier = 2
    else:  # tier_reached == 2
        recipients = list(rule.tier_3_recipients.all())
        verb = Notification.Verb.MEL_INDICATOR_ESCALATION_TIER3
        next_countdown = None
        new_tier = 3

    if not recipients:
        # Nobody to notify at this tier — log and stop the chain so we don't
        # spin forever.
        logger.info(
            "mel.escalation no-recipients escalation=%s tier=%s",
            esc.pk, new_tier,
        )
        esc.resolved_at = now
        esc.save(update_fields=["resolved_at", "updated_at"])
        return "no-recipients"

    message = (
        f"ESCALATED (tier {new_tier}): indicator {esc.indicator.code} "
        f"({esc.indicator.name}) for {esc.period_label} has not been "
        f"acknowledged."
    )
    for user in recipients:
        try:
            send_notification(user, message, verb=verb)
        except Exception:  # pragma: no cover - never break the chain
            logger.exception("mel.escalation notification failed user=%s", user.pk)

    esc.tier_reached = new_tier
    esc.last_notified_at = now
    esc.save(update_fields=["tier_reached", "last_notified_at", "updated_at"])

    if next_countdown is not None:
        try:
            from apps.mel.indicators.tasks import escalate_alert_task

            escalate_alert_task.apply_async(args=[esc.pk], countdown=next_countdown)
        except Exception:  # pragma: no cover
            logger.exception("mel.escalation re-enqueue failed pk=%s", esc.pk)
    return f"tier-{new_tier}"


@transaction.atomic
def acknowledge_escalation(escalation: AlertEscalation, *, user) -> AlertEscalation:
    """Mark an escalation acknowledged. Subsequent escalate_alert calls no-op."""
    fresh = AlertEscalation.objects.select_for_update().get(pk=escalation.pk)
    if fresh.acknowledged_at is not None:
        return fresh
    fresh.acknowledged_at = timezone.now()
    fresh.acknowledged_by = user
    fresh.save(update_fields=["acknowledged_at", "acknowledged_by", "updated_at"])
    logger.info(
        "mel.escalation acknowledged pk=%s by=%s",
        fresh.pk, getattr(user, "pk", None),
    )
    return fresh


# ---------------------------------------------------------------------------
# Helpers for receivers
# ---------------------------------------------------------------------------


def distinct_period_labels(indicator: Indicator | None = None) -> list[str]:
    """Return period labels known to the system, newest first.

    Scopes to a single indicator when provided — otherwise returns the union
    across every indicator's DataPoints and IndicatorTargets. Used to populate
    period <select> inputs on list + dashboard pages so operators pick from
    real periods rather than guess a label.
    """
    dp_qs = DataPoint.objects.all()
    tgt_qs = IndicatorTarget.objects.all()
    if indicator is not None:
        dp_qs = dp_qs.filter(indicator=indicator)
        tgt_qs = tgt_qs.filter(indicator=indicator)

    labels = set(
        dp_qs.exclude(period_label="").values_list("period_label", flat=True).distinct()
    )
    labels.update(
        tgt_qs.exclude(period_label="").values_list("period_label", flat=True).distinct()
    )
    # Reverse-lexical sort puts newer-looking labels first for the common
    # formats ('2026-Q2' > '2026-Q1', 'FY2026' > 'FY2025', '2026-04' > '2026-03').
    return sorted(labels, reverse=True)


def current_period_label(frequency: str, when: date | None = None) -> str:
    """Return a period label aligned to the indicator's reporting frequency."""
    from apps.mel.indicators.models import IndicatorFrequency  # local import avoids cycle

    d = when or timezone.now().date()
    match frequency:
        case IndicatorFrequency.DAILY:
            return d.strftime("%Y-%m-%d")
        case IndicatorFrequency.WEEKLY:
            iso = d.isocalendar()
            return f"{iso.year}-W{iso.week:02d}"
        case IndicatorFrequency.MONTHLY:
            return d.strftime("%Y-%m")
        case IndicatorFrequency.QUARTERLY:
            q = (d.month - 1) // 3 + 1
            return f"{d.year}-Q{q}"
        case IndicatorFrequency.SEMI_ANNUAL:
            half = 1 if d.month <= 6 else 2
            return f"{d.year}-H{half}"
        case IndicatorFrequency.ANNUAL:
            return f"FY{d.year}"
        case _:
            return d.strftime("%Y-%m-%d")


def record_automated_point(
    *,
    indicator_code: str,
    source_module: str,
    source_event: str,
    source_object_id: str,
    value: Decimal | float | int = 1,
    qualitative_note: str = "",
) -> DataPointResult | None:
    """Record a data point driven by an upstream signal; return None when no
    indicator with ``indicator_code`` exists (M&EL officer hasn't configured it
    yet — we log and move on rather than crash the upstream transaction)."""
    # M&E SRS Table 64 — only published (non-DRAFT) indicators are available for
    # monitoring; an upstream signal for a still-being-authored indicator is
    # logged and skipped rather than accruing auto-verified data against a draft.
    indicator = (
        Indicator.objects.filter(code=indicator_code, is_active=True)
        .exclude(workflow_status=IndicatorState.DRAFT)
        .first()
    )
    if indicator is None:
        logger.info(
            "mel.datapoint skipped (no indicator): code=%s source=%s/%s object=%s",
            indicator_code,
            source_module,
            source_event,
            source_object_id,
        )
        return None
    return record_data_point(
        indicator=indicator,
        value=value,
        period_label=current_period_label(indicator.frequency),
        source_module=source_module,
        source_event=source_event,
        source_object_id=source_object_id,
        qualitative_note=qualitative_note,
        auto_verify=True,
    )


# ---------------------------------------------------------------------------
# Impact evaluation analytics (FRMFL026)
# ---------------------------------------------------------------------------


@dataclass(frozen=True)
class CohensDResult:
    cohens_d: float
    mean_treatment: float
    mean_control: float
    pooled_std: float
    n_treatment: int
    n_control: int

    @property
    def magnitude(self) -> str:
        """Cohen's conventional thresholds: 0.2 / 0.5 / 0.8."""
        a = abs(self.cohens_d)
        if a < 0.2:
            return "negligible"
        if a < 0.5:
            return "small"
        if a < 0.8:
            return "medium"
        return "large"


def calculate_cohens_d(evaluation: ImpactEvaluation) -> CohensDResult | None:
    """Effect-size analysis on the treatment vs control datapoints of an
    impact evaluation (FRMFL026).

    Returns ``None`` when either group is empty, when both groups have a
    single observation (sample variance is undefined), or when the pooled
    standard deviation is zero.
    """
    rows = list(
        evaluation.datapoints.values_list("group", "value")
    )
    treatment = [
        float(v) for g, v in rows if g == ImpactDatapoint.Group.TREATMENT
    ]
    control = [
        float(v) for g, v in rows if g == ImpactDatapoint.Group.CONTROL
    ]

    n_t, n_c = len(treatment), len(control)
    if n_t == 0 or n_c == 0:
        return None
    if n_t + n_c < 3:
        return None

    mean_t = sum(treatment) / n_t
    mean_c = sum(control) / n_c

    def _sample_variance(values: list[float], mean: float) -> float:
        if len(values) < 2:
            return 0.0
        return sum((x - mean) ** 2 for x in values) / (len(values) - 1)

    var_t = _sample_variance(treatment, mean_t)
    var_c = _sample_variance(control, mean_c)
    pooled_var = ((n_t - 1) * var_t + (n_c - 1) * var_c) / (n_t + n_c - 2)
    pooled_std = pooled_var ** 0.5
    if pooled_std == 0:
        return None

    return CohensDResult(
        cohens_d=(mean_t - mean_c) / pooled_std,
        mean_treatment=mean_t,
        mean_control=mean_c,
        pooled_std=pooled_std,
        n_treatment=n_t,
        n_control=n_c,
    )


# ---------------------------------------------------------------------------
# G21 — Polymorphic evidence attachment
# ---------------------------------------------------------------------------


def attach_evidence(
    *,
    host,
    file,
    caption: str = "",
    mime_type: str = "",
    uploaded_by,
):
    """Attach ``file`` to ``host`` (any model). Idempotent on SHA-256 hash.

    Returns ``(attachment, created)``. When a row with the same
    (content_type, object_id, content_hash) already exists, returns the
    existing row with ``created=False`` — same-file replays are no-ops.
    """
    import hashlib

    from django.contrib.contenttypes.models import ContentType

    from apps.mel.indicators.models import EvidenceAttachment

    # Stream the file to compute SHA-256 without loading the whole thing.
    hasher = hashlib.sha256()
    file.seek(0)
    for chunk in iter(lambda: file.read(64 * 1024), b""):
        hasher.update(chunk)
    file.seek(0)
    content_hash = hasher.hexdigest()

    ct = ContentType.objects.get_for_model(host)
    existing = EvidenceAttachment.objects.filter(
        content_type=ct, object_id=host.pk, content_hash=content_hash,
    ).first()
    if existing is not None:
        return existing, False

    attachment = EvidenceAttachment.objects.create(
        content_type=ct,
        object_id=host.pk,
        file=file,
        content_hash=content_hash,
        mime_type=mime_type or _guess_mime(getattr(file, "name", "")),
        caption=caption,
        uploaded_by=uploaded_by,
    )
    return attachment, True


def _guess_mime(filename: str) -> str:
    import mimetypes

    if not filename:
        return ""
    mime, _ = mimetypes.guess_type(filename)
    return mime or ""


# ---------------------------------------------------------------------------
# G10 — Bulk CSV import for DataPoints
# ---------------------------------------------------------------------------


def _batch_id_for_file(file_bytes: bytes) -> str:
    """Hash a CSV's bytes for replay-idempotent ``source_object_id`` generation."""
    import hashlib

    return hashlib.sha256(file_bytes).hexdigest()[:16]


def _parse_decimal_value(raw):
    if raw is None or str(raw).strip() == "":
        raise ValueError("value is required")
    from decimal import InvalidOperation

    try:
        return Decimal(str(raw).strip())
    except InvalidOperation:
        # Defect 35 — Decimal() raises with a raw "[<class
        # 'decimal.ConversionSyntax'>]" payload; give the operator a sentence.
        raise ValueError("value must be a number")


def import_data_points_csv(
    *,
    csv_text: str,
    csv_bytes: bytes,
    user,
    dry_run: bool = True,
) -> dict:
    """Parse a CSV and route each row through :func:`record_data_point`.

    Replay idempotency: each row's ``source_object_id`` is
    ``f"{batch_id}:{row_idx}"`` where ``batch_id`` is a SHA-256 of the file
    bytes — the same file uploaded twice produces the same batch_id and the
    inner idempotency in ``record_data_point`` blocks duplicate writes.

    Returns ``{"batch_id", "successes": [...], "errors": [...]}``.
    Errors row format: ``{"row": int, "indicator_code": str, "error": str}``.
    """
    import csv
    import io

    batch_id = _batch_id_for_file(csv_bytes)
    successes: list[dict] = []
    errors: list[dict] = []

    reader = csv.DictReader(io.StringIO(csv_text))
    required_cols = {"indicator_code", "period_label", "value"}
    if reader.fieldnames is None or not required_cols.issubset(set(reader.fieldnames)):
        errors.append({
            "row": 0,
            "indicator_code": "",
            "error": f"CSV is missing required columns: {sorted(required_cols)}",
        })
        return {"batch_id": batch_id, "successes": [], "errors": errors}

    for idx, row in enumerate(reader, start=1):
        code = (row.get("indicator_code") or "").strip()
        period_label = (row.get("period_label") or "").strip()
        evidence_url = (row.get("evidence_url") or "").strip()
        notes = (row.get("notes") or "").strip()
        reported_at_raw = (row.get("reported_at") or "").strip() or None
        try:
            value = _parse_decimal_value(row.get("value"))
        except (ValueError, ArithmeticError) as exc:
            errors.append({"row": idx, "indicator_code": code, "error": str(exc)})
            continue
        if not code:
            errors.append({"row": idx, "indicator_code": "", "error": "indicator_code is required"})
            continue
        if not period_label:
            errors.append({"row": idx, "indicator_code": code, "error": "period_label is required"})
            continue
        try:
            indicator = Indicator.objects.get(code=code)
        except Indicator.DoesNotExist:
            errors.append({"row": idx, "indicator_code": code, "error": "indicator not found"})
            continue

        if dry_run:
            # In dry-run we still validate (G9) so the operator sees rule
            # violations in the preview.
            try:
                validate_value(indicator, value)
            except DataPointValidationError as exc:
                errors.append({"row": idx, "indicator_code": code, "error": str(exc)})
                continue
            successes.append({"row": idx, "indicator_code": code, "value": str(value), "period_label": period_label})
            continue

        try:
            from datetime import datetime as _dt

            reported_at = (
                timezone.make_aware(_dt.fromisoformat(reported_at_raw))
                if reported_at_raw else None
            )
            result = record_data_point(
                indicator=indicator,
                value=value,
                period_label=period_label,
                reported_at=reported_at,
                source_module="csv_import",
                source_event="csv_row",
                source_object_id=f"{batch_id}:{idx}",
                reported_by=user,
                qualitative_note=notes,
            )
        except DataPointValidationError as exc:
            errors.append({"row": idx, "indicator_code": code, "error": str(exc)})
            continue
        except Exception as exc:  # pragma: no cover - defensive
            errors.append({"row": idx, "indicator_code": code, "error": f"unexpected: {exc}"})
            continue
        # DataPoint has no URL field — surface the evidence_url in the row
        # success record so the operator can verify it was captured upstream
        # (the qualitative_note column carries any free text).
        successes.append({
            "row": idx,
            "indicator_code": code,
            "value": str(value),
            "period_label": period_label,
            "data_point_id": result.data_point.pk if result.data_point else None,
            "created": result.created,
        })

    return {"batch_id": batch_id, "successes": successes, "errors": errors}


# ---------------------------------------------------------------------------
# G7 — Performance notifications for non-reporting responsibles
# ---------------------------------------------------------------------------


def scan_silent_responsibles(*, today: date | None = None) -> int:
    """Find Indicators whose responsible user hasn't recorded data for the
    current period and dispatch one MEL_INDICATOR_SILENT notification per
    indicator/period pair.

    Idempotency: an :class:`IndicatorSilentAlert` row is created per
    (indicator, period_label). A repeat scan within the same period is a
    no-op because the unique constraint blocks duplicate inserts. When the
    period rolls and the user still hasn't reported, a new row is created
    and a new notification fires — that's the desired escalation behaviour.

    Returns the number of notifications dispatched.
    """
    from apps.core.notifications.models import Notification
    from apps.core.notifications.services import send_notification

    when = today or timezone.now().date()
    qs = (
        Indicator.objects.filter(is_active=True, responsible__isnull=False)
        # M&E SRS Table 64 — DRAFT indicators aren't monitored, so their
        # silence must not page anyone.
        .exclude(workflow_status=IndicatorState.DRAFT)
        .select_related("responsible", "logframe_row__logframe")
    )
    notified = 0
    for ind in qs:
        period_label = current_period_label(ind.frequency, when=when)
        if DataPoint.objects.filter(indicator=ind, period_label=period_label).exists():
            continue
        alert, created = IndicatorSilentAlert.objects.get_or_create(
            indicator=ind,
            period_label=period_label,
            defaults={"recipient": ind.responsible},
        )
        if not created:
            continue
        message = (
            f"Indicator {ind.code} ({ind.name}) is overdue — no data point recorded "
            f"for {period_label}. The indicator is on a {ind.get_frequency_display()} cadence."
        )
        send_notification(
            ind.responsible,
            message,
            verb=Notification.Verb.MEL_INDICATOR_SILENT,
        )
        notified += 1
    return notified


# ---------------------------------------------------------------------------
# G6 — Post-project monitoring (LogFrame lifecycle)
# ---------------------------------------------------------------------------


class LogFrameClosedError(ValueError):
    """Raised when a write is attempted on a closed log frame."""


def is_logframe_editable(logframe: LogFrame | None) -> bool:
    """Single gate for write operations against a log frame.

    Returns True when the log frame is editable (ACTIVE) or when there is no
    log frame at all (e.g. an orphan indicator). Returns False when closed or
    archived.
    """
    if logframe is None:
        return True
    return logframe.is_editable


def assert_logframe_editable(logframe: LogFrame | None, *, action: str = "modify") -> None:
    """Raise :class:`LogFrameClosedError` if ``logframe`` is closed.

    Use at the entry point of any view/service that mutates state tied to a
    log frame (indicator creation, data entry, target updates, etc.).
    """
    if not is_logframe_editable(logframe):
        raise LogFrameClosedError(
            f"This programme is closed — you cannot {action}. "
            "Reopen the Strategic Objective via the admin if you need to amend records."
        )


@transaction.atomic
def close_logframe(logframe: LogFrame, *, closed_by, reason: str = "") -> LogFrame:
    """Transition a log frame to CLOSED.

    Idempotent: closing an already-closed log frame is a no-op (returns the
    row unchanged). Stamps ``closed_at``, ``closed_by``, ``close_reason`` and
    flips ``active=False`` so list/dashboard queries that still filter on the
    legacy boolean degrade gracefully.

    Arms the post-project tracer beat: any :class:`TracerStudy` configured to
    dispatch after this log frame's closure will fire when
    ``closed_at + post_project_window_months <= now`` (see
    ``scan_post_project_tracers``).
    """
    from apps.mel.indicators.models import LogFrameStatus

    fresh = LogFrame.objects.select_for_update().get(pk=logframe.pk)
    if fresh.status == LogFrameStatus.CLOSED:
        return fresh
    fresh.status = LogFrameStatus.CLOSED
    fresh.closed_at = timezone.now()
    fresh.closed_by = closed_by
    fresh.close_reason = (reason or "").strip()
    fresh.active = False
    fresh.save(
        update_fields=[
            "status",
            "closed_at",
            "closed_by",
            "close_reason",
            "active",
            "updated_at",
        ]
    )
    logger.info(
        "mel.logframe closed slug=%s by=%s reason=%s",
        fresh.slug,
        getattr(closed_by, "pk", None),
        (reason or "")[:80],
    )
    return fresh


@transaction.atomic
def reopen_logframe(logframe: LogFrame, *, reopened_by) -> LogFrame:
    """Flip a CLOSED log frame back to ACTIVE.

    Clears closure stamps but keeps the audit trail (HistoricalRecords).
    Idempotent on already-active log frames.
    """
    from apps.mel.indicators.models import LogFrameStatus

    fresh = LogFrame.objects.select_for_update().get(pk=logframe.pk)
    if fresh.status == LogFrameStatus.ACTIVE:
        return fresh
    fresh.status = LogFrameStatus.ACTIVE
    fresh.closed_at = None
    fresh.closed_by = None
    fresh.close_reason = ""
    fresh.active = True
    fresh.save(
        update_fields=[
            "status",
            "closed_at",
            "closed_by",
            "close_reason",
            "active",
            "updated_at",
        ]
    )
    logger.info(
        "mel.logframe reopened slug=%s by=%s",
        fresh.slug,
        getattr(reopened_by, "pk", None),
    )
    return fresh


# ---------------------------------------------------------------------------
# M&E SRS Table 62 — Results Framework approval workflow (draft→submit→approve)
# ---------------------------------------------------------------------------


def submission_blockers(logframe: LogFrame) -> list[str]:
    """Return reasons a Results Framework is not yet ready to submit.

    M&E framework: "Only complete Results Frameworks may be submitted for
    approval." A complete Strategic Objective has, at minimum, the full causal
    chain Impact → Intermediate Outcome → Output and passes hierarchy validation
    (no orphan Outcomes/Outputs).
    """
    from apps.mel.indicators.models import LogFrameLevel

    issues = list(validate_hierarchy(logframe))
    levels = set(logframe.rows.values_list("level", flat=True))
    if LogFrameLevel.IMPACT not in levels:
        issues.append("Add at least one Impact (Long-Term Outcome).")
    if LogFrameLevel.OUTCOME not in levels:
        issues.append("Add at least one Intermediate Outcome.")
    if LogFrameLevel.OUTPUT not in levels:
        issues.append("Add at least one Output (University or Secretariat).")
    return issues


def indicator_publish_blockers(indicator: Indicator) -> list[str]:
    """Return reasons a DRAFT indicator is not ready to publish.

    M&E SRS Table 64 — "The system shall prevent submission if mandatory
    information is missing or invalid": an indicator needs a definition, at
    least one disaggregation category, at least one value per category, and
    complete performance information (baseline, target, data source,
    collection method, frequency, responsible) on every value.
    """
    blockers: list[str] = []
    if not (indicator.definition or "").strip():
        blockers.append("Add a definition describing what this indicator measures.")
    categories = list(
        indicator.disaggregation_categories.prefetch_related("values__responsible")
    )
    if not categories:
        blockers.append("Define at least one disaggregation category.")
    for category in categories:
        values = list(category.values.all())
        if not values:
            blockers.append(
                f"Category “{category.name}” needs at least one disaggregation value."
            )
            continue
        incomplete = [v.name for v in values if not v.is_complete]
        if incomplete:
            blockers.append(
                f"Category “{category.name}” — complete the baseline, target, "
                f"data source, collection method, frequency and responsible "
                f"officer for: {', '.join(incomplete)}."
            )
    return blockers


class ResultsFrameworkIncomplete(Exception):
    """Raised when a Results Framework is submitted before it is complete."""

    def __init__(self, blockers: list[str]):
        self.blockers = blockers
        super().__init__("; ".join(blockers))


@transaction.atomic
def submit_logframe_for_approval(logframe: LogFrame, *, submitted_by) -> LogFrame:
    """Move a DRAFT Results Framework to SUBMITTED once it is complete.

    Idempotent on already-submitted frameworks. Raises
    :class:`ResultsFrameworkIncomplete` when the completeness gate fails.
    """
    from apps.mel.indicators.models import LogFrameApproval

    fresh = LogFrame.objects.select_for_update().get(pk=logframe.pk)
    if fresh.approval_status == LogFrameApproval.SUBMITTED:
        return fresh
    blockers = submission_blockers(fresh)
    if blockers:
        raise ResultsFrameworkIncomplete(blockers)
    fresh.approval_status = LogFrameApproval.SUBMITTED
    fresh.submitted_at = timezone.now()
    fresh.submitted_by = submitted_by
    fresh.approved_at = None
    fresh.approved_by = None
    fresh.save(
        update_fields=[
            "approval_status",
            "submitted_at",
            "submitted_by",
            "approved_at",
            "approved_by",
            "updated_at",
        ]
    )
    logger.info(
        "mel.logframe submitted slug=%s by=%s",
        fresh.slug,
        getattr(submitted_by, "pk", None),
    )
    return fresh


@transaction.atomic
def approve_logframe(logframe: LogFrame, *, approved_by) -> LogFrame:
    """Approve a SUBMITTED Results Framework. Idempotent on APPROVED frames."""
    from apps.mel.indicators.models import LogFrameApproval

    fresh = LogFrame.objects.select_for_update().get(pk=logframe.pk)
    if fresh.approval_status == LogFrameApproval.APPROVED:
        return fresh
    fresh.approval_status = LogFrameApproval.APPROVED
    fresh.approved_at = timezone.now()
    fresh.approved_by = approved_by
    fresh.save(
        update_fields=["approval_status", "approved_at", "approved_by", "updated_at"]
    )
    logger.info(
        "mel.logframe approved slug=%s by=%s",
        fresh.slug,
        getattr(approved_by, "pk", None),
    )
    return fresh


@transaction.atomic
def return_logframe_to_draft(logframe: LogFrame, *, actor) -> LogFrame:
    """Bounce a SUBMITTED/APPROVED framework back to DRAFT for revision."""
    from apps.mel.indicators.models import LogFrameApproval

    fresh = LogFrame.objects.select_for_update().get(pk=logframe.pk)
    if fresh.approval_status == LogFrameApproval.DRAFT:
        return fresh
    fresh.approval_status = LogFrameApproval.DRAFT
    fresh.submitted_at = None
    fresh.submitted_by = None
    fresh.approved_at = None
    fresh.approved_by = None
    fresh.save(
        update_fields=[
            "approval_status",
            "submitted_at",
            "submitted_by",
            "approved_at",
            "approved_by",
            "updated_at",
        ]
    )
    logger.info(
        "mel.logframe returned-to-draft slug=%s by=%s",
        fresh.slug,
        getattr(actor, "pk", None),
    )
    return fresh


# ---------------------------------------------------------------------------
# G4 — Programme-level health score
# ---------------------------------------------------------------------------


_HEALTH_DEFAULT_WEIGHTS = {
    "rag_health": 0.4,
    "activity_on_time": 0.3,
    "budget_adherence": 0.2,
    "feedback_satisfaction": 0.1,
}


def _redistribute_weights(parts: dict[str, float | None]) -> dict[str, float]:
    """Take a {term: value-or-None} map and return {term: effective_weight}.

    Missing terms (value=None) are dropped and their default weight is shared
    proportionally across the remaining terms. Returns ``{}`` when every term
    is missing — callers should treat that as "no score".
    """
    available = {k: v for k, v in parts.items() if v is not None}
    if not available:
        return {}
    base_sum = sum(_HEALTH_DEFAULT_WEIGHTS[k] for k in available)
    if base_sum == 0:
        return {k: 0.0 for k in available}
    return {k: _HEALTH_DEFAULT_WEIGHTS[k] / base_sum for k in available}


def _rag_health_term(logframe: LogFrame, period_label: str) -> tuple[float | None, int]:
    """Compute the RAG-health term: (green + 0.5*amber) / total × 100.

    Returns ``(score, total_indicator_count)``. Returns ``(None, 0)`` when no
    indicator has been measured for the period.
    """
    indicators = list(
        Indicator.objects.filter(logframe_row__logframe=logframe, is_active=True)
        .select_related("logframe_row__logframe")
    )
    if not indicators:
        return None, 0
    counted = 0
    score = 0.0
    for ind in indicators:
        progress = calculate_progress(ind, period_label)
        rag = progress.get("rag")
        if rag == RagStatus.UNKNOWN:
            continue
        counted += 1
        if rag == RagStatus.GREEN:
            score += 1.0
        elif rag == RagStatus.AMBER:
            score += 0.5
        # RED contributes 0.
    if counted == 0:
        return None, len(indicators)
    return (score / counted) * 100.0, counted


def _activity_on_time_term(
    logframe: LogFrame, period_start, period_end,
) -> tuple[float | None, int]:
    """Compute the activity-on-time term over activities scheduled in the window.

    Counts activities under any row of ``logframe`` whose ``scheduled_end``
    falls within ``[period_start, period_end]`` (inclusive). An activity is
    "on time" when status=COMPLETED and (actual_end is null OR actual_end <=
    scheduled_end). Returns ``(None, 0)`` when no activity matches.
    """
    if period_start is None or period_end is None:
        return None, 0
    try:
        from apps.mel.tracking.models import Activity, ActivityStatus
    except ImportError:  # pragma: no cover - tracking app should always be present
        return None, 0

    qs = Activity.objects.filter(
        logframe_row__logframe=logframe,
        scheduled_end__gte=period_start,
        scheduled_end__lte=period_end,
    )
    total = qs.count()
    if total == 0:
        return None, 0
    on_time = 0
    for act in qs.only("status", "scheduled_end", "actual_end"):
        if act.status != ActivityStatus.COMPLETED:
            continue
        if act.actual_end is None or act.actual_end <= act.scheduled_end:
            on_time += 1
    return (on_time / total) * 100.0, total


def _budget_adherence_term(logframe: LogFrame) -> float | None:
    """Compute |spent_ratio − elapsed_ratio| → adherence (100 − gap).

    Reuses the existing :func:`apps.mel.reports.builders.build_budget_variance`
    output to avoid re-implementing pacing maths. Returns ``None`` when the
    log frame has no finance binding or when pacing cannot be computed.
    """
    if not logframe.finance_budget_id:
        return None
    try:
        from apps.mel.reports.builders import build_budget_variance

        summary = build_budget_variance(logframe=logframe)
    except Exception:
        return None
    if not isinstance(summary, dict) or not summary.get("variance"):
        return None
    var = summary["variance"]
    try:
        spent_ratio = float(var.get("spent_ratio", 0.0))
        elapsed_ratio = float(var.get("elapsed_ratio", 0.0))
    except (TypeError, ValueError):
        return None
    gap_pct = abs(spent_ratio - elapsed_ratio) * 100.0
    return max(0.0, 100.0 - min(100.0, gap_pct))


def _feedback_satisfaction_term(
    logframe: LogFrame, period_start, period_end,
) -> tuple[float | None, int]:
    """Average FeedbackSubmission.rating across the period, scaled 0–100.

    Filters submissions whose ``submitted_at`` falls in ``[period_start,
    period_end]``. Without a date window we cannot scope to the log frame
    period, so we return ``(None, 0)`` in that case.

    NB: the current FeedbackSubmission schema is global rather than per-log-
    frame. We treat *all* feedback in the window as a programme-wide signal
    — refining to per-log-frame requires a future scope link from feedback
    channels to log frames.
    """
    if period_start is None or period_end is None:
        return None, 0
    try:
        from apps.mel.feedback.models import FeedbackSubmission
    except ImportError:  # pragma: no cover
        return None, 0

    qs = FeedbackSubmission.objects.filter(
        rating__isnull=False,
        created_at__date__gte=period_start,
        created_at__date__lte=period_end,
    ).values_list("rating", flat=True)
    ratings = [int(r) for r in qs if r is not None]
    if not ratings:
        return None, 0
    mean = sum(ratings) / len(ratings)
    # 1–5 Likert → 0–100 (rating 1 = 0, rating 5 = 100).
    return max(0.0, min(100.0, (mean - 1.0) / 4.0 * 100.0)), len(ratings)


@transaction.atomic
def compute_programme_health(
    logframe: LogFrame,
    period_label: str,
    *,
    period_start=None,
    period_end=None,
) -> ProgrammeHealthScore:
    """Idempotent upsert of the per-period programme health score.

    Falls back to ``logframe.period_start`` / ``logframe.period_end`` when the
    caller doesn't pass an explicit window — the caller can override for
    sub-period rollups (e.g. quarterly within an annual log frame).
    """
    period_start = period_start or logframe.period_start
    period_end = period_end or logframe.period_end

    rag_score, ind_count = _rag_health_term(logframe, period_label)
    on_time, act_count = _activity_on_time_term(logframe, period_start, period_end)
    adherence = _budget_adherence_term(logframe)
    sat, fb_count = _feedback_satisfaction_term(logframe, period_start, period_end)

    parts = {
        "rag_health": rag_score,
        "activity_on_time": on_time,
        "budget_adherence": adherence,
        "feedback_satisfaction": sat,
    }
    weights = _redistribute_weights(parts)
    if not weights:
        score = Decimal("0.00")
    else:
        composite = sum(parts[k] * weights[k] for k in weights)
        score = Decimal(str(composite)).quantize(Decimal("0.01"))

    def _q(value: float | None) -> Decimal | None:
        if value is None:
            return None
        return Decimal(str(value)).quantize(Decimal("0.01"))

    row, _ = ProgrammeHealthScore.objects.update_or_create(
        logframe=logframe,
        period_label=period_label,
        defaults={
            "period_start": period_start,
            "period_end": period_end,
            "rag_health": _q(rag_score),
            "activity_on_time": _q(on_time),
            "budget_adherence": _q(adherence),
            "feedback_satisfaction": _q(sat),
            "score": score,
            "weights_applied": weights,
            "indicator_count": ind_count,
            "activity_count": act_count,
            "feedback_count": fb_count,
        },
    )
    logger.info(
        "mel.health computed slug=%s period=%s score=%s weights=%s",
        logframe.slug, period_label, score, weights,
    )
    return row


def recompute_health_for_all_active() -> int:
    """Convenience wrapper for the nightly Celery beat (G4).

    Walks every ACTIVE log frame and computes the score for the current
    period derived from the log frame's most-recent indicator frequency
    (falls back to monthly when no indicator exists yet).
    """
    from apps.mel.indicators.models import IndicatorFrequency, LogFrameStatus

    count = 0
    for lf in LogFrame.objects.filter(status=LogFrameStatus.ACTIVE):
        # Derive the current period label from the first active indicator's
        # frequency. If no indicator yet, default to monthly so the row still
        # exists and the dashboard hero card has something to render.
        ind = (
            Indicator.objects.filter(logframe_row__logframe=lf, is_active=True)
            .only("frequency")
            .first()
        )
        freq = ind.frequency if ind else IndicatorFrequency.MONTHLY
        period_label = current_period_label(freq)
        try:
            compute_programme_health(lf, period_label)
            count += 1
        except Exception:
            logger.exception("mel.health recompute failed slug=%s", lf.slug)
    return count