1"""Non-discriminating evaluator detection from run history."""

2

3 from __future__ import annotations

4

5 from dataclasses import dataclass, field

6 from pathlib import Path

7 from typing import Any

8

9

10@dataclass

11 class EvaluatorVariance:

12 """Variance analysis for a single evaluator state.

13

14 Attributes:

15 state: State name.

16 evaluator_type: Evaluator type (e.g. llm_structured, exit_code).

17 pass_count: Number of positive verdicts.

18 total: Total number of evaluate events for this state.

19 pass_rate: pass_count / total (0.0 if total == 0).

20 variance: Bernoulli variance p*(1-p).

21 recommendation: Human-readable recommendation if variance is low, else None.

22 ci: Wilson 95% CI (lower, upper) for pass_rate, or None if not computed.

23 """

24

25 state: str

26 evaluator_type: str

27 pass_count: int

28 total: int

29 pass_rate: float

30 variance: float

31 recommendation: str | None = None

32 ci: tuple[float, float] | None = None

33

34 def to_dict(self) -> dict[str, Any]:

35 """Convert to dictionary for JSON serialization."""

36 result: dict[str, Any] = {

37 "state": self.state,

38 "evaluator_type": self.evaluator_type,

39 "pass_count": self.pass_count,

40 "total": self.total,

41 "pass_rate": round(self.pass_rate, 4),

42 "variance": round(self.variance, 4),

43 }

44 if self.ci is not None:

45 result["ci_lower"] = round(self.ci[0], 4)

46 result["ci_upper"] = round(self.ci[1], 4)

47 if self.recommendation:

48 result["recommendation"] = self.recommendation

49 return result

50

51

52@dataclass

53 class VarianceReport:

54 """Full variance report for a loop.

55

56 Attributes:

57 loop: Loop name.

58 total_runs: Number of runs analyzed.

59 states: Per-state variance results, sorted by variance ascending.

60 """

61

62 loop: str

63 total_runs: int

64 states: list[EvaluatorVariance] = field(default_factory=list)

65

66 def to_dict(self) -> dict[str, Any]:

67 """Convert to dictionary for JSON serialization."""

68 return {

69 "loop": self.loop,

70 "total_runs": self.total_runs,

71 "states": [s.to_dict() for s in self.states],

72 }

73

74

75 def _correlate_verdicts(

76 events: list[dict[str, Any]],

77) -> dict[str, list[bool]]:

78 """Correlate evaluate events with state_enter events to get per-state verdicts.

79

80 Walks events chronologically, tracking current state from state_enter

81 events, then pairs each evaluate event with the tracked state.

82

83 Args:

84 events: List of event dicts from events.jsonl.

85

86 Returns:

87 Dict mapping state name to list of bool verdicts (True = positive).

88 """

89 from little_loops.fsm.persistence import _verdict_is_yes

90

91 state_verdicts: dict[str, list[bool]] = {}

92 current_state: str | None = None

93

94 for event in events:

95 if event.get("event") == "state_enter":

96 current_state = event.get("state")

97 elif event.get("event") == "evaluate" and current_state:

98 verdict_str = event.get("verdict", "")

99 state_verdicts.setdefault(current_state, []).append(_verdict_is_yes(verdict_str))

100

101 return state_verdicts

102

103

104 def _generate_recommendation(

105 state: str,

106 evaluator_type: str,

107 pass_rate: float,

108 variance: float,

109 prompt: str | None = None,

110 target: Any = None,

111) -> str | None:

112 """Generate a recommendation for a low-variance evaluator.

113

114 Pattern-matches common failure modes:

115 - High pass-rate + llm_structured → "broaden judge criteria"

116 - 100% pass + output_numeric → "target may be too loose"

117 - 100% pass + exit_code → "command may not exercise the feature"

118

119 Args:

120 state: State name.

121 evaluator_type: Evaluator type string.

122 pass_rate: Observed pass rate.

123 variance: Bernoulli variance.

124 prompt: Evaluator prompt (for llm_structured).

125 target: Target value (for output_numeric).

126

127 Returns:

128 Recommendation string or None if evaluator appears discriminating.

129 """

130 if variance > 0.05:

131 return None

132

133 if pass_rate >= 0.95 and evaluator_type == "llm_structured":

134 prompt_preview = ""

135 if prompt:

136 truncated = prompt[:100] + "..." if len(prompt) > 100 else prompt

137 prompt_preview = f'\n ↳ judge prompt: "{truncated}"'

138 return (

139 f"Judge prompt may be too broad — most inputs pass trivially.{prompt_preview}\n"

140 f" Recommendation: tighten to require specific evidence "

141 f"(e.g., confidence_score increase, new codebase references added)."

142 )

143

144 if pass_rate >= 0.99 and evaluator_type == "output_numeric":

145 target_str = f" (target={target})" if target is not None else ""

146 return (

147 f"Target may be too loose for actual output values{target_str}.\n"

148 f" Recommendation: lower the target or inspect typical run outputs "

149 f"to find a more discriminating threshold."

150 )

151

152 if pass_rate >= 0.99 and evaluator_type == "exit_code":

153 return (

154 "Command may not exercise the feature — exits 0 regardless of intent.\n"

155 " Recommendation: replace with a command that fails on meaningful "

156 "conditions (e.g., grep for expected output, diff against baseline)."

157 )

158

159 return None

160

161

162 def compute_evaluator_variance(

163 loop_name: str,

164 loops_dir: Path,

165 threshold: float = 0.05,

166 min_runs: int = 10,

167) -> VarianceReport | None:

168 """Compute per-state evaluator variance from run history.

169

170 Walks .loops/.history/*-{loop_name}/events.jsonl, correlates evaluate

171 events with state_enter events, computes Bernoulli variance p*(1-p)

172 per state, and generates recommendations for low-variance evaluators.

173

174 Args:

175 loop_name: Name of the loop to analyze.

176 loops_dir: Base directory containing .loops/.

177 threshold: Variance floor below which a state is flagged (default 0.05).

178 min_runs: Minimum runs required to compute meaningful variance (default 10).

179

180 Returns:

181 VarianceReport if history exists and min_runs is met, None otherwise.

182 """

183 from little_loops.fsm.persistence import HISTORY_DIR

184

185 history_root = loops_dir / HISTORY_DIR

186 if not history_root.exists():

187 return None

188

189 suffix = f"-{loop_name}"

190 all_verdicts: dict[str, list[bool]] = {}

191 run_count = 0

192

193 for run_dir in sorted(history_root.iterdir(), key=lambda d: d.name):

194 if not run_dir.is_dir() or not run_dir.name.endswith(suffix):

195 continue

196 events_file = run_dir / "events.jsonl"

197 if not events_file.exists():

198 continue

199 import json as _json

200

201 events: list[dict[str, Any]] = []

202 try:

203 for line in events_file.read_text(encoding="utf-8").splitlines():

204 line = line.strip()

205 if line:

206 try:

207 events.append(_json.loads(line))

208 except _json.JSONDecodeError:

209 pass

210 except OSError:

211 continue

212

213 run_verdicts = _correlate_verdicts(events)

214 for state, verdicts in run_verdicts.items():

215 all_verdicts.setdefault(state, []).extend(verdicts)

216 run_count += 1

217

218 if run_count < min_runs:

219 return None

220

221 # Load loop YAML to get evaluator configs

222 evaluator_configs: dict[str, dict[str, Any]] = {}

223 try:

224 from little_loops.cli.loop._helpers import load_loop

225 from little_loops.logger import Logger

226

227 fsm = load_loop(loop_name, loops_dir, Logger(verbose=False))

228 for name, state_cfg in fsm.states.items():

229 if state_cfg.evaluate is not None:

230 evaluator_configs[name] = {

231 "type": state_cfg.evaluate.type,

232 "prompt": state_cfg.evaluate.prompt,

233 "target": state_cfg.evaluate.target,

234 }

235 except (FileNotFoundError, ValueError):

236 pass

237

238 from little_loops.stats import wilson_ci

239

240 states: list[EvaluatorVariance] = []

241 for state_name in sorted(all_verdicts.keys()):

242 verdicts = all_verdicts[state_name]

243 total = len(verdicts)

244 if total == 0:

245 continue

246 pass_count = sum(1 for v in verdicts if v)

247 pass_rate = pass_count / total

248 variance = pass_rate * (1 - pass_rate)

249 ci = wilson_ci(pass_count, total)

250

251 config = evaluator_configs.get(state_name, {})

252 eval_type = config.get("type", "unknown")

253 recommendation = _generate_recommendation(

254 state_name,

255 eval_type,

256 pass_rate,

257 variance,

258 prompt=config.get("prompt"),

259 target=config.get("target"),

260 )

261

262 states.append(

263 EvaluatorVariance(

264 state=state_name,

265 evaluator_type=eval_type,

266 pass_count=pass_count,

267 total=total,

268 pass_rate=pass_rate,

269 variance=variance,

270 recommendation=recommendation,

271 ci=ci,

272 )

273 )

274

275 # Sort by variance ascending (lowest variance first — most suspicious)

276 states.sort(key=lambda s: s.variance)

277

278 return VarianceReport(

279 loop=loop_name,

280 total_runs=run_count,

281 states=states,

282 )