Coverage for /home/admin/Documents/AI/applications/lexigram-dev/lexigram/experimental/ai/lexigram-ai-evaluation/src/lexigram/ai/evaluation/config.py: 100%

1"""Evaluation configuration for the Lexigram framework.""" 

2 

3from __future__ import annotations 

4 

5from typing import ClassVar, cast 

6 

7from lexigram.ai.evaluation import constants as const 

8from lexigram.config.base import BaseConfig 

9from lexigram.validation import ConfigDict, Field 

10 

11ENV_PREFIX: str = const.ENV_PREFIX 

12ENV_NESTED_DELIMITER: str = const.ENV_NESTED_DELIMITER 

13 

14 

15class EvaluationConfig(BaseConfig): 

16 """Configuration for the evaluation subsystem. 

17 

18 Attributes: 

19 enabled: Enable the AI evaluation subsystem. 

20 default_threshold: Default score threshold for passing evaluations. 

21 embedding_model: Model to use for embedding-based evaluations. 

22 include_metadata: Whether to include metadata in run reports. 

23 max_samples: Maximum number of samples per evaluation run. 

24 max_retries: Maximum retries for failed evaluations. 

25 timeout_seconds: Timeout for evaluation execution in seconds. 

26 

27 Example:: 

28 

29 config = EvaluationConfig( 

30 default_threshold=0.9, 

31 embedding_model="text-embedding-3-large" 

32 ) 

33 """ 

34 

35 config_section: ClassVar[str] = "ai_evaluation" 

36 

37 model_config: ClassVar[ConfigDict] = cast( 

38 "ConfigDict", 

39 { 

40 "env_prefix": ENV_PREFIX, 

41 "env_nested_delimiter": ENV_NESTED_DELIMITER, 

42 "extra": "ignore", 

43 }, 

44 ) 

45 

46 enabled: bool = Field( 

47 default=True, description="Enable the AI evaluation subsystem" 

48 ) 

49 

50 experiment_dir: str | None = Field( 

51 default=None, 

52 description="Base directory for experiment tracking and checkpoint artifacts", 

53 ) 

54 

55 default_seed: int | None = Field( 

56 default=None, 

57 ge=0, 

58 description="Default seed for reproducible experiment runs", 

59 ) 

60 

61 default_threshold: float = Field( 

62 default=0.8, 

63 ge=0.0, 

64 le=1.0, 

65 description="Default score threshold for passing evaluations", 

66 ) 

67 

68 embedding_model: str = Field( 

69 default="text-embedding-3-small", 

70 description="Model to use for embedding-based evaluations", 

71 ) 

72 

73 include_metadata: bool = Field( 

74 default=True, 

75 description="Whether to include metadata in run reports", 

76 ) 

77 

78 max_samples: int | None = Field( 

79 default=None, 

80 ge=1, 

81 description="Maximum number of samples per evaluation run", 

82 ) 

83 

84 max_retries: int = Field( 

85 default=3, 

86 ge=0, 

87 description="Maximum retries for failed evaluations", 

88 ) 

89 

90 timeout_seconds: int = Field( 

91 default=30, 

92 ge=1, 

93 description="Timeout for evaluation execution in seconds", 

94 ) 

95 

96 

97__all__ = ["ENV_NESTED_DELIMITER", "ENV_PREFIX", "EvaluationConfig"]