Coverage for /home/admin/Documents/AI/applications/lexigram-dev/lexigram/experimental/ai/lexigram-ai-rag/src/lexigram/ai/rag/chunking/types.py: 97%

1"""Types and data models for document chunking.""" 

2 

3from __future__ import annotations 

4 

5from dataclasses import dataclass 

6from enum import StrEnum 

7from typing import Any 

8 

9from lexigram.contracts.ai.chunks import Chunk as ChunkBase 

10from lexigram.domain import DomainModel 

11from lexigram.validation import Field 

12 

13 

14@dataclass(init=False, frozen=True) 

15class Chunk(DomainModel, ChunkBase): 

16 """A chunk of text with metadata. 

17 

18 Attributes: 

19 text: The chunk text content 

20 source: Source document identifier 

21 start_index: Starting character position in original document 

22 end_index: Ending character position in original document 

23 chunk_index: Sequential index of this chunk 

24 metadata: Optional metadata dictionary 

25 """ 

26 

27 text: str = Field(description="Chunk text content") 

28 source: str = Field(default="unknown", description="Source document identifier") 

29 score: float | None = Field(default=None, description="Optional retrieval score") 

30 chunk_index: int = Field(description="Index of this chunk") 

31 start_index: int | None = Field( 

32 default=None, description="Starting character position" 

33 ) 

34 end_index: int | None = Field(default=None, description="Ending character position") 

35 embedding: list[float] | None = Field( 

36 default=None, description="Optional embedding associated with the chunk" 

37 ) 

38 metadata: dict[str, Any] = Field(default_factory=dict, description="Chunk metadata") 

39 

40 def __len__(self) -> int: 

41 """Get chunk length in characters.""" 

42 return len(self.text) 

43 

44 

45class ChunkingStrategy(StrEnum): 

46 """Chunking strategy types.""" 

47 

48 FIXED_SIZE = "fixed_size" # Fixed character/token count 

49 RECURSIVE = "recursive" # Recursive splitting by separators 

50 SEMANTIC = "semantic" # Sentence/paragraph boundaries 

51 SLIDING_WINDOW = "sliding_window" # Overlapping windows 

52 TOKEN = "token" # noqa: S105 # chunking strategy name, not a credential 

53 

54 

55@dataclass(init=False) 

56class ChunkingConfig(DomainModel): 

57 """Configuration for chunking. 

58 

59 Example: 

60 >>> config = ChunkingConfig( 

61 ... strategy=ChunkingStrategy.FIXED_SIZE, 

62 ... chunk_size=1000, 

63 ... overlap=200 

64 ... ) 

65 """ 

66 

67 strategy: ChunkingStrategy = Field( 

68 default=ChunkingStrategy.FIXED_SIZE, 

69 description="Chunking strategy to use", 

70 ) 

71 chunk_size: int = Field( 

72 default=1000, 

73 ge=1, 

74 description="Target chunk size in characters", 

75 ) 

76 overlap: int = Field( 

77 default=200, 

78 ge=0, 

79 description="Overlap between chunks (for applicable strategies)", 

80 ) 

81 min_chunk_size: int = Field( 

82 default=100, 

83 ge=1, 

84 description="Minimum chunk size (semantic strategy)", 

85 ) 

86 separators: list[str] | None = Field( 

87 default=None, 

88 description="Separators for recursive chunking", 

89 ) 

90 encoding_name: str = Field( 

91 default="cl100k_base", 

92 description="Tokenizer encoding for token chunking", 

93 )