« prev ^ index » next coverage.py v7.15.4, created at 2026-08-25 07:19 +0800
1"""Fixed-size chunking strategy."""
3from __future__ import annotations
5from typing import Any
7from lexigram.ai.rag.chunking.base import AbstractChunker
8from lexigram.ai.rag.chunking.types import Chunk
11class FixedSizeChunker(AbstractChunker):
12 """Fixed-size chunking with optional overlap.
14 Splits text into chunks of approximately equal size with configurable overlap.
16 Example:
17 >>> chunker = FixedSizeChunker(chunk_size=500, overlap=50)
18 >>> chunks = chunker.chunk("Long text...")
19 """
21 def __init__(
22 self,
23 chunk_size: int = 1000,
24 overlap: int = 200,
25 separator: str = " ",
26 keep_separator: bool = True,
27 ):
28 """Initialize fixed-size chunker.
30 Args:
31 chunk_size: Target size of each chunk in characters
32 overlap: Number of overlapping characters between chunks
33 separator: Character/string to split on (default: space)
34 keep_separator: Whether to keep separator in chunks
35 """
36 self.chunk_size = chunk_size
37 self.overlap = overlap
38 self.separator = separator
39 self.keep_separator = keep_separator
41 if overlap >= chunk_size:
42 msg = "overlap must be less than chunk_size"
43 raise ValueError(msg)
45 def chunk(self, text: str, metadata: dict[str, Any] | None = None) -> list[Chunk]:
46 """Split text into fixed-size chunks.
48 Args:
49 text: Text to chunk
50 metadata: Optional metadata
52 Returns:
53 List of chunks
54 """
55 if not text:
56 return []
58 chunks: list[Chunk] = []
59 chunk_index = 0
60 step = self.chunk_size - self.overlap
62 if step <= 0:
63 step = 1 # Prevent infinite loop
65 position = 0
66 while position < len(text):
67 end = min(position + self.chunk_size, len(text))
68 chunk_text = text[position:end]
70 if chunk_text.strip():
71 chunks.append(
72 Chunk(
73 text=chunk_text,
74 start_index=position,
75 end_index=end,
76 chunk_index=chunk_index,
77 metadata=metadata,
78 ),
79 )
80 chunk_index += 1
82 position += step
84 return chunks