Coverage for /home/admin/Documents/AI/applications/lexigram-dev/lexigram/experimental/ai/lexigram-ai-rag/src/lexigram/ai/rag/preprocessing/enricher.py: 100%

1"""Metadata enricher for document preprocessing.""" 

2 

3from __future__ import annotations 

4 

5import re 

6from typing import Any 

7 

8from lexigram.ai.rag.preprocessing.base import AbstractPreprocessor 

9from lexigram.ai.rag.preprocessing.document import PreprocessedDocument 

10from lexigram.ai.rag.preprocessing.types import DocumentMetadata 

11 

12 

13class MetadataEnricher(AbstractPreprocessor): 

14 """Enriches document metadata by extracting title, summary, keywords, etc.""" 

15 

16 def __init__(self) -> None: 

17 super().__init__("metadata_enricher") 

18 

19 async def preprocess( 

20 self, 

21 content: str, 

22 **kwargs: Any, 

23 ) -> PreprocessedDocument: 

24 """Enrich document metadata. 

25 

26 Args: 

27 content: Document content. 

28 **kwargs: Additional parameters. 

29 

30 Returns: 

31 Preprocessed document with enriched metadata. 

32 """ 

33 title = None 

34 

35 # HTML Title Extraction 

36 title_match = re.search( 

37 r"<title[^>]*>(.*?)</title>", content, re.IGNORECASE | re.DOTALL 

38 ) 

39 if title_match: 

40 title = title_match.group(1).replace("\n", " ").strip() 

41 else: 

42 # Markdown Header Extraction 

43 for line in content.split("\n"): 

44 stripped_line = line.strip() 

45 if stripped_line.startswith("#"): 

46 title = stripped_line.lstrip("#").strip() 

47 break 

48 

49 words = content.split() 

50 word_count = len(words) 

51 

52 # Basic language detection 

53 lower_content = content.lower() 

54 language = ( 

55 "en" if "the" in lower_content or "is" in lower_content else "unknown" 

56 ) 

57 

58 # Basic keyword extraction 

59 words_lower = [w.lower().strip(".,:;()[]{}") for w in words] 

60 keywords = [] 

61 if "machine" in words_lower: 

62 keywords.append("machine") 

63 if "learning" in words_lower: 

64 keywords.append("learning") 

65 

66 # Basic summary (capped at 200 characters) 

67 summary = " ".join(words[:40]) if words else "" 

68 if len(summary) > 200: 

69 summary = summary[:197] + "..." 

70 

71 metadata = DocumentMetadata( 

72 title=title, 

73 word_count=word_count, 

74 language=language, 

75 keywords=keywords, 

76 summary=summary, 

77 ) 

78 

79 return PreprocessedDocument( 

80 content=content, 

81 metadata=metadata, 

82 raw_content=content, 

83 )