Coverage for /home/admin/Documents/AI/applications/lexigram-dev/lexigram/experimental/ai/lexigram-ai-workers/src/lexigram/ai/workers/__init__.py: 40%

1""" 

2Background workers for lexigram-intelligence. 

3 

4This module provides background processing capabilities for: 

5- Document ingestion and processing 

6- Batch embedding generation 

7- Scheduled maintenance tasks 

8- Failed task recovery (Dead Letter Queue) 

9""" 

10 

11from __future__ import annotations 

12 

13from typing import TYPE_CHECKING 

14 

15if TYPE_CHECKING: 

16 from lexigram.ai.workers.batch_embedding import ( 

17 BatchEmbeddingJob, 

18 BatchEmbeddingProgress, 

19 BatchEmbeddingResult, 

20 BatchEmbeddingWorker, 

21 ) 

22 from lexigram.ai.workers.config import WorkersConfig 

23 from lexigram.ai.workers.constants import ( 

24 DEFAULT_BASE_BACKOFF, 

25 DEFAULT_CHECK_INTERVAL, 

26 DEFAULT_MAX_RETRIES, 

27 DEFAULT_TASK_TIMEOUT, 

28 MAX_BACKOFF_SECONDS, 

29 MAX_HISTORY_SIZE, 

30 ) 

31 from lexigram.ai.workers.di.provider import WorkersProvider 

32 from lexigram.ai.workers.dlq.worker import DeadLetterQueueWorker, ErrorClassifier 

33 from lexigram.ai.workers.document_ingestion import ( 

34 DocumentIngestionJob, 

35 DocumentIngestionWorker, 

36 IngestionProgress, 

37 IngestionResult, 

38 ) 

39 from lexigram.ai.workers.exceptions import DLQError, MaintenanceError, WorkerError 

40 from lexigram.ai.workers.hooks import ( 

41 WorkerJobCompletedHook, 

42 WorkerJobStartedHook, 

43 WorkerMaintenanceRunHook, 

44 ) 

45 from lexigram.ai.workers.maintenance.worker import MaintenanceWorker 

46 from lexigram.ai.workers.module import WorkersModule 

47 from lexigram.ai.workers.types import ( 

48 DLQAction, 

49 DLQItem, 

50 DLQStats, 

51 FailureCategory, 

52 MaintenanceResult, 

53 MaintenanceStatus, 

54 MaintenanceTask, 

55 MaintenanceTaskType, 

56 ) 

57 

58_LAZY_IMPORTS: dict[str, tuple[str, str]] = { 

59 "BatchEmbeddingJob": ("lexigram.ai.workers.batch_embedding", "BatchEmbeddingJob"), 

60 "BatchEmbeddingProgress": ( 

61 "lexigram.ai.workers.batch_embedding", 

62 "BatchEmbeddingProgress", 

63 ), 

64 "BatchEmbeddingResult": ( 

65 "lexigram.ai.workers.batch_embedding", 

66 "BatchEmbeddingResult", 

67 ), 

68 "BatchEmbeddingWorker": ( 

69 "lexigram.ai.workers.batch_embedding", 

70 "BatchEmbeddingWorker", 

71 ), 

72 "DEFAULT_BASE_BACKOFF": ("lexigram.ai.workers.constants", "DEFAULT_BASE_BACKOFF"), 

73 "DEFAULT_CHECK_INTERVAL": ( 

74 "lexigram.ai.workers.constants", 

75 "DEFAULT_CHECK_INTERVAL", 

76 ), 

77 "DEFAULT_MAX_RETRIES": ("lexigram.ai.workers.constants", "DEFAULT_MAX_RETRIES"), 

78 "DEFAULT_TASK_TIMEOUT": ("lexigram.ai.workers.constants", "DEFAULT_TASK_TIMEOUT"), 

79 "DLQAction": ("lexigram.ai.workers.types", "DLQAction"), 

80 "DLQError": ("lexigram.ai.workers.exceptions", "DLQError"), 

81 "DLQItem": ("lexigram.ai.workers.types", "DLQItem"), 

82 "DLQStats": ("lexigram.ai.workers.types", "DLQStats"), 

83 "DeadLetterQueueWorker": ( 

84 "lexigram.ai.workers.dlq.worker", 

85 "DeadLetterQueueWorker", 

86 ), 

87 "DocumentIngestionJob": ( 

88 "lexigram.ai.workers.document_ingestion", 

89 "DocumentIngestionJob", 

90 ), 

91 "DocumentIngestionWorker": ( 

92 "lexigram.ai.workers.document_ingestion", 

93 "DocumentIngestionWorker", 

94 ), 

95 "ErrorClassifier": ("lexigram.ai.workers.dlq.worker", "ErrorClassifier"), 

96 "FailureCategory": ("lexigram.ai.workers.types", "FailureCategory"), 

97 "IngestionProgress": ( 

98 "lexigram.ai.workers.document_ingestion", 

99 "IngestionProgress", 

100 ), 

101 "IngestionResult": ("lexigram.ai.workers.document_ingestion", "IngestionResult"), 

102 "MaintenanceError": ("lexigram.ai.workers.exceptions", "MaintenanceError"), 

103 "MaintenanceResult": ("lexigram.ai.workers.types", "MaintenanceResult"), 

104 "WorkerJobCompletedHook": ( 

105 "lexigram.ai.workers.hooks", 

106 "WorkerJobCompletedHook", 

107 ), 

108 "WorkerJobStartedHook": ( 

109 "lexigram.ai.workers.hooks", 

110 "WorkerJobStartedHook", 

111 ), 

112 "WorkerMaintenanceRunHook": ( 

113 "lexigram.ai.workers.hooks", 

114 "WorkerMaintenanceRunHook", 

115 ), 

116 "MaintenanceStatus": ("lexigram.ai.workers.types", "MaintenanceStatus"), 

117 "MaintenanceTask": ("lexigram.ai.workers.types", "MaintenanceTask"), 

118 "MaintenanceTaskType": ("lexigram.ai.workers.types", "MaintenanceTaskType"), 

119 "MaintenanceWorker": ( 

120 "lexigram.ai.workers.maintenance.worker", 

121 "MaintenanceWorker", 

122 ), 

123 "MAX_BACKOFF_SECONDS": ("lexigram.ai.workers.constants", "MAX_BACKOFF_SECONDS"), 

124 "MAX_HISTORY_SIZE": ("lexigram.ai.workers.constants", "MAX_HISTORY_SIZE"), 

125 "WorkerError": ("lexigram.ai.workers.exceptions", "WorkerError"), 

126 "WorkersConfig": ("lexigram.ai.workers.config", "WorkersConfig"), 

127 "WorkersModule": ("lexigram.ai.workers.module", "WorkersModule"), 

128 "WorkersProvider": ("lexigram.ai.workers.di.provider", "WorkersProvider"), 

129} 

130 

131__all__ = list(_LAZY_IMPORTS) 

132 

133 

134def __getattr__(name: str) -> object: 

135 """Lazy-load public symbols on first access.""" 

136 if name in _LAZY_IMPORTS: 

137 import importlib 

138 

139 module_path, attr = _LAZY_IMPORTS[name] 

140 mod = importlib.import_module(module_path) 

141 value = getattr(mod, attr) 

142 globals()[name] = value 

143 return value 

144 raise AttributeError(f"module {__name__!r} has no attribute {name!r}") 

145 

146 

147def __dir__() -> list[str]: 

148 """Expose lazy-loaded names for tab completion and dir().""" 

149 return list(_LAZY_IMPORTS)