LICENSE
MANIFEST.in
README.md
pyproject.toml
requirements.txt
setup.cfg
crawlo/__init__.py
crawlo/__version__.py
crawlo/application.py
crawlo/cli.py
crawlo/config.py
crawlo/crawler.py
crawlo/error_types.py
crawlo/event.py
crawlo/exceptions.py
crawlo/framework.py
crawlo/interfaces.py
crawlo/project.py
crawlo/subscriber.py
crawlo/task_manager.py
crawlo.egg-info/PKG-INFO
crawlo.egg-info/SOURCES.txt
crawlo.egg-info/dependency_links.txt
crawlo.egg-info/entry_points.txt
crawlo.egg-info/requires.txt
crawlo.egg-info/top_level.txt
crawlo/backpressure/__init__.py
crawlo/backpressure/intelligent_calculator.py
crawlo/backpressure/interfaces.py
crawlo/backpressure/metrics_collector.py
crawlo/backpressure/monitor.py
crawlo/backpressure/strategies.py
crawlo/bot/__init__.py
crawlo/bot/channels/__init__.py
crawlo/bot/channels/base.py
crawlo/bot/channels/dingtalk.py
crawlo/bot/channels/email.py
crawlo/bot/channels/feishu.py
crawlo/bot/channels/sms.py
crawlo/bot/channels/wecom.py
crawlo/bot/core/__init__.py
crawlo/bot/core/handlers.py
crawlo/bot/core/models.py
crawlo/bot/core/notifier.py
crawlo/bot/monitoring/__init__.py
crawlo/bot/monitoring/enums.py
crawlo/bot/monitoring/templates.py
crawlo/bot/templates/__init__.py
crawlo/bot/templates/enums.py
crawlo/bot/templates/manager.py
crawlo/bot/utils/__init__.py
crawlo/bot/utils/config_loader.py
crawlo/bot/utils/deduplicator.py
crawlo/checkpoint/__init__.py
crawlo/checkpoint/manager.py
crawlo/checkpoint/storage.py
crawlo/commands/__init__.py
crawlo/commands/check.py
crawlo/commands/genspider.py
crawlo/commands/help.py
crawlo/commands/list.py
crawlo/commands/run.py
crawlo/commands/schedule.py
crawlo/commands/shell.py
crawlo/commands/startproject.py
crawlo/commands/stats.py
crawlo/commands/utils.py
crawlo/core/__init__.py
crawlo/core/engine.py
crawlo/core/engine_helpers.py
crawlo/core/processor.py
crawlo/core/scheduler.py
crawlo/downloader/__init__.py
crawlo/downloader/aiohttp_downloader.py
crawlo/downloader/camoufox_downloader.py
crawlo/downloader/cffi_downloader.py
crawlo/downloader/constants.py
crawlo/downloader/drissionpage_downloader.py
crawlo/downloader/httpx_downloader.py
crawlo/downloader/hybrid_downloader.py
crawlo/downloader/playwright_downloader.py
crawlo/downloader/stealth_scripts/__init__.py
crawlo/downloader/stealth_scripts/canvas.py
crawlo/downloader/stealth_scripts/chrome_runtime.py
crawlo/downloader/stealth_scripts/drissionpage.py
crawlo/downloader/stealth_scripts/navigator.py
crawlo/downloader/stealth_scripts/webgl.py
crawlo/extension/__init__.py
crawlo/extension/health_check.py
crawlo/extension/log_interval.py
crawlo/extension/log_stats.py
crawlo/extension/logging_extension.py
crawlo/extension/memory_monitor.py
crawlo/extension/mysql_monitor.py
crawlo/extension/redis_monitor.py
crawlo/extension/request_recorder.py
crawlo/factories/__init__.py
crawlo/factories/base.py
crawlo/factories/crawler.py
crawlo/factories/registry.py
crawlo/factories/utils.py
crawlo/filters/__init__.py
crawlo/filters/aioredis_filter.py
crawlo/filters/memory_filter.py
crawlo/initialization/__init__.py
crawlo/initialization/built_in.py
crawlo/initialization/context.py
crawlo/initialization/core.py
crawlo/initialization/phases.py
crawlo/initialization/registry.py
crawlo/initialization/utils.py
crawlo/items/__init__.py
crawlo/items/base.py
crawlo/items/fields.py
crawlo/items/items.py
crawlo/logging/__init__.py
crawlo/logging/config.py
crawlo/logging/factory.py
crawlo/logging/manager.py
crawlo/mcp/__init__.py
crawlo/mcp/quick_fetcher.py
crawlo/mcp/server.py
crawlo/middleware/__init__.py
crawlo/middleware/cloudflare_bypass.py
crawlo/middleware/default_header.py
crawlo/middleware/download_delay.py
crawlo/middleware/dynamic_render_middleware.py
crawlo/middleware/file_middleware.py
crawlo/middleware/middleware_manager.py
crawlo/middleware/offsite.py
crawlo/middleware/proxy.py
crawlo/middleware/request_ignore.py
crawlo/middleware/response_code.py
crawlo/middleware/response_filter.py
crawlo/middleware/retry.py
crawlo/middleware/user_agents.py
crawlo/network/__init__.py
crawlo/network/request.py
crawlo/network/response.py
crawlo/pipelines/__init__.py
crawlo/pipelines/base_pipeline.py
crawlo/pipelines/bloom_dedup_pipeline.py
crawlo/pipelines/console_pipeline.py
crawlo/pipelines/csv_pipeline.py
crawlo/pipelines/database_dedup_pipeline.py
crawlo/pipelines/json_pipeline.py
crawlo/pipelines/memory_dedup_pipeline.py
crawlo/pipelines/mongo_pipeline.py
crawlo/pipelines/mysql_pipeline.py
crawlo/pipelines/pipeline_manager.py
crawlo/pipelines/redis_dedup_pipeline.py
crawlo/queue/__init__.py
crawlo/queue/disk_queue.py
crawlo/queue/interfaces.py
crawlo/queue/memory_queue.py
crawlo/queue/pqueue.py
crawlo/queue/queue_manager.py
crawlo/queue/queue_types.py
crawlo/queue/redis_priority_queue.py
crawlo/scheduling/__init__.py
crawlo/scheduling/daemon_scheduler.py
crawlo/scheduling/job.py
crawlo/scheduling/parameter_manager.py
crawlo/scheduling/registry.py
crawlo/scheduling/trigger.py
crawlo/scheduling/daemon/__init__.py
crawlo/scheduling/daemon/cleanup.py
crawlo/scheduling/daemon/executor.py
crawlo/scheduling/daemon/scheduler.py
crawlo/settings/__init__.py
crawlo/settings/default_settings.py
crawlo/settings/setting_manager.py
crawlo/shell/__init__.py
crawlo/shell/core.py
crawlo/spider/__init__.py
crawlo/stats/__init__.py
crawlo/stats/backends.py
crawlo/stats/collector.py
crawlo/templates/crawlo.cfg.tmpl
crawlo/templates/run.py.tmpl
crawlo/templates/spiders_init.py.tmpl
crawlo/templates/project/__init__.py.tmpl
crawlo/templates/project/items.py.tmpl
crawlo/templates/project/middlewares.py.tmpl
crawlo/templates/project/pipelines.py.tmpl
crawlo/templates/project/settings.py.tmpl
crawlo/templates/project/settings_distributed.py.tmpl
crawlo/templates/project/settings_gentle.py.tmpl
crawlo/templates/project/settings_high_performance.py.tmpl
crawlo/templates/project/settings_minimal.py.tmpl
crawlo/templates/project/settings_simple.py.tmpl
crawlo/templates/project/spiders/__init__.py.tmpl
crawlo/templates/spider/spider.py.tmpl
crawlo/tools/__init__.py
crawlo/tools/file_downloader.py
crawlo/tools/mysql_exists_checker.py
crawlo/tools/text_utils.py
crawlo/tools/time_utils.py
crawlo/tools/adaptive_selector/__init__.py
crawlo/tools/adaptive_selector/element_fingerprint.py
crawlo/tools/adaptive_selector/similarity_matcher.py
crawlo/tools/adaptive_selector/storage.py
crawlo/utils/__init__.py
crawlo/utils/async_lock.py
crawlo/utils/config_manager.py
crawlo/utils/curl_parser.py
crawlo/utils/encoding_detector.py
crawlo/utils/error_handler.py
crawlo/utils/func_tools.py
crawlo/utils/misc.py
crawlo/utils/page_utils.py
crawlo/utils/priority.py
crawlo/utils/process_utils.py
crawlo/utils/queue_helper.py
crawlo/utils/resource_manager.py
crawlo/utils/singleton.py
crawlo/utils/time_utils.py
crawlo/utils/batch/__init__.py
crawlo/utils/batch/batch_manager.py
crawlo/utils/batch/batch_processor.py
crawlo/utils/db/__init__.py
crawlo/utils/db/database_connection_pool.py
crawlo/utils/db/mongo_connection_pool.py
crawlo/utils/db/mysql_connection_pool.py
crawlo/utils/db/mysql_helper.py
crawlo/utils/db/pipeline_utils.py
crawlo/utils/db/sql_builder.py
crawlo/utils/monitor/__init__.py
crawlo/utils/monitor/monitor_manager.py
crawlo/utils/monitor/performance_monitor.py
crawlo/utils/redis/__init__.py
crawlo/utils/redis/config.py
crawlo/utils/redis/keys.py
crawlo/utils/redis/pool.py
crawlo/utils/request/__init__.py
crawlo/utils/request/fingerprint.py
crawlo/utils/request/request.py
crawlo/utils/request/request_serializer.py
crawlo/utils/request/response_helper.py
crawlo/utils/spider/__init__.py
crawlo/utils/spider/controlled_spider_mixin.py
crawlo/utils/spider/spider_loader.py
crawlo/utils/spider/spider_resolver.py
tests/ANTI_DETECTION_TEST_REPORT.md
tests/RESOURCE_LEAK_TEST_REPORT.md
tests/TEST_REPORT.md
tests/__init__.py
tests/advanced_tools_example.py
tests/authenticated_proxy_example.py
tests/baidu_performance_test.py
tests/baidu_test.py
tests/bug_check_test.py
tests/china_cloudflare_test_spider.py
tests/cleaners_example.py
tests/cloudflare_test_spider.py
tests/comprehensive_framework_test.py
tests/comprehensive_test.py
tests/comprehensive_test_suite.py
tests/comprehensive_testing_summary.md
tests/config_validation_demo.py
tests/controlled_spider_example.py
tests/date_tools_example.py
tests/debug_configure.py
tests/debug_framework_logger.py
tests/debug_log_config.py
tests/debug_log_levels.py
tests/debug_pipelines.py
tests/detailed_log_test.py
tests/distributed_dedup_test.py
tests/distributed_test.py
tests/distributed_test_debug.py
tests/dynamic_loading_example.py
tests/dynamic_loading_test.py
tests/error_handling_example.py
tests/explain_mysql_update_behavior.py
tests/final_comprehensive_test.py
tests/final_log_test.py
tests/final_validation_test.py
tests/final_verification.py
tests/fix_log_test.py
tests/framework_performance_test.py
tests/log_buffering_test.py
tests/log_generation_timing_test.py
tests/monitor_redis_dedup.sh
tests/optimized_performance_test.py
tests/performance_comparison.py
tests/queue_blocking_test.py
tests/queue_test.py
tests/redis_key_validation_demo.py
tests/request_params_example.py
tests/response_improvements_example.py
tests/run_all_leak_tests.py
tests/simple_cli_test.py
tests/simple_command_test.py
tests/simple_crawlo_test.py
tests/simple_follow_test.py
tests/simple_log_test2.py
tests/simple_optimization_test.py
tests/simple_queue_type_test.py
tests/simple_response_selector_test.py
tests/simple_selector_test.py
tests/simple_spider_test.py
tests/simple_url_test.py
tests/simulate_mysql_update_test.py
tests/spider_log_timing_test.py
tests/test_ack_call_analysis.py
tests/test_ack_call_fix.py
tests/test_ack_method_debug.py
tests/test_ack_method_fix.py
tests/test_ack_method_verification.py
tests/test_adaptive_multiple_scenarios.py
tests/test_adaptive_real_scenario.py
tests/test_adaptive_selector.py
tests/test_advanced_tools.py
tests/test_all_commands.py
tests/test_all_pipeline_fingerprints.py
tests/test_all_queues.py
tests/test_all_redis_key_configs.py
tests/test_anti_detection.py
tests/test_anti_detection_real_world.py
tests/test_async_lock.py
tests/test_asyncmy_usage.py
tests/test_backpressure.py
tests/test_backpressure_effectiveness.py
tests/test_batch_processor.py
tests/test_browser_leak.py
tests/test_cache_leak.py
tests/test_circular_reference_leak.py
tests/test_cleaners.py
tests/test_cli_arguments.py
tests/test_close_signal_handling.py
tests/test_cloudflare_real_sites.py
tests/test_complete_ack_solution.py
tests/test_component_factory.py
tests/test_concurrency_control.py
tests/test_config_consistency.py
tests/test_config_merge.py
tests/test_config_validator.py
tests/test_controlled_spider_mixin.py
tests/test_coroutine_leak.py
tests/test_crawler_process_import.py
tests/test_crawler_process_spider_modules.py
tests/test_crawlo_proxy_integration.py
tests/test_critical1_singleton_lock.py
tests/test_critical3_double_cleanup.py
tests/test_curl_checkpoint.py
tests/test_database_connection_leak.py
tests/test_date_tools.py
tests/test_dedup_fix.py
tests/test_dedup_pipeline_consistency.py
tests/test_default_header_middleware.py
tests/test_distributed.py
tests/test_double_check_exit.py
tests/test_double_crawlo_fix.py
tests/test_double_crawlo_fix_simple.py
tests/test_downloader_proxy_compatibility.py
tests/test_dynamic_downloader.py
tests/test_edge_cases.py
tests/test_encoding_core.py
tests/test_encoding_detection.py
tests/test_engine_loop_optimization.py
tests/test_engine_loop_performance.py
tests/test_enhanced_error_handler.py
tests/test_enhanced_error_handler_comprehensive.py
tests/test_error_handler_compatibility.py
tests/test_error_handling.py
tests/test_extract_spider_name.py
tests/test_factories.py
tests/test_factory_compatibility.py
tests/test_file_handle_leak.py
tests/test_final_validation.py
tests/test_fingerprint_consistency.py
tests/test_fingerprint_performance.py
tests/test_fingerprint_simple.py
tests/test_get_component_logger.py
tests/test_hash_performance.py
tests/test_high1_subscriber_silent_error.py
tests/test_high2_processor_race.py
tests/test_high3_semaphore_lock.py
tests/test_high4_coroutine_result_loss.py
tests/test_http_connection_leak.py
tests/test_hybrid_downloader_optimization.py
tests/test_integration.py
tests/test_intelligent_backpressure.py
tests/test_item_dedup_redis_key.py
tests/test_key_format_fix.py
tests/test_key_format_fix_verification.py
tests/test_large_scale_helper.py
tests/test_logging_enhancements.py
tests/test_logging_final.py
tests/test_logging_integration.py
tests/test_logging_system.py
tests/test_medium6_intelligent_scheduler.py
tests/test_medium_low_defects.py
tests/test_memory_filter_async.py
tests/test_middleware_debug.py
tests/test_middleware_lifecycle.py
tests/test_mode_consistency.py
tests/test_multi_directory.py
tests/test_multiple_spider_modules.py
tests/test_mysql_exists_checker.py
tests/test_mysql_exists_checker_e2e.py
tests/test_mysql_optimizations.py
tests/test_mysql_pipeline.py
tests/test_mysql_pipeline_comprehensive.py
tests/test_mysql_pipeline_config.py
tests/test_mysql_pipeline_error.py
tests/test_mysql_pipeline_init_log.py
tests/test_mysql_pipeline_integration.py
tests/test_mysql_pipeline_refactor.py
tests/test_mysql_pipeline_refactor_simple.py
tests/test_mysql_pipeline_robustness.py
tests/test_mysql_pipeline_types.py
tests/test_mysql_update_columns.py
tests/test_offsite_middleware.py
tests/test_offsite_middleware_simple.py
tests/test_optimized_selector_naming.py
tests/test_page_action_handler.py
tests/test_parsel.py
tests/test_performance.py
tests/test_performance_monitor.py
tests/test_pipeline_fingerprint_consistency.py
tests/test_priority_behavior.py
tests/test_priority_consistency.py
tests/test_priority_consistency_fixed.py
tests/test_processing_queue_debug.py
tests/test_processing_queue_simple.py
tests/test_proxy_health_check.py
tests/test_proxy_middleware.py
tests/test_proxy_middleware_enhanced.py
tests/test_proxy_middleware_integration.py
tests/test_proxy_middleware_refactored.py
tests/test_proxy_only.py
tests/test_proxy_providers.py
tests/test_proxy_stats.py
tests/test_proxy_strategies.py
tests/test_proxy_with_downloader.py
tests/test_queue_empty_check.py
tests/test_queue_leak.py
tests/test_queue_manager_double_crawlo.py
tests/test_queue_manager_redis_key.py
tests/test_queue_mode_params.py
tests/test_queue_naming.py
tests/test_queue_scores.py
tests/test_queue_type.py
tests/test_queue_type_redis_config_consistency.py
tests/test_random_headers_default.py
tests/test_random_headers_necessity.py
tests/test_random_user_agent.py
tests/test_redis_config.py
tests/test_redis_connection_leak.py
tests/test_redis_connection_pool.py
tests/test_redis_key_consistency.py
tests/test_redis_key_integration.py
tests/test_redis_key_manager.py
tests/test_redis_key_naming.py
tests/test_redis_key_structure.py
tests/test_redis_key_validator.py
tests/test_redis_queue.py
tests/test_redis_queue_name_fix.py
tests/test_redis_queue_type_fallback.py
tests/test_request_ignore_middleware.py
tests/test_request_params.py
tests/test_request_serialization.py
tests/test_resource_leak_detection.py
tests/test_response_code_middleware.py
tests/test_response_filter_middleware.py
tests/test_response_follow.py
tests/test_response_improvements.py
tests/test_response_selector_methods.py
tests/test_response_url_methods.py
tests/test_response_urljoin.py
tests/test_retry_middleware.py
tests/test_retry_middleware_realistic.py
tests/test_scheduler.py
tests/test_scheduler_config_update.py
tests/test_scheduling.py
tests/test_scheduling_integration.py
tests/test_scrapy_style_encoding.py
tests/test_selector_optimizations.py
tests/test_shell.py
tests/test_simple_response.py
tests/test_spider_loader.py
tests/test_spider_loader_comprehensive.py
tests/test_spider_modules.py
tests/test_spider_name_in_redis_keys.py
tests/test_storage_backends.py
tests/test_telecom_spider_redis_key.py
tests/test_template_content.py
tests/test_template_redis_key.py
tests/test_thread_leak.py
tests/test_throttle_simple_config.py
tests/test_today_changes.py
tests/test_tools.py
tests/test_user_agent_randomness.py
tests/test_user_agents.py
tests/untested_features_report.md
tests/verify_debug.py
tests/verify_distributed.py
tests/verify_log_fix.py
tests/verify_mysql_warnings.py
tests/ofweek_scrapy/scrapy.cfg
tests/ofweek_scrapy/ofweek_scrapy/__init__.py
tests/ofweek_scrapy/ofweek_scrapy/items.py
tests/ofweek_scrapy/ofweek_scrapy/middlewares.py
tests/ofweek_scrapy/ofweek_scrapy/pipelines.py
tests/ofweek_scrapy/ofweek_scrapy/settings.py
tests/ofweek_scrapy/ofweek_scrapy/spiders/__init__.py
tests/scrapy_comparison/ofweek_scrapy.py
tests/scrapy_comparison/scrapy_test.py
tests/test_spiders/__init__.py
tests/test_spiders/test_spider.py