Sneakpeek
0.2
Overview
Quick start
Local handler debugging
Design
Deployment options
Middleware
Rate limiter
Robots.txt
User Agent injector
Proxy middleware
Requests logging middleware
Implementing your own middleware
API
Sneakpeek
»
Index
Index
A
|
B
|
C
|
D
|
E
|
F
|
G
|
H
|
I
|
K
|
L
|
M
|
N
|
O
|
P
|
Q
|
R
|
S
|
T
|
U
|
W
A
acquired (sneakpeek.scheduler.model.Lease attribute)
acquired_until (sneakpeek.scheduler.model.Lease attribute)
B
BaseMiddleware (class in sneakpeek.middleware.base)
browsers (sneakpeek.middleware.user_agent_injecter_middleware.UserAgentInjecterMiddlewareConfig attribute)
C
config (sneakpeek.scraper.model.Scraper attribute)
configure_logging() (in module sneakpeek.logging)
count_invocations() (in module sneakpeek.metrics)
create() (sneakpeek.server.SneakpeekServer static method)
create_api() (in module sneakpeek.api)
created_at (sneakpeek.queue.model.Task attribute)
CRONTAB (sneakpeek.scheduler.model.TaskSchedule attribute)
D
DEAD (sneakpeek.queue.model.TaskStatus attribute)
delete() (sneakpeek.scraper.model.ScraperContextABC method)
delete_old_tasks() (sneakpeek.queue.in_memory_storage.InMemoryQueueStorage method)
(sneakpeek.queue.model.QueueABC method)
(sneakpeek.queue.model.QueueStorageABC method)
(sneakpeek.queue.queue.Queue method)
(sneakpeek.queue.redis_storage.RedisQueueStorage method)
dequeue() (sneakpeek.queue.model.QueueABC method)
(sneakpeek.queue.queue.Queue method)
dequeue_task() (sneakpeek.queue.in_memory_storage.InMemoryQueueStorage method)
(sneakpeek.queue.model.QueueStorageABC method)
(sneakpeek.queue.redis_storage.RedisQueueStorage method)
download_file() (sneakpeek.scraper.context.ScraperContext method)
(sneakpeek.scraper.model.ScraperContextABC method)
download_files() (sneakpeek.scraper.context.ScraperContext method)
(sneakpeek.scraper.model.ScraperContextABC method)
DynamicScraperHandler (class in sneakpeek.scraper.dynamic_scraper_handler)
E
enqueue() (sneakpeek.queue.model.QueueABC method)
(sneakpeek.queue.queue.Queue method)
enqueue_task() (sneakpeek.queue.in_memory_storage.InMemoryQueueStorage method)
(sneakpeek.queue.model.QueueStorageABC method)
(sneakpeek.queue.redis_storage.RedisQueueStorage method)
EnqueueTaskRequest (class in sneakpeek.queue.model)
EVERY_DAY (sneakpeek.scheduler.model.TaskSchedule attribute)
EVERY_HOUR (sneakpeek.scheduler.model.TaskSchedule attribute)
EVERY_MINUTE (sneakpeek.scheduler.model.TaskSchedule attribute)
EVERY_MONTH (sneakpeek.scheduler.model.TaskSchedule attribute)
EVERY_SECOND (sneakpeek.scheduler.model.TaskSchedule attribute)
EVERY_WEEK (sneakpeek.scheduler.model.TaskSchedule attribute)
F
FAILED (sneakpeek.queue.model.TaskStatus attribute)
filter() (sneakpeek.logging.TaskContextInjectingFilter method)
finished_at (sneakpeek.queue.model.Task attribute)
full_match (sneakpeek.middleware.parser.RegexMatch attribute)
G
get() (sneakpeek.scraper.model.ScraperContextABC method)
get_api_entrypoint() (in module sneakpeek.api)
get_queue_len() (sneakpeek.queue.in_memory_storage.InMemoryQueueStorage method)
(sneakpeek.queue.model.QueueABC method)
(sneakpeek.queue.model.QueueStorageABC method)
(sneakpeek.queue.queue.Queue method)
(sneakpeek.queue.redis_storage.RedisQueueStorage method)
get_task_instance() (sneakpeek.queue.in_memory_storage.InMemoryQueueStorage method)
(sneakpeek.queue.model.QueueABC method)
(sneakpeek.queue.model.QueueStorageABC method)
(sneakpeek.queue.queue.Queue method)
(sneakpeek.queue.redis_storage.RedisQueueStorage method)
get_task_instances() (sneakpeek.queue.in_memory_storage.InMemoryQueueStorage method)
(sneakpeek.queue.model.QueueABC method)
(sneakpeek.queue.model.QueueStorageABC method)
(sneakpeek.queue.queue.Queue method)
(sneakpeek.queue.redis_storage.RedisQueueStorage method)
get_tasks() (sneakpeek.queue.in_memory_storage.InMemoryQueueStorage method)
(sneakpeek.queue.model.QueueStorageABC method)
(sneakpeek.queue.redis_storage.RedisQueueStorage method)
groups (sneakpeek.middleware.parser.RegexMatch attribute)
H
handler (sneakpeek.scraper.model.Scraper attribute)
head() (sneakpeek.scraper.model.ScraperContextABC method)
HIGH (sneakpeek.queue.model.TaskPriority attribute)
HttpMethod (class in sneakpeek.scraper.model)
I
id (sneakpeek.queue.model.Task attribute)
(sneakpeek.scraper.model.Scraper attribute)
INACTIVE (sneakpeek.scheduler.model.TaskSchedule attribute)
InMemoryLeaseStorage (class in sneakpeek.scheduler.in_memory_lease_storage)
InMemoryQueueStorage (class in sneakpeek.queue.in_memory_storage)
InMemoryScraperStorage (class in sneakpeek.scraper.in_memory_storage)
K
kill_dead_tasks() (sneakpeek.queue.model.QueueABC method)
(sneakpeek.queue.queue.Queue method)
KILLED (sneakpeek.queue.model.TaskStatus attribute)
L
last_active_at (sneakpeek.queue.model.Task attribute)
Lease (class in sneakpeek.scheduler.model)
LeaseStorageABC (class in sneakpeek.scheduler.model)
LOG (sneakpeek.middleware.robots_txt_middleware.RobotsTxtViolationStrategy attribute)
log_request (sneakpeek.middleware.requests_logging_middleware.RequestsLoggingMiddlewareConfig attribute)
log_response (sneakpeek.middleware.requests_logging_middleware.RequestsLoggingMiddlewareConfig attribute)
M
max_requests (sneakpeek.middleware.rate_limiter_middleware.RateLimiterMiddlewareConfig attribute)
maybe_acquire_lease() (sneakpeek.scheduler.in_memory_lease_storage.InMemoryLeaseStorage method)
(sneakpeek.scheduler.model.LeaseStorageABC method)
(sneakpeek.scheduler.redis_lease_storage.RedisLeaseStorage method)
measure_latency() (in module sneakpeek.metrics)
Middleware (class in sneakpeek.scraper.model)
module
sneakpeek.api
sneakpeek.logging
sneakpeek.metrics
sneakpeek.middleware.base
sneakpeek.middleware.parser
sneakpeek.middleware.proxy_middleware
sneakpeek.middleware.rate_limiter_middleware
sneakpeek.middleware.requests_logging_middleware
sneakpeek.middleware.robots_txt_middleware
sneakpeek.middleware.user_agent_injecter_middleware
sneakpeek.queue.consumer
sneakpeek.queue.in_memory_storage
sneakpeek.queue.model
sneakpeek.queue.queue
sneakpeek.queue.redis_storage
sneakpeek.queue.tasks
sneakpeek.scheduler.in_memory_lease_storage
sneakpeek.scheduler.model
sneakpeek.scheduler.redis_lease_storage
sneakpeek.scheduler.scheduler
sneakpeek.scraper.context
sneakpeek.scraper.dynamic_scraper_handler
sneakpeek.scraper.in_memory_storage
sneakpeek.scraper.model
sneakpeek.scraper.redis_storage
sneakpeek.scraper.runner
sneakpeek.scraper.task_handler
sneakpeek.server
MultiPeriodicTasksStorage (class in sneakpeek.scheduler.model)
N
name (sneakpeek.middleware.base.BaseMiddleware property)
(sneakpeek.middleware.parser.ParserMiddleware property)
(sneakpeek.middleware.proxy_middleware.ProxyMiddleware property)
(sneakpeek.middleware.rate_limiter_middleware.RateLimiterMiddleware property)
(sneakpeek.middleware.requests_logging_middleware.RequestsLoggingMiddleware property)
(sneakpeek.middleware.robots_txt_middleware.RobotsTxtMiddleware property)
(sneakpeek.middleware.user_agent_injecter_middleware.UserAgentInjecterMiddleware property)
(sneakpeek.scheduler.model.Lease attribute)
(sneakpeek.scraper.dynamic_scraper_handler.DynamicScraperHandler property)
(sneakpeek.scraper.model.Middleware property)
(sneakpeek.scraper.model.Scraper attribute)
(sneakpeek.scraper.model.ScraperHandler property)
NORMAL (sneakpeek.queue.model.TaskPriority attribute)
O
on_request() (sneakpeek.middleware.base.BaseMiddleware method)
(sneakpeek.middleware.proxy_middleware.ProxyMiddleware method)
(sneakpeek.middleware.rate_limiter_middleware.RateLimiterMiddleware method)
(sneakpeek.middleware.requests_logging_middleware.RequestsLoggingMiddleware method)
(sneakpeek.middleware.robots_txt_middleware.RobotsTxtMiddleware method)
(sneakpeek.middleware.user_agent_injecter_middleware.UserAgentInjecterMiddleware method)
(sneakpeek.scraper.model.Middleware method)
on_response() (sneakpeek.middleware.base.BaseMiddleware method)
(sneakpeek.middleware.requests_logging_middleware.RequestsLoggingMiddleware method)
(sneakpeek.scraper.model.Middleware method)
options() (sneakpeek.scraper.model.ScraperContextABC method)
owner_id (sneakpeek.scheduler.model.Lease attribute)
P
ParserMiddleware (class in sneakpeek.middleware.parser)
payload (sneakpeek.queue.model.EnqueueTaskRequest attribute)
(sneakpeek.queue.model.Task attribute)
PENDING (sneakpeek.queue.model.TaskStatus attribute)
PeriodicTasksStorageABC (class in sneakpeek.scheduler.model)
ping_task() (sneakpeek.queue.model.QueueABC method)
(sneakpeek.queue.queue.Queue method)
post() (sneakpeek.scraper.model.ScraperContextABC method)
priority (sneakpeek.queue.model.EnqueueTaskRequest attribute)
(sneakpeek.queue.model.Task attribute)
(sneakpeek.scraper.model.Scraper attribute)
proxy (sneakpeek.middleware.proxy_middleware.ProxyMiddlewareConfig attribute)
proxy_auth (sneakpeek.middleware.proxy_middleware.ProxyMiddlewareConfig attribute)
ProxyMiddleware (class in sneakpeek.middleware.proxy_middleware)
ProxyMiddlewareConfig (class in sneakpeek.middleware.proxy_middleware)
put() (sneakpeek.scraper.model.ScraperContextABC method)
Q
Queue (class in sneakpeek.queue.queue)
QueueABC (class in sneakpeek.queue.model)
QueueStorageABC (class in sneakpeek.queue.model)
R
rate_limited_strategy (sneakpeek.middleware.rate_limiter_middleware.RateLimiterMiddlewareConfig attribute)
RateLimitedException
RateLimitedStrategy (class in sneakpeek.middleware.rate_limiter_middleware)
RateLimiterMiddleware (class in sneakpeek.middleware.rate_limiter_middleware)
RateLimiterMiddlewareConfig (class in sneakpeek.middleware.rate_limiter_middleware)
RedisLeaseStorage (class in sneakpeek.scheduler.redis_lease_storage)
RedisQueueStorage (class in sneakpeek.queue.redis_storage)
RedisScraperStorage (class in sneakpeek.scraper.redis_storage)
regex() (sneakpeek.middleware.parser.ParserMiddleware method)
RegexMatch (class in sneakpeek.middleware.parser)
release_lease() (sneakpeek.scheduler.in_memory_lease_storage.InMemoryLeaseStorage method)
(sneakpeek.scheduler.model.LeaseStorageABC method)
(sneakpeek.scheduler.redis_lease_storage.RedisLeaseStorage method)
Request (class in sneakpeek.scraper.model)
request() (sneakpeek.scraper.context.ScraperContext method)
(sneakpeek.scraper.model.ScraperContextABC method)
RequestsLoggingMiddleware (class in sneakpeek.middleware.requests_logging_middleware)
RequestsLoggingMiddlewareConfig (class in sneakpeek.middleware.requests_logging_middleware)
result (sneakpeek.queue.model.Task attribute)
RobotsTxtMiddleware (class in sneakpeek.middleware.robots_txt_middleware)
RobotsTxtMiddlewareConfig (class in sneakpeek.middleware.robots_txt_middleware)
RobotsTxtViolationException
RobotsTxtViolationStrategy (class in sneakpeek.middleware.robots_txt_middleware)
run() (sneakpeek.scraper.dynamic_scraper_handler.DynamicScraperHandler method)
(sneakpeek.scraper.model.ScraperHandler method)
(sneakpeek.scraper.model.ScraperRunnerABC method)
(sneakpeek.scraper.runner.ScraperRunner method)
S
schedule (sneakpeek.scraper.model.Scraper attribute)
schedule_crontab (sneakpeek.scraper.model.Scraper attribute)
Scheduler (class in sneakpeek.scheduler.scheduler)
SchedulerABC (class in sneakpeek.scheduler.model)
Scraper (class in sneakpeek.scraper.model)
ScraperContext (class in sneakpeek.scraper.context)
ScraperContextABC (class in sneakpeek.scraper.model)
ScraperHandler (class in sneakpeek.scraper.model)
ScraperNotFoundError
ScraperRunner (class in sneakpeek.scraper.runner)
ScraperRunnerABC (class in sneakpeek.scraper.model)
ScraperStorageABC (class in sneakpeek.scraper.model)
serve() (sneakpeek.server.SneakpeekServer method)
sneakpeek.api
module
sneakpeek.logging
module
sneakpeek.metrics
module
sneakpeek.middleware.base
module
sneakpeek.middleware.parser
module
sneakpeek.middleware.proxy_middleware
module
sneakpeek.middleware.rate_limiter_middleware
module
sneakpeek.middleware.requests_logging_middleware
module
sneakpeek.middleware.robots_txt_middleware
module
sneakpeek.middleware.user_agent_injecter_middleware
module
sneakpeek.queue.consumer
module
sneakpeek.queue.in_memory_storage
module
sneakpeek.queue.model
module
sneakpeek.queue.queue
module
sneakpeek.queue.redis_storage
module
sneakpeek.queue.tasks
module
sneakpeek.scheduler.in_memory_lease_storage
module
sneakpeek.scheduler.model
module
sneakpeek.scheduler.redis_lease_storage
module
sneakpeek.scheduler.scheduler
module
sneakpeek.scraper.context
module
sneakpeek.scraper.dynamic_scraper_handler
module
sneakpeek.scraper.in_memory_storage
module
sneakpeek.scraper.model
module
sneakpeek.scraper.redis_storage
module
sneakpeek.scraper.runner
module
sneakpeek.scraper.task_handler
module
sneakpeek.server
module
SneakpeekServer (class in sneakpeek.server)
STARTED (sneakpeek.queue.model.TaskStatus attribute)
started_at (sneakpeek.queue.model.Task attribute)
state (sneakpeek.scraper.model.Scraper attribute)
StaticPeriodicTasksStorage (class in sneakpeek.scheduler.model)
status (sneakpeek.queue.model.Task attribute)
stop() (sneakpeek.server.SneakpeekServer method)
StorageIsReadOnlyError
SUCCEEDED (sneakpeek.queue.model.TaskStatus attribute)
T
Task (class in sneakpeek.queue.model)
task_context() (in module sneakpeek.logging)
task_handler (sneakpeek.queue.model.EnqueueTaskRequest attribute)
(sneakpeek.queue.model.Task attribute)
task_name (sneakpeek.queue.model.EnqueueTaskRequest attribute)
(sneakpeek.queue.model.Task attribute)
TaskContextInjectingFilter (class in sneakpeek.logging)
TaskHasActiveRunError
TaskNotFoundError
TaskPingFinishedError
TaskPingNotStartedError
TaskPriority (class in sneakpeek.queue.model)
TaskSchedule (class in sneakpeek.scheduler.model)
TaskStatus (class in sneakpeek.queue.model)
TaskTimedOut
THROW (sneakpeek.middleware.rate_limiter_middleware.RateLimitedStrategy attribute)
(sneakpeek.middleware.robots_txt_middleware.RobotsTxtViolationStrategy attribute)
time_window (sneakpeek.middleware.rate_limiter_middleware.RateLimiterMiddlewareConfig attribute)
timeout (sneakpeek.queue.model.EnqueueTaskRequest attribute)
(sneakpeek.queue.model.Task attribute)
(sneakpeek.scraper.model.Scraper attribute)
U
UnknownScraperHandlerError
UnknownTaskHandlerError
update_scraper_state() (sneakpeek.scraper.context.ScraperContext method)
(sneakpeek.scraper.model.ScraperContextABC method)
update_task() (sneakpeek.queue.in_memory_storage.InMemoryQueueStorage method)
(sneakpeek.queue.model.QueueABC method)
(sneakpeek.queue.model.QueueStorageABC method)
(sneakpeek.queue.queue.Queue method)
(sneakpeek.queue.redis_storage.RedisQueueStorage method)
update_tasks() (sneakpeek.scheduler.scheduler.Scheduler method)
use_external_data (sneakpeek.middleware.user_agent_injecter_middleware.UserAgentInjecterMiddlewareConfig attribute)
UserAgentInjecterMiddleware (class in sneakpeek.middleware.user_agent_injecter_middleware)
UserAgentInjecterMiddlewareConfig (class in sneakpeek.middleware.user_agent_injecter_middleware)
UTMOST (sneakpeek.queue.model.TaskPriority attribute)
W
WAIT (sneakpeek.middleware.rate_limiter_middleware.RateLimitedStrategy attribute)