Sneakpeek
0.1
Overview
Quick start
Design
Deployment options
Plugins
Rate limiter
Robots.txt
User Agent injector
Proxy plugin
Requests logging middleware
Adding your own plugin
API
Sneakpeek
»
Index
Index
A
|
B
|
C
|
D
|
E
|
F
|
G
|
H
|
I
|
K
|
L
|
M
|
N
|
O
|
P
|
Q
|
R
|
S
|
T
|
U
|
W
A
acquired (sneakpeek.lib.models.Lease attribute)
acquired_until (sneakpeek.lib.models.Lease attribute)
add_scraper_job() (sneakpeek.lib.storage.base.ScraperJobsStorage method)
(sneakpeek.lib.storage.in_memory_storage.InMemoryScraperJobsStorage method)
(sneakpeek.lib.storage.redis_storage.RedisScraperJobsStorage method)
after_response() (sneakpeek.plugins.requests_logging_plugin.RequestsLoggingPlugin method)
(sneakpeek.scraper_context.AfterResponsePlugin method)
AfterResponsePlugin (class in sneakpeek.scraper_context)
B
before_request() (sneakpeek.plugins.proxy_plugin.ProxyPlugin method)
(sneakpeek.plugins.rate_limiter_plugin.RateLimiterPlugin method)
(sneakpeek.plugins.requests_logging_plugin.RequestsLoggingPlugin method)
(sneakpeek.plugins.robots_txt_plugin.RobotsTxtPlugin method)
(sneakpeek.plugins.user_agent_injecter_plugin.UserAgentInjecterPlugin method)
(sneakpeek.scraper_context.BeforeRequestPlugin method)
BeforeRequestPlugin (class in sneakpeek.scraper_context)
browsers (sneakpeek.plugins.user_agent_injecter_plugin.UserAgentInjecterPluginConfig attribute)
C
config (sneakpeek.lib.models.Scraper attribute)
configure_logging() (in module sneakpeek.logging)
count_invocations() (in module sneakpeek.metrics)
create() (sneakpeek.server.SneakpeekServer static method)
create_api() (in module sneakpeek.api)
create_scraper() (sneakpeek.lib.storage.base.ScrapersStorage method)
(sneakpeek.lib.storage.in_memory_storage.InMemoryScrapersStorage method)
(sneakpeek.lib.storage.redis_storage.RedisScrapersStorage method)
created_at (sneakpeek.lib.models.ScraperJob attribute)
CRONTAB (sneakpeek.lib.models.ScraperSchedule attribute)
D
DEAD (sneakpeek.lib.models.ScraperJobStatus attribute)
delete() (sneakpeek.scraper_context.ScraperContext method)
delete_old_scraper_jobs() (sneakpeek.lib.storage.base.ScraperJobsStorage method)
(sneakpeek.lib.storage.in_memory_storage.InMemoryScraperJobsStorage method)
(sneakpeek.lib.storage.redis_storage.RedisScraperJobsStorage method)
delete_scraper() (sneakpeek.lib.storage.base.ScrapersStorage method)
(sneakpeek.lib.storage.in_memory_storage.InMemoryScrapersStorage method)
(sneakpeek.lib.storage.redis_storage.RedisScrapersStorage method)
dequeue() (sneakpeek.lib.queue.QueueABC method)
dequeue_scraper_job() (sneakpeek.lib.storage.base.ScraperJobsStorage method)
(sneakpeek.lib.storage.in_memory_storage.InMemoryScraperJobsStorage method)
(sneakpeek.lib.storage.redis_storage.RedisScraperJobsStorage method)
E
enqueue() (sneakpeek.lib.queue.QueueABC method)
EVERY_DAY (sneakpeek.lib.models.ScraperSchedule attribute)
EVERY_HOUR (sneakpeek.lib.models.ScraperSchedule attribute)
EVERY_MINUTE (sneakpeek.lib.models.ScraperSchedule attribute)
EVERY_MONTH (sneakpeek.lib.models.ScraperSchedule attribute)
EVERY_SECOND (sneakpeek.lib.models.ScraperSchedule attribute)
EVERY_WEEK (sneakpeek.lib.models.ScraperSchedule attribute)
F
FAILED (sneakpeek.lib.models.ScraperJobStatus attribute)
filter() (sneakpeek.logging.ScraperContextInjectingFilter method)
finished_at (sneakpeek.lib.models.ScraperJob attribute)
G
get() (sneakpeek.scraper_context.ScraperContext method)
get_api_entrypoint() (in module sneakpeek.api)
get_queue_len() (sneakpeek.lib.queue.QueueABC method)
(sneakpeek.lib.storage.base.ScraperJobsStorage method)
(sneakpeek.lib.storage.in_memory_storage.InMemoryScraperJobsStorage method)
(sneakpeek.lib.storage.redis_storage.RedisScraperJobsStorage method)
get_scraper() (sneakpeek.lib.storage.base.ScrapersStorage method)
(sneakpeek.lib.storage.in_memory_storage.InMemoryScrapersStorage method)
(sneakpeek.lib.storage.redis_storage.RedisScrapersStorage method)
get_scraper_job() (sneakpeek.lib.storage.base.ScraperJobsStorage method)
(sneakpeek.lib.storage.in_memory_storage.InMemoryScraperJobsStorage method)
(sneakpeek.lib.storage.redis_storage.RedisScraperJobsStorage method)
get_scraper_jobs() (sneakpeek.lib.storage.base.ScraperJobsStorage method)
(sneakpeek.lib.storage.in_memory_storage.InMemoryScraperJobsStorage method)
(sneakpeek.lib.storage.redis_storage.RedisScraperJobsStorage method)
get_scrapers() (sneakpeek.lib.storage.base.ScrapersStorage method)
(sneakpeek.lib.storage.in_memory_storage.InMemoryScrapersStorage method)
(sneakpeek.lib.storage.redis_storage.RedisScrapersStorage method)
H
handler (sneakpeek.lib.models.Scraper attribute)
head() (sneakpeek.scraper_context.ScraperContext method)
HIGH (sneakpeek.lib.models.ScraperJobPriority attribute)
HttpMethod (class in sneakpeek.scraper_context)
I
id (sneakpeek.lib.models.Scraper attribute)
(sneakpeek.lib.models.ScraperJob attribute)
INACTIVE (sneakpeek.lib.models.ScraperSchedule attribute)
InMemoryLeaseStorage (class in sneakpeek.lib.storage.in_memory_storage)
InMemoryScraperJobsStorage (class in sneakpeek.lib.storage.in_memory_storage)
InMemoryScrapersStorage (class in sneakpeek.lib.storage.in_memory_storage)
is_read_only() (sneakpeek.lib.storage.base.ScrapersStorage method)
(sneakpeek.lib.storage.in_memory_storage.InMemoryScrapersStorage method)
(sneakpeek.lib.storage.redis_storage.RedisScrapersStorage method)
K
kill_dead_scraper_jobs() (sneakpeek.lib.queue.QueueABC method)
KILLED (sneakpeek.lib.models.ScraperJobStatus attribute)
L
last_active_at (sneakpeek.lib.models.ScraperJob attribute)
Lease (class in sneakpeek.lib.models)
LeaseStorage (class in sneakpeek.lib.storage.base)
LOG (sneakpeek.plugins.robots_txt_plugin.RobotsTxtViolationStrategy attribute)
log_request (sneakpeek.plugins.requests_logging_plugin.RequestsLoggingPluginConfig attribute)
log_response (sneakpeek.plugins.requests_logging_plugin.RequestsLoggingPluginConfig attribute)
M
max_requests (sneakpeek.plugins.rate_limiter_plugin.RateLimiterPluginConfig attribute)
maybe_acquire_lease() (sneakpeek.lib.storage.base.LeaseStorage method)
(sneakpeek.lib.storage.in_memory_storage.InMemoryLeaseStorage method)
(sneakpeek.lib.storage.redis_storage.RedisLeaseStorage method)
maybe_get_scraper() (sneakpeek.lib.storage.base.ScrapersStorage method)
(sneakpeek.lib.storage.in_memory_storage.InMemoryScrapersStorage method)
(sneakpeek.lib.storage.redis_storage.RedisScrapersStorage method)
measure_latency() (in module sneakpeek.metrics)
module
sneakpeek.api
sneakpeek.lib.errors
sneakpeek.lib.models
sneakpeek.lib.queue
sneakpeek.lib.storage.base
sneakpeek.lib.storage.in_memory_storage
sneakpeek.lib.storage.redis_storage
sneakpeek.logging
sneakpeek.metrics
sneakpeek.plugins.proxy_plugin
sneakpeek.plugins.rate_limiter_plugin
sneakpeek.plugins.requests_logging_plugin
sneakpeek.plugins.robots_txt_plugin
sneakpeek.plugins.user_agent_injecter_plugin
sneakpeek.plugins.utils
sneakpeek.runner
sneakpeek.scheduler
sneakpeek.scraper_config
sneakpeek.scraper_context
sneakpeek.scraper_handler
sneakpeek.server
sneakpeek.worker
N
name (sneakpeek.lib.models.Lease attribute)
(sneakpeek.lib.models.Scraper attribute)
(sneakpeek.plugins.proxy_plugin.ProxyPlugin property)
(sneakpeek.plugins.rate_limiter_plugin.RateLimiterPlugin property)
(sneakpeek.plugins.requests_logging_plugin.RequestsLoggingPlugin property)
(sneakpeek.plugins.robots_txt_plugin.RobotsTxtPlugin property)
(sneakpeek.plugins.user_agent_injecter_plugin.UserAgentInjecterPlugin property)
(sneakpeek.scraper_context.AfterResponsePlugin property)
(sneakpeek.scraper_context.BeforeRequestPlugin property)
(sneakpeek.scraper_handler.ScraperHandler property)
NORMAL (sneakpeek.lib.models.ScraperJobPriority attribute)
O
options() (sneakpeek.scraper_context.ScraperContext method)
owner_id (sneakpeek.lib.models.Lease attribute)
P
params (sneakpeek.scraper_config.ScraperConfig attribute)
PENDING (sneakpeek.lib.models.ScraperJobStatus attribute)
ping_scraper_job() (sneakpeek.lib.queue.QueueABC method)
ping_session() (sneakpeek.scraper_context.ScraperContext method)
plugins (sneakpeek.scraper_config.ScraperConfig attribute)
post() (sneakpeek.scraper_context.ScraperContext method)
priority (sneakpeek.lib.models.ScraperJob attribute)
proxy (sneakpeek.plugins.proxy_plugin.ProxyPluginConfig attribute)
proxy_auth (sneakpeek.plugins.proxy_plugin.ProxyPluginConfig attribute)
ProxyPlugin (class in sneakpeek.plugins.proxy_plugin)
ProxyPluginConfig (class in sneakpeek.plugins.proxy_plugin)
put() (sneakpeek.scraper_context.ScraperContext method)
Q
Queue (class in sneakpeek.lib.queue)
QueueABC (class in sneakpeek.lib.queue)
R
rate_limited_strategy (sneakpeek.plugins.rate_limiter_plugin.RateLimiterPluginConfig attribute)
RateLimitedException
RateLimitedStrategy (class in sneakpeek.plugins.rate_limiter_plugin)
RateLimiterPlugin (class in sneakpeek.plugins.rate_limiter_plugin)
RateLimiterPluginConfig (class in sneakpeek.plugins.rate_limiter_plugin)
RedisLeaseStorage (class in sneakpeek.lib.storage.redis_storage)
RedisScraperJobsStorage (class in sneakpeek.lib.storage.redis_storage)
RedisScrapersStorage (class in sneakpeek.lib.storage.redis_storage)
release_lease() (sneakpeek.lib.storage.base.LeaseStorage method)
(sneakpeek.lib.storage.in_memory_storage.InMemoryLeaseStorage method)
(sneakpeek.lib.storage.redis_storage.RedisLeaseStorage method)
Request (class in sneakpeek.scraper_context)
RequestsLoggingPlugin (class in sneakpeek.plugins.requests_logging_plugin)
RequestsLoggingPluginConfig (class in sneakpeek.plugins.requests_logging_plugin)
result (sneakpeek.lib.models.ScraperJob attribute)
RobotsTxtPlugin (class in sneakpeek.plugins.robots_txt_plugin)
RobotsTxtPluginConfig (class in sneakpeek.plugins.robots_txt_plugin)
RobotsTxtViolationException
RobotsTxtViolationStrategy (class in sneakpeek.plugins.robots_txt_plugin)
run() (sneakpeek.runner.Runner method)
(sneakpeek.runner.RunnerABC method)
(sneakpeek.scraper_handler.ScraperHandler method)
Runner (class in sneakpeek.runner)
RunnerABC (class in sneakpeek.runner)
S
schedule (sneakpeek.lib.models.Scraper attribute)
schedule_crontab (sneakpeek.lib.models.Scraper attribute)
schedule_priority (sneakpeek.lib.models.Scraper attribute)
Scheduler (class in sneakpeek.scheduler)
SchedulerABC (class in sneakpeek.scheduler)
Scraper (class in sneakpeek.lib.models)
scraper (sneakpeek.lib.models.ScraperJob attribute)
scraper_job_context() (in module sneakpeek.logging)
ScraperConfig (class in sneakpeek.scraper_config)
ScraperContext (class in sneakpeek.scraper_context)
ScraperContextInjectingFilter (class in sneakpeek.logging)
ScraperHandler (class in sneakpeek.scraper_handler)
ScraperHasActiveRunError
ScraperJob (class in sneakpeek.lib.models)
ScraperJobNotFoundError
ScraperJobPingFinishedError
ScraperJobPingNotStartedError
ScraperJobPriority (class in sneakpeek.lib.models)
ScraperJobsStorage (class in sneakpeek.lib.storage.base)
ScraperJobStatus (class in sneakpeek.lib.models)
ScraperNotFoundError
ScraperSchedule (class in sneakpeek.lib.models)
ScrapersStorage (class in sneakpeek.lib.storage.base)
search_scrapers() (sneakpeek.lib.storage.base.ScrapersStorage method)
(sneakpeek.lib.storage.in_memory_storage.InMemoryScrapersStorage method)
(sneakpeek.lib.storage.redis_storage.RedisScrapersStorage method)
serve() (sneakpeek.server.SneakpeekServer method)
sneakpeek.api
module
sneakpeek.lib.errors
module
sneakpeek.lib.models
module
sneakpeek.lib.queue
module
sneakpeek.lib.storage.base
module
sneakpeek.lib.storage.in_memory_storage
module
sneakpeek.lib.storage.redis_storage
module
sneakpeek.logging
module
sneakpeek.metrics
module
sneakpeek.plugins.proxy_plugin
module
sneakpeek.plugins.rate_limiter_plugin
module
sneakpeek.plugins.requests_logging_plugin
module
sneakpeek.plugins.robots_txt_plugin
module
sneakpeek.plugins.user_agent_injecter_plugin
module
sneakpeek.plugins.utils
module
sneakpeek.runner
module
sneakpeek.scheduler
module
sneakpeek.scraper_config
module
sneakpeek.scraper_context
module
sneakpeek.scraper_handler
module
sneakpeek.server
module
sneakpeek.worker
module
SneakpeekServer (class in sneakpeek.server)
STARTED (sneakpeek.lib.models.ScraperJobStatus attribute)
started_at (sneakpeek.lib.models.ScraperJob attribute)
status (sneakpeek.lib.models.ScraperJob attribute)
stop() (sneakpeek.server.SneakpeekServer method)
SUCCEEDED (sneakpeek.lib.models.ScraperJobStatus attribute)
T
THROW (sneakpeek.plugins.rate_limiter_plugin.RateLimitedStrategy attribute)
(sneakpeek.plugins.robots_txt_plugin.RobotsTxtViolationStrategy attribute)
time_window (sneakpeek.plugins.rate_limiter_plugin.RateLimiterPluginConfig attribute)
U
UnknownScraperHandlerError
update_scraper() (sneakpeek.lib.storage.base.ScrapersStorage method)
(sneakpeek.lib.storage.in_memory_storage.InMemoryScrapersStorage method)
(sneakpeek.lib.storage.redis_storage.RedisScrapersStorage method)
update_scraper_job() (sneakpeek.lib.storage.base.ScraperJobsStorage method)
(sneakpeek.lib.storage.in_memory_storage.InMemoryScraperJobsStorage method)
(sneakpeek.lib.storage.redis_storage.RedisScraperJobsStorage method)
use_external_data (sneakpeek.plugins.user_agent_injecter_plugin.UserAgentInjecterPluginConfig attribute)
UserAgentInjecterPlugin (class in sneakpeek.plugins.user_agent_injecter_plugin)
UserAgentInjecterPluginConfig (class in sneakpeek.plugins.user_agent_injecter_plugin)
UTMOST (sneakpeek.lib.models.ScraperJobPriority attribute)
W
WAIT (sneakpeek.plugins.rate_limiter_plugin.RateLimitedStrategy attribute)
Worker (class in sneakpeek.worker)
WorkerABC (class in sneakpeek.worker)