cmoncrawl.processor.pipeline.downloader.AsyncDownloader#

class cmoncrawl.processor.pipeline.downloader.AsyncDownloader(base_url: str = 'https://data.commoncrawl.org/', digest_verification: bool = True, max_retry: int = 5, sleep_step: int = 10, encoding: str = 'latin-1')#
__init__(base_url: str = 'https://data.commoncrawl.org/', digest_verification: bool = True, max_retry: int = 5, sleep_step: int = 10, encoding: str = 'latin-1')#

Methods

__init__([base_url, digest_verification, ...])

aclose(exc_type, exc_val[, exc_tb])

aopen()

download(domain_record)

unwrap(response, domain_record)