from __future__ import annotations
from typing import TYPE_CHECKING
from w3lib.url import parse_data_uri
from scrapy.core.downloader.handlers.base import BaseDownloadHandler
from scrapy.http import Response, TextResponse
from scrapy.responsetypes import responsetypes
if TYPE_CHECKING:
from scrapy import Request
[docs]
class DataURIDownloadHandler(BaseDownloadHandler):
async def download_request(self, request: Request) -> Response:
uri = parse_data_uri(request.url)
respcls = responsetypes.from_mimetype(uri.media_type)
if issubclass(respcls, TextResponse) and uri.media_type.split("/")[0] == "text":
charset = uri.media_type_parameters.get("charset")
return respcls(url=request.url, body=uri.data, encoding=charset)
return respcls(url=request.url, body=uri.data)