diff --git a/fastcore/net.py b/fastcore/net.py index 90ebb184..05178376 100644 --- a/fastcore/net.py +++ b/fastcore/net.py @@ -74,9 +74,9 @@ class HTTP5xxServerError(HTTPError): pass # %% ../nbs/03b_net.ipynb #128b5f4a -def urlopener(): +def urlopener(headers=None): _opener = urllib.request.build_opener() - _opener.addheaders = list(url_default_headers.items()) + if headers: _opener.addheaders = list(headers.items()) return _opener # %% ../nbs/03b_net.ipynb #e0470139 @@ -108,7 +108,7 @@ def urlopen(url, data=None, headers=None, timeout=None, **kwargs): if data is not None: if not isinstance(data, (str,bytes)): data = urlencode(data) if not isinstance(data, bytes): data = data.encode('ascii') - try: return urlopener().open(urlwrap(url, data=data, headers=headers), timeout=timeout) + try: return urlopener(headers=headers).open(urlwrap(url, data=data, headers=headers), timeout=timeout) except HTTPError as e: e.msg += f"\n====Error Body====\n{e.read().decode(errors='ignore')}" raise diff --git a/nbs/03b_net.ipynb b/nbs/03b_net.ipynb index 6595404a..7b6ee900 100644 --- a/nbs/03b_net.ipynb +++ b/nbs/03b_net.ipynb @@ -270,12 +270,87 @@ "outputs": [], "source": [ "#| export\n", - "def urlopener():\n", + "def urlopener(headers=None):\n", " _opener = urllib.request.build_opener()\n", - " _opener.addheaders = list(url_default_headers.items())\n", + " if headers: _opener.addheaders = list(headers.items())\n", " return _opener" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "861c12fc", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\n", + " \"headers\": {\n", + " \"Accept\": \"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9\", \n", + " \"Accept-Encoding\": \"identity\", \n", + " \"Accept-Language\": \"en-US,en;q=0.9\", \n", + " \"Cache-Control\": \"max-age=0\", \n", + " \"Host\": \"httpbin.org\", \n", + " \"Sec-Fetch-Dest\": \"document\", \n", + " \"Sec-Fetch-Mode\": \"navigate\", \n", + " \"Sec-Fetch-Site\": \"none\", \n", + " \"Sec-Fetch-User\": \"?1\", \n", + " \"Upgrade-Insecure-Requests\": \"1\", \n", + " \"User-Agent\": \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36\", \n", + " \"X-Amzn-Trace-Id\": \"Root=1-6986dd0b-0561cf30030f998a5282e86d\", \n", + " \"X-Custom-Header\": \"should-appear\"\n", + " }\n", + "}\n", + "\n" + ] + } + ], + "source": [ + "# Validate if the custom headers are applied to requests\n", + "from fastcore.net import urlread\n", + "\n", + "url = \"https://httpbin.org/headers\"\n", + "headers = {\"X-Custom-Header\": \"should-appear\"}\n", + "\n", + "resp = urlread(url, headers=headers)\n", + "print(resp) # X-Custom-Header missing from response (only defaults applied)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "55d23fbf", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\n", + " \"headers\": {\n", + " \"Accept-Encoding\": \"identity\", \n", + " \"Host\": \"httpbin.org\", \n", + " \"X-Amzn-Trace-Id\": \"Root=1-6986dd24-658093293c3ca5741a3ae92c\", \n", + " \"X-Custom-Header\": \"should-appear\"\n", + " }\n", + "}\n", + "\n" + ] + } + ], + "source": [ + "# Validate if the custom headers are applied to requests\n", + "from fastcore.net import urlread\n", + "\n", + "url = \"https://httpbin.org/headers\"\n", + "headers = {\"X-Custom-Header\": \"should-appear\"}\n", + "\n", + "resp = urlread(url, headers=headers)\n", + "print(resp) # X-Custom-Header missing from response (only defaults applied)" + ] + }, { "cell_type": "code", "execution_count": null, @@ -328,7 +403,7 @@ " if data is not None:\n", " if not isinstance(data, (str,bytes)): data = urlencode(data)\n", " if not isinstance(data, bytes): data = data.encode('ascii')\n", - " try: return urlopener().open(urlwrap(url, data=data, headers=headers), timeout=timeout)\n", + " try: return urlopener(headers=headers).open(urlwrap(url, data=data, headers=headers), timeout=timeout)\n", " except HTTPError as e: \n", " e.msg += f\"\\n====Error Body====\\n{e.read().decode(errors='ignore')}\"\n", " raise" @@ -850,9 +925,13 @@ "split_at_heading": true }, "kernelspec": { - "display_name": "python3", + "display_name": "fastcore-dev", "language": "python", "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12.3" } }, "nbformat": 4,