From 13911c7e4324d852c87b0910312debc481282e3a Mon Sep 17 00:00:00 2001 From: seemab-yamin Date: Tue, 3 Feb 2026 10:12:00 +0500 Subject: [PATCH 1/3] fix urlopener/urlread/urlopen/urlread/urlcheck/urlretrieve: add missing headers param propagation --- fastcore/net.py | 14 +++++++------- nbs/03b_net.ipynb | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/fastcore/net.py b/fastcore/net.py index 90ebb184..497642e8 100644 --- a/fastcore/net.py +++ b/fastcore/net.py @@ -74,9 +74,9 @@ class HTTP5xxServerError(HTTPError): pass # %% ../nbs/03b_net.ipynb #128b5f4a -def urlopener(): +def urlopener(headers=url_default_headers): _opener = urllib.request.build_opener() - _opener.addheaders = list(url_default_headers.items()) + _opener.addheaders = list(headers.items()) return _opener # %% ../nbs/03b_net.ipynb #e0470139 @@ -102,19 +102,19 @@ def _init(self, url, hdrs, fp, msg=msg, code=code): HTTP4xxClientError.__init__( _all_ = ['HTTP400BadRequestError', 'HTTP401UnauthorizedError', 'HTTP402PaymentRequiredError', 'HTTP403ForbiddenError', 'HTTP404NotFoundError', 'HTTP405MethodNotAllowedError', 'HTTP406NotAcceptableError', 'HTTP407ProxyAuthRequiredError', 'HTTP408RequestTimeoutError', 'HTTP409ConflictError', 'HTTP410GoneError', 'HTTP411LengthRequiredError', 'HTTP412PreconditionFailedError', 'HTTP413PayloadTooLargeError', 'HTTP414URITooLongError', 'HTTP415UnsupportedMediaTypeError', 'HTTP416RangeNotSatisfiableError', 'HTTP417ExpectationFailedError', 'HTTP418AmAteapotError', 'HTTP421MisdirectedRequestError', 'HTTP422UnprocessableEntityError', 'HTTP423LockedError', 'HTTP424FailedDependencyError', 'HTTP425TooEarlyError', 'HTTP426UpgradeRequiredError', 'HTTP428PreconditionRequiredError', 'HTTP429TooManyRequestsError', 'HTTP431HeaderFieldsTooLargeError', 'HTTP451LegalReasonsError'] # %% ../nbs/03b_net.ipynb #0d72881c -def urlopen(url, data=None, headers=None, timeout=None, **kwargs): +def urlopen(url, data=None, headers=url_default_headers, timeout=None, **kwargs): "Like `urllib.request.urlopen`, but first `urlwrap` the `url`, and encode `data`" if kwargs and not data: data=kwargs if data is not None: if not isinstance(data, (str,bytes)): data = urlencode(data) if not isinstance(data, bytes): data = data.encode('ascii') - try: return urlopener().open(urlwrap(url, data=data, headers=headers), timeout=timeout) + try: return urlopener(headers=headers).open(urlwrap(url, data=data, headers=headers), timeout=timeout) except HTTPError as e: e.msg += f"\n====Error Body====\n{e.read().decode(errors='ignore')}" raise # %% ../nbs/03b_net.ipynb #fa6a4dfe -def urlread(url, data=None, headers=None, decode=True, return_json=False, return_headers=False, timeout=None, **kwargs): +def urlread(url, data=None, headers=url_default_headers, decode=True, return_json=False, return_headers=False, timeout=None, **kwargs): "Retrieve `url`, using `data` dict or `kwargs` to `POST` if present" try: with urlopen(url, data=data, headers=headers, timeout=timeout, **kwargs) as u: res,hdrs = u.read(),u.headers @@ -133,7 +133,7 @@ def urljson(url, data=None, headers=None, timeout=None): return json.loads(res) if res else {} # %% ../nbs/03b_net.ipynb #cfe4aafd -def urlcheck(url, headers=None, timeout=10): +def urlcheck(url, headers=url_default_headers, timeout=10): if not url: return True try: with urlopen(url, headers=headers, timeout=timeout) as u: return u.status<400 @@ -147,7 +147,7 @@ def urlclean(url): return urlunparse(urlparse(str(url))[:3]+('','','')) # %% ../nbs/03b_net.ipynb #0a5565b3 -def urlretrieve(url, filename=None, reporthook=None, data=None, headers=None, timeout=None): +def urlretrieve(url, filename=None, reporthook=None, data=None, headers=url_default_headers, timeout=None): "Same as `urllib.request.urlretrieve` but also works with `Request` objects" with contextlib.closing(urlopen(url, data, headers=headers, timeout=timeout)) as fp: headers = fp.info() diff --git a/nbs/03b_net.ipynb b/nbs/03b_net.ipynb index 6595404a..87c1636f 100644 --- a/nbs/03b_net.ipynb +++ b/nbs/03b_net.ipynb @@ -270,9 +270,9 @@ "outputs": [], "source": [ "#| export\n", - "def urlopener():\n", + "def urlopener(headers=url_default_headers):\n", " _opener = urllib.request.build_opener()\n", - " _opener.addheaders = list(url_default_headers.items())\n", + " _opener.addheaders = list(headers.items())\n", " return _opener" ] }, @@ -322,13 +322,13 @@ "outputs": [], "source": [ "#| export\n", - "def urlopen(url, data=None, headers=None, timeout=None, **kwargs):\n", + "def urlopen(url, data=None, headers=url_default_headers, timeout=None, **kwargs):\n", " \"Like `urllib.request.urlopen`, but first `urlwrap` the `url`, and encode `data`\"\n", " if kwargs and not data: data=kwargs\n", " if data is not None:\n", " if not isinstance(data, (str,bytes)): data = urlencode(data)\n", " if not isinstance(data, bytes): data = data.encode('ascii')\n", - " try: return urlopener().open(urlwrap(url, data=data, headers=headers), timeout=timeout)\n", + " try: return urlopener(headers=headers).open(urlwrap(url, data=data, headers=headers), timeout=timeout)\n", " except HTTPError as e: \n", " e.msg += f\"\\n====Error Body====\\n{e.read().decode(errors='ignore')}\"\n", " raise" @@ -377,7 +377,7 @@ "outputs": [], "source": [ "#| export\n", - "def urlread(url, data=None, headers=None, decode=True, return_json=False, return_headers=False, timeout=None, **kwargs):\n", + "def urlread(url, data=None, headers=url_default_headers, decode=True, return_json=False, return_headers=False, timeout=None, **kwargs):\n", " \"Retrieve `url`, using `data` dict or `kwargs` to `POST` if present\"\n", " try:\n", " with urlopen(url, data=data, headers=headers, timeout=timeout, **kwargs) as u: res,hdrs = u.read(),u.headers\n", @@ -423,7 +423,7 @@ "outputs": [], "source": [ "#| export\n", - "def urlcheck(url, headers=None, timeout=10):\n", + "def urlcheck(url, headers=url_default_headers, timeout=10):\n", " if not url: return True\n", " try:\n", " with urlopen(url, headers=headers, timeout=timeout) as u: return u.status<400\n", @@ -463,7 +463,7 @@ "outputs": [], "source": [ "#| export\n", - "def urlretrieve(url, filename=None, reporthook=None, data=None, headers=None, timeout=None):\n", + "def urlretrieve(url, filename=None, reporthook=None, data=None, headers=url_default_headers, timeout=None):\n", " \"Same as `urllib.request.urlretrieve` but also works with `Request` objects\"\n", " with contextlib.closing(urlopen(url, data, headers=headers, timeout=timeout)) as fp:\n", " headers = fp.info()\n", From 6e56f2bd586c3cc351e43beae8e848ea5fa0fefc Mon Sep 17 00:00:00 2001 From: seemab-yamin Date: Tue, 3 Feb 2026 10:32:11 +0500 Subject: [PATCH 2/3] fix urlopener/urlread/urlopen/urlread/urlcheck/urlretrieve: add missing headers param propagation --- fastcore/net.py | 12 ++++++------ nbs/03b_net.ipynb | 18 +++++++++++------- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/fastcore/net.py b/fastcore/net.py index 497642e8..05178376 100644 --- a/fastcore/net.py +++ b/fastcore/net.py @@ -74,9 +74,9 @@ class HTTP5xxServerError(HTTPError): pass # %% ../nbs/03b_net.ipynb #128b5f4a -def urlopener(headers=url_default_headers): +def urlopener(headers=None): _opener = urllib.request.build_opener() - _opener.addheaders = list(headers.items()) + if headers: _opener.addheaders = list(headers.items()) return _opener # %% ../nbs/03b_net.ipynb #e0470139 @@ -102,7 +102,7 @@ def _init(self, url, hdrs, fp, msg=msg, code=code): HTTP4xxClientError.__init__( _all_ = ['HTTP400BadRequestError', 'HTTP401UnauthorizedError', 'HTTP402PaymentRequiredError', 'HTTP403ForbiddenError', 'HTTP404NotFoundError', 'HTTP405MethodNotAllowedError', 'HTTP406NotAcceptableError', 'HTTP407ProxyAuthRequiredError', 'HTTP408RequestTimeoutError', 'HTTP409ConflictError', 'HTTP410GoneError', 'HTTP411LengthRequiredError', 'HTTP412PreconditionFailedError', 'HTTP413PayloadTooLargeError', 'HTTP414URITooLongError', 'HTTP415UnsupportedMediaTypeError', 'HTTP416RangeNotSatisfiableError', 'HTTP417ExpectationFailedError', 'HTTP418AmAteapotError', 'HTTP421MisdirectedRequestError', 'HTTP422UnprocessableEntityError', 'HTTP423LockedError', 'HTTP424FailedDependencyError', 'HTTP425TooEarlyError', 'HTTP426UpgradeRequiredError', 'HTTP428PreconditionRequiredError', 'HTTP429TooManyRequestsError', 'HTTP431HeaderFieldsTooLargeError', 'HTTP451LegalReasonsError'] # %% ../nbs/03b_net.ipynb #0d72881c -def urlopen(url, data=None, headers=url_default_headers, timeout=None, **kwargs): +def urlopen(url, data=None, headers=None, timeout=None, **kwargs): "Like `urllib.request.urlopen`, but first `urlwrap` the `url`, and encode `data`" if kwargs and not data: data=kwargs if data is not None: @@ -114,7 +114,7 @@ def urlopen(url, data=None, headers=url_default_headers, timeout=None, **kwargs) raise # %% ../nbs/03b_net.ipynb #fa6a4dfe -def urlread(url, data=None, headers=url_default_headers, decode=True, return_json=False, return_headers=False, timeout=None, **kwargs): +def urlread(url, data=None, headers=None, decode=True, return_json=False, return_headers=False, timeout=None, **kwargs): "Retrieve `url`, using `data` dict or `kwargs` to `POST` if present" try: with urlopen(url, data=data, headers=headers, timeout=timeout, **kwargs) as u: res,hdrs = u.read(),u.headers @@ -133,7 +133,7 @@ def urljson(url, data=None, headers=None, timeout=None): return json.loads(res) if res else {} # %% ../nbs/03b_net.ipynb #cfe4aafd -def urlcheck(url, headers=url_default_headers, timeout=10): +def urlcheck(url, headers=None, timeout=10): if not url: return True try: with urlopen(url, headers=headers, timeout=timeout) as u: return u.status<400 @@ -147,7 +147,7 @@ def urlclean(url): return urlunparse(urlparse(str(url))[:3]+('','','')) # %% ../nbs/03b_net.ipynb #0a5565b3 -def urlretrieve(url, filename=None, reporthook=None, data=None, headers=url_default_headers, timeout=None): +def urlretrieve(url, filename=None, reporthook=None, data=None, headers=None, timeout=None): "Same as `urllib.request.urlretrieve` but also works with `Request` objects" with contextlib.closing(urlopen(url, data, headers=headers, timeout=timeout)) as fp: headers = fp.info() diff --git a/nbs/03b_net.ipynb b/nbs/03b_net.ipynb index 87c1636f..81c844fb 100644 --- a/nbs/03b_net.ipynb +++ b/nbs/03b_net.ipynb @@ -270,9 +270,9 @@ "outputs": [], "source": [ "#| export\n", - "def urlopener(headers=url_default_headers):\n", + "def urlopener(headers=None):\n", " _opener = urllib.request.build_opener()\n", - " _opener.addheaders = list(headers.items())\n", + " if headers: _opener.addheaders = list(headers.items())\n", " return _opener" ] }, @@ -322,7 +322,7 @@ "outputs": [], "source": [ "#| export\n", - "def urlopen(url, data=None, headers=url_default_headers, timeout=None, **kwargs):\n", + "def urlopen(url, data=None, headers=None, timeout=None, **kwargs):\n", " \"Like `urllib.request.urlopen`, but first `urlwrap` the `url`, and encode `data`\"\n", " if kwargs and not data: data=kwargs\n", " if data is not None:\n", @@ -377,7 +377,7 @@ "outputs": [], "source": [ "#| export\n", - "def urlread(url, data=None, headers=url_default_headers, decode=True, return_json=False, return_headers=False, timeout=None, **kwargs):\n", + "def urlread(url, data=None, headers=None, decode=True, return_json=False, return_headers=False, timeout=None, **kwargs):\n", " \"Retrieve `url`, using `data` dict or `kwargs` to `POST` if present\"\n", " try:\n", " with urlopen(url, data=data, headers=headers, timeout=timeout, **kwargs) as u: res,hdrs = u.read(),u.headers\n", @@ -423,7 +423,7 @@ "outputs": [], "source": [ "#| export\n", - "def urlcheck(url, headers=url_default_headers, timeout=10):\n", + "def urlcheck(url, headers=None, timeout=10):\n", " if not url: return True\n", " try:\n", " with urlopen(url, headers=headers, timeout=timeout) as u: return u.status<400\n", @@ -463,7 +463,7 @@ "outputs": [], "source": [ "#| export\n", - "def urlretrieve(url, filename=None, reporthook=None, data=None, headers=url_default_headers, timeout=None):\n", + "def urlretrieve(url, filename=None, reporthook=None, data=None, headers=None, timeout=None):\n", " \"Same as `urllib.request.urlretrieve` but also works with `Request` objects\"\n", " with contextlib.closing(urlopen(url, data, headers=headers, timeout=timeout)) as fp:\n", " headers = fp.info()\n", @@ -850,9 +850,13 @@ "split_at_heading": true }, "kernelspec": { - "display_name": "python3", + "display_name": "fastcore-dev", "language": "python", "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12.3" } }, "nbformat": 4, From 83c8aeac9800e0171e78c62ed7dbccbf5ff67f77 Mon Sep 17 00:00:00 2001 From: seemab-yamin Date: Sat, 7 Feb 2026 11:39:27 +0500 Subject: [PATCH 3/3] fix urlopener/urlread/urlopen/urlread/urlcheck/urlretrieve: add missing headers param propagation\nValidation Cells Added --- nbs/03b_net.ipynb | 75 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/nbs/03b_net.ipynb b/nbs/03b_net.ipynb index 81c844fb..7b6ee900 100644 --- a/nbs/03b_net.ipynb +++ b/nbs/03b_net.ipynb @@ -276,6 +276,81 @@ " return _opener" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "861c12fc", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\n", + " \"headers\": {\n", + " \"Accept\": \"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9\", \n", + " \"Accept-Encoding\": \"identity\", \n", + " \"Accept-Language\": \"en-US,en;q=0.9\", \n", + " \"Cache-Control\": \"max-age=0\", \n", + " \"Host\": \"httpbin.org\", \n", + " \"Sec-Fetch-Dest\": \"document\", \n", + " \"Sec-Fetch-Mode\": \"navigate\", \n", + " \"Sec-Fetch-Site\": \"none\", \n", + " \"Sec-Fetch-User\": \"?1\", \n", + " \"Upgrade-Insecure-Requests\": \"1\", \n", + " \"User-Agent\": \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36\", \n", + " \"X-Amzn-Trace-Id\": \"Root=1-6986dd0b-0561cf30030f998a5282e86d\", \n", + " \"X-Custom-Header\": \"should-appear\"\n", + " }\n", + "}\n", + "\n" + ] + } + ], + "source": [ + "# Validate if the custom headers are applied to requests\n", + "from fastcore.net import urlread\n", + "\n", + "url = \"https://httpbin.org/headers\"\n", + "headers = {\"X-Custom-Header\": \"should-appear\"}\n", + "\n", + "resp = urlread(url, headers=headers)\n", + "print(resp) # X-Custom-Header missing from response (only defaults applied)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "55d23fbf", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\n", + " \"headers\": {\n", + " \"Accept-Encoding\": \"identity\", \n", + " \"Host\": \"httpbin.org\", \n", + " \"X-Amzn-Trace-Id\": \"Root=1-6986dd24-658093293c3ca5741a3ae92c\", \n", + " \"X-Custom-Header\": \"should-appear\"\n", + " }\n", + "}\n", + "\n" + ] + } + ], + "source": [ + "# Validate if the custom headers are applied to requests\n", + "from fastcore.net import urlread\n", + "\n", + "url = \"https://httpbin.org/headers\"\n", + "headers = {\"X-Custom-Header\": \"should-appear\"}\n", + "\n", + "resp = urlread(url, headers=headers)\n", + "print(resp) # X-Custom-Header missing from response (only defaults applied)" + ] + }, { "cell_type": "code", "execution_count": null,