diff --git a/CHANGES b/CHANGES index c3823d65..f4e790b4 100644 --- a/CHANGES +++ b/CHANGES @@ -13,6 +13,8 @@ Unreleased include the ``params`` and ``req_kwargs`` attributes added by ``responses``. * Fixed non-strict `json_params_matcher` raising `TypeError` when an actual nested object is compared with an expected scalar value. +* Fixed a generator/iterator request body not being consumed, leaving it + sitting unread even after a successful mocked call. See #713 0.26.3 ------ diff --git a/responses/__init__.py b/responses/__init__.py index 5b1546a3..a1de2341 100644 --- a/responses/__init__.py +++ b/responses/__init__.py @@ -1102,7 +1102,7 @@ def _parse_request_params(self, url: str) -> RequestParams: return params def _read_filelike_body( - self, body: Union[str, bytes, BufferedReader, None] + self, body: Union[str, bytes, BufferedReader, Iterable[Union[str, bytes]], None] ) -> Union[str, bytes, None]: # Requests/urllib support multiple types of body, including file-like objects. # Read from the file if it's a file-like object to avoid storing a closed file @@ -1115,6 +1115,11 @@ def _read_filelike_body( # https://github.com/urllib3/urllib3/blob/abbfbcb1dd274fc54b4f0a7785fd04d59b634195/src/urllib3/util/request.py#L220 if hasattr(body, "read") or isinstance(body, BufferedReader): return body.read() + # Drain a streamed iterator/generator body too, same as a real send would. GH #713. + if isinstance(body, Iterable): + # urllib3 encodes a str chunk as utf-8 before writing it, same as any other chunk. + chunks = (chunk.encode("utf-8") if isinstance(chunk, str) else chunk for chunk in body) + return b"".join(chunks) return body def _on_request( diff --git a/responses/tests/test_responses.py b/responses/tests/test_responses.py index bc8f1e04..37e02822 100644 --- a/responses/tests/test_responses.py +++ b/responses/tests/test_responses.py @@ -2874,3 +2874,48 @@ def run(): run() assert_reset() + + +def test_generator_body_in_request(): + """Validate that a generator/iterator body is fully consumed, the same way a real + network send would read it, instead of being stored unconsumed. GH #713. + """ + + @responses.activate + def run(): + responses.add(responses.POST, "https://example.com") + + def gen(): + yield b"te" + yield b"st" + + generator = gen() + requests.post("https://example.com", data=generator) + + assert list(generator) == [] + assert len(responses.calls) == 1 + assert responses.calls[0].request.body == b"test" + + run() + assert_reset() + + +def test_str_generator_body_in_request(): + """A generator yielding str chunks is also valid for requests/urllib3 - make sure + draining it doesn't raise, and chunks are encoded the same way urllib3 would. GH #713. + """ + + @responses.activate + def run(): + responses.add(responses.POST, "https://example.com") + + def gen(): + yield "te" + yield "st" + + requests.post("https://example.com", data=gen()) + + assert responses.calls[0].request.body == b"test" + + run() + assert_reset()