diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1fb79cb4..8e90e3fe 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -55,6 +55,11 @@ version 1.8.1-dev input buffer when ``max_length`` split the end of a stream. At the end of a stream, ``isal_zlib.Decompress.unconsumed_tail`` is empty, matching the CPython fix for gh-158169. ++ Fix ``igzip_lib.IgzipDecompressor.needs_input``, which could be ``True`` + after ``decompress()`` returned ``max_length`` bytes while more output was + still available. Readers that stop at the end of the file when + ``needs_input`` is ``True``, such as CPython's ``DecompressReader``, raised + ``EOFError`` for valid data. + Fix ``isal_zlib.decompressobj`` with a ``wbits`` of 40 to 47, which detects zlib or gzip data. It failed on gzip data when the first ``decompress()`` call got fewer than 2 bytes, for example, when the data arrived byte by diff --git a/src/isal/igzip_libmodule.c b/src/isal/igzip_libmodule.c index 0f5779f0..d2896bb3 100644 --- a/src/isal/igzip_libmodule.c +++ b/src/isal/igzip_libmodule.c @@ -260,7 +260,11 @@ decompress(IgzipDecompressor *self, uint8_t *data, size_t len, Py_ssize_t max_le } else if (self->avail_in_real == 0) { self->state.next_in = NULL; - self->needs_input = 1; + /* When the output is full, isal_inflate can still read all of the + input and keep the decompressed data in its own buffer. Ask for + more input only when the output was not full, like CPython's + LZMADecompressor does. */ + self->needs_input = (self->state.avail_out != 0); } else { self->needs_input = 0; diff --git a/tests/test_igzip_lib.py b/tests/test_igzip_lib.py index f2ccce1b..963d135f 100644 --- a/tests/test_igzip_lib.py +++ b/tests/test_igzip_lib.py @@ -284,6 +284,20 @@ def test_igzip_decompressor_unused_data_after_max_length(flag, wbits): assert decompressor.unused_data == compressed[stream_end:position] +@pytest.mark.parametrize("max_length", [100, 8192]) +def test_igzip_decompressor_needs_input_after_all_input_is_read(max_length): + # When the output is full, ISA-L can read all of the input and keep the + # decompressed data in its own buffer. needs_input must stay False until + # decompress() has returned that data. + compressed = igzip_lib.compress(DATA) + decompressor = IgzipDecompressor() + result = [decompressor.decompress(compressed, max_length)] + while not decompressor.eof: + assert not decompressor.needs_input + result.append(decompressor.decompress(b"", max_length)) + assert b"".join(result) == DATA + + @pytest.mark.skipif(not hasattr(sys, "getrefcount"), reason="Needs reference counting") def test_igzip_decompressor_keeps_zdict_reference():