Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions CHANGELOG.rst
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,11 @@ version 1.8.1-dev
input buffer when ``max_length`` split the end of a stream. At the end of a
stream, ``isal_zlib.Decompress.unconsumed_tail`` is empty, matching the
CPython fix for gh-158169.
+ Fix ``igzip_lib.IgzipDecompressor.needs_input``, which could be ``True``
after ``decompress()`` returned ``max_length`` bytes while more output was
still available. Readers that stop at the end of the file when
``needs_input`` is ``True``, such as CPython's ``DecompressReader``, raised
``EOFError`` for valid data.
+ Fix ``isal_zlib.decompressobj`` with a ``wbits`` of 40 to 47, which detects
zlib or gzip data. It failed on gzip data when the first ``decompress()``
call got fewer than 2 bytes, for example, when the data arrived byte by
Expand Down
6 changes: 5 additions & 1 deletion src/isal/igzip_libmodule.c
Original file line number Diff line number Diff line change
Expand Up @@ -260,7 +260,11 @@ decompress(IgzipDecompressor *self, uint8_t *data, size_t len, Py_ssize_t max_le
}
else if (self->avail_in_real == 0) {
self->state.next_in = NULL;
self->needs_input = 1;
/* When the output is full, isal_inflate can still read all of the
input and keep the decompressed data in its own buffer. Ask for
more input only when the output was not full, like CPython's
LZMADecompressor does. */
self->needs_input = (self->state.avail_out != 0);
}
else {
self->needs_input = 0;
Expand Down
14 changes: 14 additions & 0 deletions tests/test_igzip_lib.py
Original file line number Diff line number Diff line change
Expand Up @@ -284,6 +284,20 @@ def test_igzip_decompressor_unused_data_after_max_length(flag, wbits):
assert decompressor.unused_data == compressed[stream_end:position]


@pytest.mark.parametrize("max_length", [100, 8192])
def test_igzip_decompressor_needs_input_after_all_input_is_read(max_length):
# When the output is full, ISA-L can read all of the input and keep the
# decompressed data in its own buffer. needs_input must stay False until
# decompress() has returned that data.
compressed = igzip_lib.compress(DATA)
decompressor = IgzipDecompressor()
result = [decompressor.decompress(compressed, max_length)]
while not decompressor.eof:
assert not decompressor.needs_input
result.append(decompressor.decompress(b"", max_length))
assert b"".join(result) == DATA


@pytest.mark.skipif(not hasattr(sys, "getrefcount"),
reason="Needs reference counting")
def test_igzip_decompressor_keeps_zdict_reference():
Expand Down
Loading