From 0acf3514a1d5c9ca669a81c86d69d5102e44cfbe Mon Sep 17 00:00:00 2001 From: Ray Gardner Date: Mon, 22 Feb 2021 16:25:00 -0700 Subject: [PATCH 1/4] Modify GifEncode.c and Gif.h to use LZW encoding --- src/libImaging/Gif.h | 41 ++- src/libImaging/GifEncode.c | 542 +++++++++++++++++++------------------ 2 files changed, 303 insertions(+), 280 deletions(-) diff --git a/src/libImaging/Gif.h b/src/libImaging/Gif.h index a85ce2b6e3c..4029bbfe5f1 100644 --- a/src/libImaging/Gif.h +++ b/src/libImaging/Gif.h @@ -9,10 +9,10 @@ /* Max size for a LZW code word. */ -#define GIFBITS 12 +#define GIFBITS 12 -#define GIFTABLE (1 << GIFBITS) -#define GIFBUFFER (1 << GIFBITS) +#define GIFTABLE (1<block || context->block->size == 255) { - GIFENCODERBLOCK *block; +/* Values of entry_state */ +enum { LZW_INITIAL, LZW_TRY_IN1, LZW_TRY_IN2, LZW_TRY_OUT1, LZW_TRY_OUT2, + LZW_FINISHED }; - /* no room in the current block (or no current block); - allocate a new one */ +/* Values of control_state */ +enum { PUT_HEAD, PUT_INIT_CLEAR, PUT_CLEAR, PUT_LAST_HEAD, PUT_END }; - /* add current block to end of flush queue */ - if (context->block) { - block = context->flush; - while (block && block->next) { - block = block->next; +static void glzwe_reset(GIFENCODERSTATE *st) { + st->next_code = st->end_code + 1; + st->max_code = 2 * st->clear_code - 1; + st->code_width = st->bits + 1; + memset(st->codes, 0, sizeof(st->codes)); +} + +static void glzwe_init(GIFENCODERSTATE *st) { + st->clear_code = 1 << st->bits; + st->end_code = st->clear_code + 1; + glzwe_reset(st); + st->entry_state = LZW_INITIAL; + st->buf_bits_left = 8; + st->code_buffer = 0; +} + +static int glzwe(GIFENCODERSTATE *st, const UINT8 *in_ptr, UINT8 *out_ptr, + UINT32 *in_avail, UINT32 *out_avail, + UINT32 end_of_data) { + switch (st->entry_state) { + + case LZW_TRY_IN1: +get_first_byte: + if (!*in_avail) { + if (end_of_data) { + goto end_of_data; + } + st->entry_state = LZW_TRY_IN1; + return GLZW_NO_INPUT_AVAIL; + } + st->head = *in_ptr++; + (*in_avail)--; + + case LZW_TRY_IN2: +encode_loop: + if (!*in_avail) { + if (end_of_data) { + st->code = st->head; + st->put_state = PUT_LAST_HEAD; + goto put_code; } - if (block) { - block->next = context->block; + st->entry_state = LZW_TRY_IN2; + return GLZW_NO_INPUT_AVAIL; + } + st->tail = *in_ptr++; + (*in_avail)--; + + /* Knuth TAOCP vol 3 sec. 6.4 algorithm D. */ + /* Hash found experimentally to be pretty good. */ + /* This works ONLY with TABLE_SIZE a power of 2. */ + st->probe = ((st->head ^ (st->tail << 6)) * 31) & (TABLE_SIZE - 1); + while (st->codes[st->probe]) { + if ((st->codes[st->probe] & 0xFFFFF) == + ((st->head << 8) | st->tail)) { + st->head = st->codes[st->probe] >> 20; + goto encode_loop; } else { - context->flush = context->block; + /* Reprobe decrement must be nonzero and relatively prime to table + * size. So, any odd positive number for power-of-2 size. */ + if ((st->probe -= ((st->tail << 2) | 1)) < 0) { + st->probe += TABLE_SIZE; + } } } - - /* get a new block */ - if (context->free) { - block = context->free; - context->free = NULL; + /* Key not found, probe is at empty slot. */ + st->code = st->head; + st->put_state = PUT_HEAD; + goto put_code; +insert_code_or_clear: /* jump here after put_code */ + if (st->next_code < CODE_LIMIT) { + st->codes[st->probe] = (st->next_code << 20) | + (st->head << 8) | st->tail; + if (st->next_code > st->max_code) { + st->max_code = st->max_code * 2 + 1; + st->code_width++; + } + st->next_code++; } else { - /* malloc check ok, small constant allocation */ - block = malloc(sizeof(GIFENCODERBLOCK)); - if (!block) { - return 0; + st->code = st->clear_code; + st->put_state = PUT_CLEAR; + goto put_code; +reset_after_clear: /* jump here after put_code */ + glzwe_reset(st); + } + st->head = st->tail; + goto encode_loop; + + case LZW_INITIAL: + glzwe_reset(st); + st->code = st->clear_code; + st->put_state = PUT_INIT_CLEAR; +put_code: + st->code_bits_left = st->code_width; +check_buf_bits: + if (!st->buf_bits_left) { /* out buffer full */ + + case LZW_TRY_OUT1: + if (!*out_avail) { + st->entry_state = LZW_TRY_OUT1; + return GLZW_NO_OUTPUT_AVAIL; } + *out_ptr++ = st->code_buffer; + (*out_avail)--; + st->code_buffer = 0; + st->buf_bits_left = 8; + } + /* code bits to pack */ + UINT32 n = st->buf_bits_left < st->code_bits_left + ? st->buf_bits_left : st->code_bits_left; + st->code_buffer |= + (st->code & ((1 << n) - 1)) << (8 - st->buf_bits_left); + st->code >>= n; + st->buf_bits_left -= n; + st->code_bits_left -= n; + if (st->code_bits_left) + goto check_buf_bits; + switch (st->put_state) { + case PUT_INIT_CLEAR: + goto get_first_byte; + case PUT_HEAD: + goto insert_code_or_clear; + case PUT_CLEAR: + goto reset_after_clear; + case PUT_LAST_HEAD: + goto end_of_data; + case PUT_END: + goto flush_code_buffer; + default: + return GLZW_INTERNAL_ERROR; } - block->size = 0; - block->next = NULL; - - context->block = block; - } - - /* write new byte to block */ - context->block->data[context->block->size++] = byte; - - return 1; -} +end_of_data: + st->code = st->end_code; + st->put_state = PUT_END; + goto put_code; +flush_code_buffer: /* jump here after put_code */ + if (st->buf_bits_left < 8) { + + case LZW_TRY_OUT2: + if (!*out_avail) { + st->entry_state = LZW_TRY_OUT2; + return GLZW_NO_OUTPUT_AVAIL; + } + *out_ptr++ = st->code_buffer; + (*out_avail)--; + } + st->entry_state = LZW_FINISHED; + return GLZW_OK; -/* write a code word to the current block. this is a macro to make - sure it's inlined on all platforms */ - -#define EMIT(code) \ - { \ - context->bitbuffer |= ((INT32)(code)) << context->bitcount; \ - context->bitcount += 9; \ - while (context->bitcount >= 8) { \ - if (!emit(context, (UINT8)context->bitbuffer)) { \ - state->errcode = IMAGING_CODEC_MEMORY; \ - return 0; \ - } \ - context->bitbuffer >>= 8; \ - context->bitcount -= 8; \ - } \ - } + case LZW_FINISHED: + return GLZW_OK; -/* write a run. we use a combination of literals and combinations of - literals. this can give quite decent compression for images with - long stretches of identical pixels. but remember: if you want - really good compression, use another file format. */ - -#define EMIT_RUN(label) \ - { \ - label: \ - while (context->count > 0) { \ - int run = 2; \ - EMIT(context->last); \ - context->count--; \ - if (state->count++ == LAST_CODE) { \ - EMIT(CLEAR_CODE); \ - state->count = FIRST_CODE; \ - goto label; \ - } \ - while (context->count >= run) { \ - EMIT(state->count - 1); \ - context->count -= run; \ - run++; \ - if (state->count++ == LAST_CODE) { \ - EMIT(CLEAR_CODE); \ - state->count = FIRST_CODE; \ - goto label; \ - } \ - } \ - if (context->count > 1) { \ - EMIT(state->count - 1 - (run - context->count)); \ - context->count = 0; \ - if (state->count++ == LAST_CODE) { \ - EMIT(CLEAR_CODE); \ - state->count = FIRST_CODE; \ - } \ - break; \ - } \ - } \ + default: + return GLZW_INTERNAL_ERROR; } +} +/* -END- GIF LZW encoder. */ int -ImagingGifEncode(Imaging im, ImagingCodecState state, UINT8 *buf, int bytes) { - UINT8 *ptr; - int this; +ImagingGifEncode(Imaging im, ImagingCodecState state, UINT8* buf, int bytes) { + UINT8* ptr; + UINT8* sub_block_ptr; + UINT8* sub_block_limit; + UINT8* buf_limit; + GIFENCODERSTATE *context = (GIFENCODERSTATE*) state->context; + int r; - GIFENCODERBLOCK *block; - GIFENCODERSTATE *context = (GIFENCODERSTATE *)state->context; + UINT32 in_avail, in_used; + UINT32 out_avail, out_used; - if (!state->state) { - /* place a clear code in the output buffer */ - context->bitbuffer = CLEAR_CODE; - context->bitcount = 9; - - state->count = FIRST_CODE; + if (state->state == INIT) { + state->state = ENCODE; + glzwe_init(context); if (context->interlace) { context->interlace = 1; @@ -159,166 +223,132 @@ ImagingGifEncode(Imaging im, ImagingCodecState state, UINT8 *buf, int bytes) { context->step = 1; } - context->last = -1; - + /* Need at least 2 bytes for data sub-block; 5 for empty image */ + if (bytes < 5) { + state->errcode = IMAGING_CODEC_CONFIG; + return 0; + } /* sanity check */ if (state->xsize <= 0 || state->ysize <= 0) { - state->state = ENCODE_EOF; + /* Is this better than an error return? */ + /* This will handle any legal "LZW Minimum Code Size" */ + memset(buf, 0, 5); + in_avail = 0; + out_avail = 5; + r = glzwe(context, (const UINT8 *)"", buf + 1, &in_avail, &out_avail, 1); + if (r == GLZW_OK) { + r = 5 - out_avail; + if (r < 1 || r > 3) { + state->errcode = IMAGING_CODEC_BROKEN; + return 0; + } + buf[0] = r; + state->errcode = IMAGING_CODEC_END; + return r + 2; + } else { + /* Should not be possible unless something external to this + * routine messes with our state data */ + state->errcode = IMAGING_CODEC_BROKEN; + return 0; + } } + /* Init state->x to make if() below true the first time through. */ + state->x = state->xsize; } - ptr = buf; + buf_limit = buf + bytes; + sub_block_limit = sub_block_ptr = ptr = buf; + /* On entry, buf is output buffer, bytes is space available in buf. + * Loop here getting input until buf is full or image is all encoded. */ for (;;) { - switch (state->state) { - case INIT: - case ENCODE: + /* Set up sub-block ptr and limit. sub_block_ptr stays at beginning + * of sub-block until it is full. ptr will advance when any data is + * placed in buf. + */ + if (ptr >= sub_block_limit) { + if (buf_limit - ptr < 2) { /* Need at least 2 for data sub-block */ + return ptr - buf; + } + sub_block_ptr = ptr; + sub_block_limit = sub_block_ptr + + (256 < buf_limit - sub_block_ptr ? + 256 : buf_limit - sub_block_ptr); + *ptr++ = 0; + } - /* identify and store a run of pixels */ + /* Get next row of pixels. */ + /* This if() originally tested state->x==0 for the first time through. + * This no longer works, as the loop will not advance state->x if + * glzwe() does not consume any input; this would advance the row + * spuriously. Now pre-init state->x above for first time, and avoid + * entering if() when state->state is FINISH, or it will loop + * infinitely. + */ + if (state->x >= state->xsize && state->state == ENCODE) { + if (!context->interlace && state->y >= state->ysize) { + state->state = FINISH; + continue; + } - if (state->x == 0 || state->x >= state->xsize) { - if (!context->interlace && state->y >= state->ysize) { - state->state = ENCODE_EOF; + /* get another line of data */ + state->shuffle( + state->buffer, + (UINT8*) im->image[state->y + state->yoff] + + state->xoff * im->pixelsize, state->xsize + ); + state->x = 0; + + /* step forward, according to the interlace settings */ + state->y += context->step; + while (context->interlace && state->y >= state->ysize) { + switch (context->interlace) { + case 1: + state->y = 4; + context->interlace = 2; break; - } - - if (context->flush) { - state->state = FLUSH; + case 2: + context->step = 4; + state->y = 2; + context->interlace = 3; break; - } - - /* get another line of data */ - state->shuffle( - state->buffer, - (UINT8 *)im->image[state->y + state->yoff] + - state->xoff * im->pixelsize, - state->xsize); - - state->x = 0; - - if (state->state == INIT) { - /* preload the run-length buffer and get going */ - context->last = state->buffer[0]; - context->count = state->x = 1; - state->state = ENCODE; - } - - /* step forward, according to the interlace settings */ - state->y += context->step; - while (context->interlace && state->y >= state->ysize) - switch (context->interlace) { - case 1: - state->y = 4; - context->interlace = 2; - break; - case 2: - context->step = 4; - state->y = 2; - context->interlace = 3; - break; - case 3: - context->step = 2; - state->y = 1; - context->interlace = 0; - break; - default: - /* just make sure we don't loop forever */ - context->interlace = 0; - } - } - /* Potential special case for xsize==1 */ - if (state->x < state->xsize) { - this = state->buffer[state->x++]; - } else { - EMIT_RUN(label0); - break; - } - - if (this == context->last) { - context->count++; - } else { - EMIT_RUN(label1); - context->last = this; - context->count = 1; - } - break; - - case ENCODE_EOF: - - /* write the final run */ - EMIT_RUN(label2); - - /* write an end of image marker */ - EMIT(EOF_CODE); - - /* empty the bit buffer */ - while (context->bitcount > 0) { - if (!emit(context, (UINT8)context->bitbuffer)) { - state->errcode = IMAGING_CODEC_MEMORY; - return 0; - } - context->bitbuffer >>= 8; - context->bitcount -= 8; - } - - /* flush the last block, and exit */ - if (context->block) { - GIFENCODERBLOCK *block; - block = context->flush; - while (block && block->next) { - block = block->next; - } - if (block) { - block->next = context->block; - } else { - context->flush = context->block; - } - context->block = NULL; - } - - state->state = EXIT; - - /* fall through... */ - - case EXIT: - case FLUSH: - - while (context->flush) { - /* get a block from the flush queue */ - block = context->flush; - - if (block->size > 0) { - /* make sure it fits into the output buffer */ - if (bytes < block->size + 1) { - return ptr - buf; - } - - ptr[0] = block->size; - memcpy(ptr + 1, block->data, block->size); - - ptr += block->size + 1; - bytes -= block->size + 1; - } - - context->flush = block->next; - - if (context->free) { - free(context->free); - } - context->free = block; - } - - if (state->state == EXIT) { - /* this was the last block! */ - if (context->free) { - free(context->free); - } - state->errcode = IMAGING_CODEC_END; - return ptr - buf; + case 3: + context->step = 2; + state->y = 1; + context->interlace = 0; + break; + default: + /* just make sure we don't loop forever */ + context->interlace = 0; } + } + } - state->state = ENCODE; - break; + in_avail = state->xsize - state->x; /* bytes left in line */ + out_avail = sub_block_limit - ptr; /* bytes left in sub-block */ + r = glzwe(context, &state->buffer[state->x], ptr, &in_avail, + &out_avail, state->state == FINISH); + out_used = sub_block_limit - ptr - out_avail; + *sub_block_ptr += out_used; + ptr += out_used; + in_used = state->xsize - state->x - in_avail; + state->x += in_used; + + if (r == GLZW_OK) { + /* Should not be possible when end-of-data flag is false. */ + state->errcode = IMAGING_CODEC_END; + return ptr - buf; + } else if (r == GLZW_NO_INPUT_AVAIL) { + /* Used all the input line; get another line */ + continue; + } else if (r == GLZW_NO_OUTPUT_AVAIL) { + /* subblock is full */ + continue; + } else { + /* Should not be possible unless something external to this + * routine messes with our state data */ + state->errcode = IMAGING_CODEC_BROKEN; + return 0; } } } From 5390786c6546449d4c5ff8e8b8a6709f10016f20 Mon Sep 17 00:00:00 2001 From: Ray Gardner Date: Thu, 25 Mar 2021 16:46:55 -0600 Subject: [PATCH 2/4] Add braces to if -- conform with Pillow style. --- src/libImaging/GifEncode.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libImaging/GifEncode.c b/src/libImaging/GifEncode.c index 8dec30f4046..0ac9dc60517 100644 --- a/src/libImaging/GifEncode.c +++ b/src/libImaging/GifEncode.c @@ -156,8 +156,9 @@ static int glzwe(GIFENCODERSTATE *st, const UINT8 *in_ptr, UINT8 *out_ptr, st->code >>= n; st->buf_bits_left -= n; st->code_bits_left -= n; - if (st->code_bits_left) + if (st->code_bits_left) { goto check_buf_bits; + } switch (st->put_state) { case PUT_INIT_CLEAR: goto get_first_byte; From 306d030fce71ec27cd1c561ed64ef763ecec19be Mon Sep 17 00:00:00 2001 From: Ray Gardner Date: Tue, 30 Mar 2021 17:35:21 -0600 Subject: [PATCH 3/4] Add comment to reference GIF LZW specification. --- src/libImaging/GifEncode.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/libImaging/GifEncode.c b/src/libImaging/GifEncode.c index 0ac9dc60517..f232454052a 100644 --- a/src/libImaging/GifEncode.c +++ b/src/libImaging/GifEncode.c @@ -27,6 +27,12 @@ enum { INIT, ENCODE, FINISH }; /* GIF LZW encoder by Raymond Gardner. */ /* Released here under PIL license. */ +/* This LZW encoder conforms to the GIF LZW format specified in the original + * Compuserve GIF 87a and GIF 89a specifications (see e.g. + * https://www.w3.org/Graphics/GIF/spec-gif87.txt Appendix C and + * https://www.w3.org/Graphics/GIF/spec-gif89a.txt Appendix F). + */ + /* Return values */ #define GLZW_OK 0 #define GLZW_NO_INPUT_AVAIL 1 From d6dfdd361704a03512d5ff6705df6158a5a57dc9 Mon Sep 17 00:00:00 2001 From: Ray Gardner Date: Wed, 31 Mar 2021 06:48:17 -0600 Subject: [PATCH 4/4] Add GIF LZW encoding to 8.2.0 release notes. --- docs/releasenotes/8.2.0.rst | 55 +++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/docs/releasenotes/8.2.0.rst b/docs/releasenotes/8.2.0.rst index d82bf45c22e..0ad5d7f2f81 100644 --- a/docs/releasenotes/8.2.0.rst +++ b/docs/releasenotes/8.2.0.rst @@ -10,6 +10,16 @@ Tk/Tcl 8.4 Support for Tk/Tcl 8.4 is deprecated and will be removed in Pillow 10.0.0 (2023-01-02), when Tk/Tcl 8.5 will be the minimum supported. +Categories +^^^^^^^^^^ + +``im.category`` is deprecated and will be removed in Pillow 10.0.0 (2023-01-02), +along with the related ``Image.NORMAL``, ``Image.SEQUENCE`` and +``Image.CONTAINER`` attributes. + +To determine if an image has multiple frames or not, +``getattr(im, "is_animated", False)`` can be used instead. + API Changes =========== @@ -35,6 +45,14 @@ This is now consistent with other IFDs, and must be accessed through These changes only affect :py:meth:`~PIL.Image.Image.getexif`, introduced in Pillow 6.0. The older ``_getexif()`` methods are unaffected. +Image._MODEINFO +^^^^^^^^^^^^^^^ + +This internal dictionary has been deprecated by a comment since PIL, and is now +removed. Instead, ``Image.getmodebase()``, ``Image.getmodetype()``, +``Image.getmodebandnames()``, ``Image.getmodebands()`` or ``ImageMode.getmode()`` +can be used. + API Additions ============= @@ -73,6 +91,15 @@ be specified through a keyword argument:: im.save("out.tif", icc_profile=...) + +ImageOps.autocontrast: preserve_tone +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The default behaviour of :py:meth:`~PIL.ImageOps.autocontrast` is to normalize +separate histograms for each color channel, changing the tone of the image. The new +``preserve_tone`` argument keeps the tone unchanged by using one luminance histogram +for all channels. + Security ======== @@ -81,6 +108,34 @@ TODO Other Changes ============= +GIF writer uses LZW encoding +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +GIF files are now written using LZW encoding, which will generate smaller files, +typically about 70% of the size generated by the older encoder. + +The pixel data is encoded using the format specified in the [Compuserve GIF +standard](https://www.w3.org/Graphics/GIF/spec-gif89a.txt). The older encoder +used a variant of run-length encoding that was compatible but less efficient. + +Libraqm and FriBiDi linking +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The way the libraqm dependency for complex text scripts is linked has been changed: + +Source builds will now link against the system version of libraqm at build time +rather than at runtime by default. + +Binary wheels now include a statically linked modified version of libraqm that +links against FriBiDi at runtime instead. This change is intended to address +issues with the previous implementation on some platforms. These are created +by building Pillow with the new build flags ``--vendor-raqm --vendor-fribidi``. + +Windows users will now need to install ``fribidi.dll`` (or ``fribidi-0.dll``) only, +``libraqm.dll`` is no longer used. + +See :doc:`installation documentation<../installation>` for more information. + PyQt6 ^^^^^