Skip to content

Add support for reading TIFFs with PlanarConfiguration=2 #5178

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wiredfool wants to merge 12 commits into from
Closed

Add support for reading TIFFs with PlanarConfiguration=2 #5178

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
4bc667b
Add Tests and support for Planar Tiff Images
kkopachev May 23, 2020
a97e08a
Implementation for PlanarConfiguration=2 Tiffs, manually merged from …
wiredfool Dec 31, 2020
afb3869
initialize the unpackers
wiredfool Dec 31, 2020
457ee6e
Merge branch 'master' into 4641_rebase
radarhere Jan 4, 2021
4aa684f
correct TIFFTAG_PLANARCONFIG size
nulano Jan 9, 2021
8c24738
the previous commit also fixes these big-endian failures
nulano Jan 9, 2021
963ade0
remove double pointer
nulano Jan 9, 2021
2e4cbc7
Merge pull request #6 from nulano/4641_rebase
wiredfool Jan 10, 2021
0f2617a
Planes should be int, not uint
wiredfool Jan 10, 2021
6e57481
Use flag instead of recalculating
wiredfool Jan 10, 2021
3c6a3fa
not
wiredfool Jan 11, 2021
b30c41e
UINT8 -> int for plane
wiredfool Jan 11, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Binary file added Tests/images/tiff_strip_planar_16bit_RGB.tiff
View file
Open in desktop
Binary file not shown.
Binary file added Tests/images/tiff_strip_planar_16bit_RGBa.tiff
View file
Open in desktop
Binary file not shown.
Binary file added Tests/images/tiff_strip_planar_lzw.tiff
View file
Open in desktop
Binary file not shown.
Binary file added Tests/images/tiff_tiled_planar_16bit_RGB.tiff
View file
Open in desktop
Binary file not shown.
Binary file added Tests/images/tiff_tiled_planar_16bit_RGBa.tiff
View file
Open in desktop
Binary file not shown.
Binary file added Tests/images/tiff_tiled_planar_lzw.tiff
View file
Open in desktop
Binary file not shown.
40 changes: 40 additions & 0 deletions Tests/test_file_libtiff.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -843,6 +843,46 @@ def test_tiled_ycbcr_jpeg_2x2_sampling(self):
with Image.open(infile) as im:
assert_image_similar_tofile(im, "Tests/images/flower.jpg", 0.5)

def test_strip_planar_rgb(self):
# gdal_translate -co TILED=no -co INTERLEAVE=BAND -co COMPRESS=LZW \
# tiff_strip_raw.tif tiff_strip_planar_lzw.tiff
infile = "Tests/images/tiff_strip_planar_lzw.tiff"
with Image.open(infile) as im:
assert_image_equal_tofile(im, "Tests/images/tiff_adobe_deflate.png")

def test_tiled_planar_rgb(self):
# gdal_translate -co TILED=yes -co INTERLEAVE=BAND -co COMPRESS=LZW \
# tiff_tiled_raw.tif tiff_tiled_planar_lzw.tiff
infile = "Tests/images/tiff_tiled_planar_lzw.tiff"
with Image.open(infile) as im:
assert_image_equal_tofile(im, "Tests/images/tiff_adobe_deflate.png")

def test_tiled_planar_16bit_RGB(self):
# gdal_translate -co TILED=yes -co INTERLEAVE=BAND -co COMPRESS=LZW \
# tiff_16bit_RGB.tiff tiff_tiled_planar_16bit_RGB.tiff
with Image.open("Tests/images/tiff_tiled_planar_16bit_RGB.tiff") as im:
assert_image_equal_tofile(im, "Tests/images/tiff_16bit_RGB_target.png")

def test_strip_planar_16bit_RGB(self):
# gdal_translate -co TILED=no -co INTERLEAVE=BAND -co COMPRESS=LZW \
# tiff_16bit_RGB.tiff tiff_strip_planar_16bit_RGB.tiff
with Image.open("Tests/images/tiff_strip_planar_16bit_RGB.tiff") as im:
assert_image_equal_tofile(im, "Tests/images/tiff_16bit_RGB_target.png")

def test_tiled_planar_16bit_RGBa(self):
# gdal_translate -co TILED=yes \
# -co INTERLEAVE=BAND -co COMPRESS=LZW -co ALPHA=PREMULTIPLIED \
# tiff_16bit_RGBa.tiff tiff_tiled_planar_16bit_RGBa.tiff
with Image.open("Tests/images/tiff_tiled_planar_16bit_RGBa.tiff") as im:
assert_image_equal_tofile(im, "Tests/images/tiff_16bit_RGBa_target.png")

def test_strip_planar_16bit_RGBa(self):
# gdal_translate -co TILED=no \
# -co INTERLEAVE=BAND -co COMPRESS=LZW -co ALPHA=PREMULTIPLIED \
# tiff_16bit_RGBa.tiff tiff_strip_planar_16bit_RGBa.tiff
with Image.open("Tests/images/tiff_strip_planar_16bit_RGBa.tiff") as im:
assert_image_equal_tofile(im, "Tests/images/tiff_16bit_RGBa_target.png")

@pytest.mark.xfail(is_big_endian(), reason="Fails on big-endian")
def test_old_style_jpeg(self):
infile = "Tests/images/old-style-jpeg-compression.tif"
Expand Down
54 changes: 54 additions & 0 deletions Tests/test_lib_pack.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -320,6 +320,23 @@ def test_RGB(self):
self.assert_unpack("RGB", "G", 1, (0, 1, 0), (0, 2, 0), (0, 3, 0))
self.assert_unpack("RGB", "B", 1, (0, 0, 1), (0, 0, 2), (0, 0, 3))

self.assert_unpack("RGB", "R;16B", 2, (1, 0, 0), (3, 0, 0), (5, 0, 0))
self.assert_unpack("RGB", "G;16B", 2, (0, 1, 0), (0, 3, 0), (0, 5, 0))
self.assert_unpack("RGB", "B;16B", 2, (0, 0, 1), (0, 0, 3), (0, 0, 5))

self.assert_unpack("RGB", "R;16L", 2, (2, 0, 0), (4, 0, 0), (6, 0, 0))
self.assert_unpack("RGB", "G;16L", 2, (0, 2, 0), (0, 4, 0), (0, 6, 0))
self.assert_unpack("RGB", "B;16L", 2, (0, 0, 2), (0, 0, 4), (0, 0, 6))

if sys.byteorder == "little":
self.assert_unpack("RGB", "R;16N", 2, (2, 0, 0), (4, 0, 0), (6, 0, 0))
self.assert_unpack("RGB", "G;16N", 2, (0, 2, 0), (0, 4, 0), (0, 6, 0))
self.assert_unpack("RGB", "B;16N", 2, (0, 0, 2), (0, 0, 4), (0, 0, 6))
else:
self.assert_unpack("RGB", "R;16N", 2, (1, 0, 0), (3, 0, 0), (5, 0, 0))
self.assert_unpack("RGB", "G;16N", 2, (0, 1, 0), (0, 3, 0), (0, 5, 0))
self.assert_unpack("RGB", "B;16N", 2, (0, 0, 1), (0, 0, 3), (0, 0, 5))

def test_RGBA(self):
self.assert_unpack("RGBA", "LA", 2, (1, 1, 1, 2), (3, 3, 3, 4), (5, 5, 5, 6))
self.assert_unpack(
Expand Down Expand Up @@ -450,6 +467,43 @@ def test_RGBA(self):
self.assert_unpack("RGBA", "B", 1, (0, 0, 1, 0), (0, 0, 2, 0), (0, 0, 3, 0))
self.assert_unpack("RGBA", "A", 1, (0, 0, 0, 1), (0, 0, 0, 2), (0, 0, 0, 3))

self.assert_unpack("RGBA", "R;16B", 2, (1, 0, 0, 0), (3, 0, 0, 0), (5, 0, 0, 0))
self.assert_unpack("RGBA", "G;16B", 2, (0, 1, 0, 0), (0, 3, 0, 0), (0, 5, 0, 0))
self.assert_unpack("RGBA", "B;16B", 2, (0, 0, 1, 0), (0, 0, 3, 0), (0, 0, 5, 0))
self.assert_unpack("RGBA", "A;16B", 2, (0, 0, 0, 1), (0, 0, 0, 3), (0, 0, 0, 5))

self.assert_unpack("RGBA", "R;16L", 2, (2, 0, 0, 0), (4, 0, 0, 0), (6, 0, 0, 0))
self.assert_unpack("RGBA", "G;16L", 2, (0, 2, 0, 0), (0, 4, 0, 0), (0, 6, 0, 0))
self.assert_unpack("RGBA", "B;16L", 2, (0, 0, 2, 0), (0, 0, 4, 0), (0, 0, 6, 0))
self.assert_unpack("RGBA", "A;16L", 2, (0, 0, 0, 2), (0, 0, 0, 4), (0, 0, 0, 6))

if sys.byteorder == "little":
self.assert_unpack(
"RGBA", "R;16N", 2, (2, 0, 0, 0), (4, 0, 0, 0), (6, 0, 0, 0)
)
self.assert_unpack(
"RGBA", "G;16N", 2, (0, 2, 0, 0), (0, 4, 0, 0), (0, 6, 0, 0)
)
self.assert_unpack(
"RGBA", "B;16N", 2, (0, 0, 2, 0), (0, 0, 4, 0), (0, 0, 6, 0)
)
self.assert_unpack(
"RGBA", "A;16N", 2, (0, 0, 0, 2), (0, 0, 0, 4), (0, 0, 0, 6)
)
else:
self.assert_unpack(
"RGBA", "R;16N", 2, (1, 0, 0, 0), (3, 0, 0, 0), (5, 0, 0, 0)
)
self.assert_unpack(
"RGBA", "G;16N", 2, (0, 1, 0, 0), (0, 3, 0, 0), (0, 5, 0, 0)
)
self.assert_unpack(
"RGBA", "B;16N", 2, (0, 0, 1, 0), (0, 0, 3, 0), (0, 0, 5, 0)
)
self.assert_unpack(
"RGBA", "A;16N", 2, (0, 0, 0, 1), (0, 0, 0, 3), (0, 0, 0, 5)
)

def test_RGBa(self):
self.assert_unpack(
"RGBa", "RGBa", 4, (1, 2, 3, 4), (5, 6, 7, 8), (9, 10, 11, 12)
Expand Down
201 changes: 128 additions & 73 deletions src/libImaging/TiffDecode.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -318,8 +318,8 @@ _decodeStripYCbCr(Imaging im, ImagingCodecState state, TIFF *tiff) {
}

int
_decodeStrip(Imaging im, ImagingCodecState state, TIFF *tiff) {
INT32 strip_row;
_decodeStrip(Imaging im, ImagingCodecState state, TIFF *tiff, int planes, ImagingShuffler *unpackers) {
INT32 strip_row = 0;
UINT8 *new_data;
UINT32 rows_per_strip, row_byte_size;
int ret;
Expand All @@ -331,7 +331,7 @@ _decodeStrip(Imaging im, ImagingCodecState state, TIFF *tiff) {
TRACE(("RowsPerStrip: %u \n", rows_per_strip));

// We could use TIFFStripSize, but for YCbCr data it returns subsampled data size
row_byte_size = (state->xsize * state->bits + 7) / 8;
row_byte_size = (state->xsize * state->bits / planes + 7) / 8;

/* overflow check for realloc */
if (INT_MAX / row_byte_size < rows_per_strip) {
Expand Down Expand Up @@ -364,35 +364,32 @@ _decodeStrip(Imaging im, ImagingCodecState state, TIFF *tiff) {
state->buffer = new_data;

for (; state->y < state->ysize; state->y += rows_per_strip) {
if (TIFFReadEncodedStrip(
tiff,
TIFFComputeStrip(tiff, state->y, 0),
(tdata_t)state->buffer,
-1) == -1) {
TRACE(("Decode Error, strip %d\n", TIFFComputeStrip(tiff, state->y, 0)));
state->errcode = IMAGING_CODEC_BROKEN;
return -1;
}
int plane;
for (plane = 0; plane < planes; plane++) {
ImagingShuffler shuffler = unpackers[plane];
if (TIFFReadEncodedStrip(tiff, TIFFComputeStrip(tiff, state->y, plane), (tdata_t)state->buffer, -1) == -1) {
TRACE(("Decode Error, strip %d\n", TIFFComputeStrip(tiff, state->y, 0)));
state->errcode = IMAGING_CODEC_BROKEN;
return -1;
}

TRACE(("Decoded strip for row %d \n", state->y));
TRACE(("Decoded strip for row %d \n", state->y));

// iterate over each row in the strip and stuff data into image
for (strip_row = 0;
strip_row < min((INT32)rows_per_strip, state->ysize - state->y);
strip_row++) {
TRACE(("Writing data into line %d ; \n", state->y + strip_row));
// iterate over each row in the strip and stuff data into image
for (strip_row = 0; strip_row < min((INT32) rows_per_strip, state->ysize - state->y); strip_row++) {
TRACE(("Writing data into line %d ; \n", state->y + strip_row));

// UINT8 * bbb = state->buffer + strip_row * (state->bytes /
// rows_per_strip); TRACE(("chars: %x %x %x %x\n", ((UINT8 *)bbb)[0],
// ((UINT8 *)bbb)[1], ((UINT8 *)bbb)[2], ((UINT8 *)bbb)[3]));
// UINT8 * bbb = state->buffer + strip_row * (state->bytes / rows_per_strip);
// TRACE(("chars: %x %x %x %x\n", ((UINT8 *)bbb)[0], ((UINT8 *)bbb)[1], ((UINT8 *)bbb)[2], ((UINT8 *)bbb)[3]));

state->shuffle(
(UINT8 *)im->image[state->y + state->yoff + strip_row] +
state->xoff * im->pixelsize,
state->buffer + strip_row * row_byte_size,
state->xsize);
shuffler((UINT8*) im->image[state->y + state->yoff + strip_row] +
state->xoff * im->pixelsize,
state->buffer + strip_row * row_byte_size,
state->xsize);
}
}
}

return 0;
}

Expand All @@ -405,6 +402,11 @@ ImagingLibTiffDecode(
TIFF *tiff;
uint16 photometric = 0; // init to not PHOTOMETRIC_YCBCR
int isYCbCr = 0;
uint16 planarconfig = 0;
int planes = 1;
ImagingShuffler unpackers[4];

memset(unpackers, 0, sizeof(ImagingShuffler) * 4);

/* buffer is the encoded file, bytes is the length of the encoded file */
/* it all ends up in state->buffer, which is a uint8* from Imaging.h */
Expand Down Expand Up @@ -501,6 +503,35 @@ ImagingLibTiffDecode(

TIFFGetField(tiff, TIFFTAG_PHOTOMETRIC, &photometric);
isYCbCr = photometric == PHOTOMETRIC_YCBCR;
TIFFGetFieldDefaulted(tiff, TIFFTAG_PLANARCONFIG, &planarconfig);

// YCbCr data is read as RGB by libtiff and we don't need to worry about planar storage in that case
// if number of bands is 1, there is no difference with contig case
if (planarconfig == PLANARCONFIG_SEPARATE &&
im->bands > 1 &&
!isYCbCr) {

uint16 bits_per_sample = 8;

TIFFGetFieldDefaulted(tiff, TIFFTAG_BITSPERSAMPLE, &bits_per_sample);
if (bits_per_sample != 8 && bits_per_sample != 16) {
TRACE(("Invalid value for bits per sample: %d\n", bits_per_sample));
state->errcode = IMAGING_CODEC_BROKEN;
goto decode_err;
}

planes = im->bands;

// We'll pick appropriate set of unpackers depending on planar_configuration
// It does not matter if data is RGB(A), CMYK or LUV really,
// we just copy it plane by plane
unpackers[0] = ImagingFindUnpacker("RGBA", bits_per_sample == 16 ? "R;16N" : "R", NULL);
unpackers[1] = ImagingFindUnpacker("RGBA", bits_per_sample == 16 ? "G;16N" : "G", NULL);
unpackers[2] = ImagingFindUnpacker("RGBA", bits_per_sample == 16 ? "B;16N" : "B", NULL);
unpackers[3] = ImagingFindUnpacker("RGBA", bits_per_sample == 16 ? "A;16N" : "A", NULL);
} else {
unpackers[0] = state->shuffle;
}

if (TIFFIsTiled(tiff)) {
INT32 x, y, tile_y;
Expand All @@ -527,7 +558,7 @@ ImagingLibTiffDecode(
} else {
// We could use TIFFTileSize, but for YCbCr data it returns subsampled data
// size
row_byte_size = (tile_width * state->bits + 7) / 8;
row_byte_size = (tile_width * state->bits / planes + 7) / 8;
}

/* overflow check for realloc */
Expand All @@ -539,7 +570,7 @@ ImagingLibTiffDecode(
state->bytes = row_byte_size * tile_length;

if (TIFFTileSize(tiff) > state->bytes) {
// If the strip size as expected by LibTiff isn't what we're expecting,
// If the tile size as expected by LibTiff isn't what we're expecting,
// abort.
state->errcode = IMAGING_CODEC_MEMORY;
goto decode_err;
Expand All @@ -558,65 +589,89 @@ ImagingLibTiffDecode(
TRACE(("TIFFTileSize: %d\n", state->bytes));

for (y = state->yoff; y < state->ysize; y += tile_length) {
for (x = state->xoff; x < state->xsize; x += tile_width) {
if (isYCbCr) {
/* To avoid dealing with YCbCr subsampling, let libtiff handle it */
if (!TIFFReadRGBATile(tiff, x, y, (UINT32 *)state->buffer)) {
TRACE(("Decode Error, Tile at %dx%d\n", x, y));
state->errcode = IMAGING_CODEC_BROKEN;
goto decode_err;
}
} else {
if (TIFFReadTile(tiff, (tdata_t)state->buffer, x, y, 0, 0) == -1) {
TRACE(("Decode Error, Tile at %dx%d\n", x, y));
state->errcode = IMAGING_CODEC_BROKEN;
goto decode_err;
}
}

TRACE(("Read tile at %dx%d; \n\n", x, y));

current_tile_width = min((INT32)tile_width, state->xsize - x);
current_tile_length = min((INT32)tile_length, state->ysize - y);
// iterate over each line in the tile and stuff data into image
for (tile_y = 0; tile_y < current_tile_length; tile_y++) {
TRACE(
("Writing tile data at %dx%d using tile_width: %d; \n",
tile_y + y,
x,
current_tile_width));

// UINT8 * bbb = state->buffer + tile_y * row_byte_size;
// TRACE(("chars: %x%x%x%x\n", ((UINT8 *)bbb)[0], ((UINT8 *)bbb)[1],
// ((UINT8 *)bbb)[2], ((UINT8 *)bbb)[3]));
/*
* For some reason the TIFFReadRGBATile() function
* chooses the lower left corner as the origin.
* Vertically mirror by shuffling the scanlines
* backwards
*/

int plane;
for (plane = 0; plane < planes; plane++) {
ImagingShuffler shuffler = unpackers[plane];
for (x = state->xoff; x < state->xsize; x += tile_width) {
if (isYCbCr) {
current_line = tile_length - tile_y - 1;
/* To avoid dealing with YCbCr subsampling, let libtiff handle it */
if (!TIFFReadRGBATile(tiff, x, y, (UINT32 *)state->buffer)) {
TRACE(("Decode Error, Tile at %dx%d\n", x, y));
state->errcode = IMAGING_CODEC_BROKEN;
goto decode_err;
}
} else {
current_line = tile_y;
if (TIFFReadTile(tiff, (tdata_t)state->buffer, x, y, 0, plane) == -1) {
TRACE(("Decode Error, Tile at %dx%d\n", x, y));
state->errcode = IMAGING_CODEC_BROKEN;
goto decode_err;
}
}

state->shuffle(
(UINT8 *)im->image[tile_y + y] + x * im->pixelsize,
state->buffer + current_line * row_byte_size,
current_tile_width);
TRACE(("Read tile at %dx%d; \n\n", x, y));

current_tile_width = min((INT32) tile_width, state->xsize - x);
current_tile_length = min((INT32) tile_length, state->ysize - y);
// iterate over each line in the tile and stuff data into image
for (tile_y = 0; tile_y < current_tile_length; tile_y++) {
TRACE(("Writing tile data at %dx%d using tile_width: %d; \n", tile_y + y, x, current_tile_width));

// UINT8 * bbb = state->buffer + tile_y * row_byte_size;
// TRACE(("chars: %x%x%x%x\n", ((UINT8 *)bbb)[0], ((UINT8 *)bbb)[1], ((UINT8 *)bbb)[2], ((UINT8 *)bbb)[3]));
/*
* For some reason the TIFFReadRGBATile() function
* chooses the lower left corner as the origin.
* Vertically mirror by shuffling the scanlines
* backwards
*/

if (isYCbCr) {
current_line = tile_length - tile_y - 1;
} else {
current_line = tile_y;
}

shuffler((UINT8*) im->image[tile_y + y] + x * im->pixelsize,
state->buffer + current_line * row_byte_size,
current_tile_width
);
Comment on lines +634 to +637
Copy link
Copy Markdown
Contributor

@nulano nulano Jan 9, 2021 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not yet sure what is going on, but compiling with debug information shows that state->buffer is NULL on this line even though it was valid on line 587 for the realloc call.

The error doesn't show up when compiling with in a debug build (setup.py build_ext --dubug), I had to change setup.py line 852 to Extension("PIL._imaging", files, extra_compile_args=["/Z7"], extra_link_args=["/DEBUG"]), and copy the PDB to the PIL directory.

Copy link
Copy Markdown
Contributor

@nulano nulano Jan 9, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Solved it, see wiredfool#6

}
}
}
}
} else {
if (!isYCbCr) {
_decodeStrip(im, state, tiff);
_decodeStrip(im, state, tiff, planes, unpackers);
} else {
_decodeStripYCbCr(im, state, tiff);
}
}

if (!state->errcode) {
// Check if raw mode was RGBa and it was stored on separate planes
// so we have to convert it to RGBA
if (planes > 3 && strcmp(im->mode, "RGBA") == 0) {
uint16 extrasamples;
uint16* sampleinfo;
ImagingShuffler shuffle;
INT32 y;

TIFFGetFieldDefaulted(tiff, TIFFTAG_EXTRASAMPLES, &extrasamples, &sampleinfo);

if (extrasamples >= 1 &&
(sampleinfo[0] == EXTRASAMPLE_UNSPECIFIED || sampleinfo[0] == EXTRASAMPLE_ASSOCALPHA)
) {
shuffle = ImagingFindUnpacker("RGBA", "RGBa", NULL);

for (y = state->yoff; y < state->ysize; y++) {
UINT8* ptr = (UINT8*) im->image[y + state->yoff] +
state->xoff * im->pixelsize;
shuffle(ptr, ptr, state->xsize);
}
}
}
}

decode_err:
TIFFClose(tiff);
TRACE(("Done Decoding, Returning \n"));
Expand Down
Loading