1 package nom.tam.image.compression.hdu;
2
3 import nom.tam.fits.BinaryTable;
4 import nom.tam.fits.BinaryTableHDU;
5 import nom.tam.fits.FitsException;
6 import nom.tam.fits.Header;
7 import nom.tam.fits.HeaderCard;
8 import nom.tam.fits.HeaderCardException;
9 import nom.tam.fits.header.Compression;
10 import nom.tam.fits.header.Standard;
11 import nom.tam.util.Cursor;
12 import nom.tam.util.type.ElementType;
13
14 /*
15 * #%L
16 * nom.tam FITS library
17 * %%
18 * Copyright (C) 1996 - 2024 nom-tam-fits
19 * %%
20 * This is free and unencumbered software released into the public domain.
21 *
22 * Anyone is free to copy, modify, publish, use, compile, sell, or
23 * distribute this software, either in source code form or as a compiled
24 * binary, for any purpose, commercial or non-commercial, and by any
25 * means.
26 *
27 * In jurisdictions that recognize copyright laws, the author or authors
28 * of this software dedicate any and all copyright interest in the
29 * software to the public domain. We make this dedication for the benefit
30 * of the public at large and to the detriment of our heirs and
31 * successors. We intend this dedication to be an overt act of
32 * relinquishment in perpetuity of all present and future rights to this
33 * software under copyright law.
34 *
35 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
36 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
37 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
38 * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
39 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
40 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
41 * OTHER DEALINGS IN THE SOFTWARE.
42 * #L%
43 */
44
45 import static nom.tam.fits.header.Compression.ZTABLE;
46
47 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
48
49 /**
50 * A header-data unit (HDU) containing a compressed binary table. A compressed table is still a binary table but with
51 * some additional constraints. The original table is divided into groups of rows (tiles) and each tile is compressed on
52 * its own. The compressed data is then stored in the 3 data columns of this binary table (compressed, gzipped and
53 * uncompressed) depending on the compression type used in the tile. Additional data columns may contain specific
54 * compression options for each tile (i.e. compressed table row) individually. Table keywords, which conflict with those
55 * in the original table are 'saved' under standard alternative names, so they may be restored with the original table
56 * as appropriate.
57 * <p>
58 * Compressing a table HDU is typically a two-step process:
59 * </p>
60 * <ol>
61 * <li>Create a <code>CompressedTableHDU</code>, e.g. with {@link #fromBinaryTableHDU(BinaryTableHDU, int, String...)},
62 * using the specified number of table rows per compressed block, and compression algorithm(s)</li>
63 * <li>Perform the compression via {@link #compress()}</li>
64 * </ol>
65 * <p>
66 * For example to compress a binary table:
67 * </p>
68 *
69 * <pre>
70 * BinaryTableHDU table = ...
71 *
72 * // 1. Create compressed HDU with the
73 * CompressedTableHDU compressed = CompressedTableHDU.fromBinaryTableHDU(table, 4, Compression.ZCMPTYPE_RICE_1);
74 *
75 * // 2. Perform the compression.
76 * compressed.compress();
77 * </pre>
78 * <p>
79 * which of course you can compact into a single line as:
80 * </p>
81 *
82 * <pre>
83 * CompressedTableHDU compressed = CompressedTableHDU.fromBinaryTableHDU(table, 4, Compression.ZCMPTYPE_RICE_1).compress();
84 * </pre>
85 * <p>
86 * The two step process (as opposed to a single-step one) was probably chosen because it mimics that of
87 * {@link CompressedImageHDU}, where further configuration steps may be inserted in-between. After the compression, the
88 * compressed table HDU can be handled just like any other HDU, and written to a file or stream, for example.
89 * </p>
90 * <p>
91 * The reverse process is simply via the {@link #asBinaryTableHDU()} method. E.g.:
92 * </p>
93 *
94 * <pre>
95 * CompressedTableHDU compressed = ...
96 * BinaryTableHDU table = compressed.asBinaryTableHDU();
97 * </pre>
98 *
99 * @see CompressedTableData
100 */
101 @SuppressWarnings("deprecation")
102 public class CompressedTableHDU extends BinaryTableHDU {
103
104 private static boolean reversedVLAIndices = false;
105
106 /**
107 * Enables or disables using reversed compressed/uncompressed heap indices for when decompressing variable-length
108 * arrays (VLAs), as was prescribed by the original FITS 4.0 standard and the Pence et al. 2013 convention.
109 * <p>
110 * The <a href="https://fits.gsfc.nasa.gov/standard40/fits_standard40aa-le.pdf">FITS 4.0 standard</a> defines the
111 * convention of compressing variable-length columns in binary tables. On page 50 it writes:
112 * </p>
113 * <blockquote> 2. Append the VLA descriptors from the uncompressed table (which may be either Q-type or P-type) to
114 * the temporary array of VLA descriptors for the compressed table. </blockquote>
115 * <p>
116 * And the original <a href="https://fits.gsfc.nasa.gov/registry/tiletablecompression/tiletable2.0.pdf">Tiled-table
117 * convention</a> by W. Pence et al. also writes:
118 * </p>
119 * <blockquote> [...] we concatenate the array of descriptors from the uncompressed table onto the end of the
120 * temporary array of descriptors (to the compressed VLAs in the compressed table) before the 2 combined arrays of
121 * descriptors are compressed and written into the heap in the compressed table. </blockquote> However, it appears
122 * that the commonly used tools <code>fpack</code> / <code>funpack</code> provided in CFITSIO do just the reverse of
123 * this presciption. They store the uncopressed pointer <i>before</i> the compressed pointers. Because these tools
124 * are widely used we want to support files created or consumed by these tools. As such we allow to deviate from the
125 * FITS standard, and swap the order of the stored VLA indices inside compressed tables.
126 *
127 * @param value <code>true</code> if we should use VLA indices in compressed tables that follow the format
128 * described in the original Pence et al. 2013 convention, and also the FITS 4.0 standard as of
129 * 2024 Mar 1; otherwise <code>false</code>. These original prescriptions define the reverse of
130 * what is actually implemented by CFITSIO and its tools <code>fpack</code> and
131 * <code>funpack</code>. (Our default is to conform to CFITSIO, and the expected revision of the
132 * standard to match).
133 *
134 * @see #hasOldStandardVLAIndexing()
135 * @see #asBinaryTableHDU()
136 *
137 * @since 1.19.1
138 *
139 * @author Attila Kovacs
140 */
141 public static void useOldStandardVLAIndexing(boolean value) {
142 reversedVLAIndices = value;
143 }
144
145 /**
146 * Checks if we should use reversed compressed/uncompressed heap indices for when decompressing variable-length
147 * arrays (VLAs).
148 *
149 * @return value <code>true</code> if we will assime VLA indices in compressed tables that follow the format
150 * described in the original Pence et al. 2013 convention, and also the FITS 4.0 standard as of 2024 Mar
151 * 1; otherwise <code>false</code>. These original prescriptions define the reverse of what is actually
152 * implemented by CFITSIO and its tools <code>fpack</code> and <code>funpack</code>. (Our default is to
153 * conform to CFITSIO, and the expcted revision of the standard to match).
154 *
155 * @see #useOldStandardVLAIndexing(boolean)
156 *
157 * @since 1.19.1
158 *
159 * @author Attila Kovacs
160 */
161 public static boolean hasOldStandardVLAIndexing() {
162 return reversedVLAIndices;
163 }
164
165 /**
166 * Prepare a compressed binary table HDU for the specified binary table. When the tile row size is specified with
167 * -1, the value will be set ti the number of rows in the table. The table will be compressed in "rows" that are
168 * defined by the tile size. Next step would be to set the compression options into the HDU and then compress it.
169 *
170 * @param binaryTableHDU the binary table to compress
171 * @param tileRows the number of rows that should be compressed per tile.
172 * @param columnCompressionAlgorithms the compression algorithms to use for the columns (optional default
173 * compression will be used if a column has no compression specified). You
174 * should typically use one or more of the enum values defined in
175 * {@link Compression}. The FITS standard currently allows only the lossless
176 * GZIP_1, GZIP_2, RICE_1, or NOCOMPRESS (e.g.
177 * {@link Compression#ZCMPTYPE_GZIP_1} or
178 * {@link Compression#ZCMPTYPE_NOCOMPRESS}.)
179 *
180 * @return the prepared compressed binary table HDU.
181 *
182 * @throws IllegalArgumentException if any of the listed compression algorithms are not approved for use with
183 * tables.
184 * @throws FitsException if the binary table could not be used to create a compressed binary table.
185 *
186 * @see Compression
187 * @see Compression#ZCMPTYPE_GZIP_1
188 * @see Compression#ZCMPTYPE_GZIP_2
189 * @see Compression#ZCMPTYPE_RICE_1
190 * @see Compression#ZCMPTYPE_NOCOMPRESS
191 * @see #asBinaryTableHDU()
192 * @see #useOldStandardVLAIndexing(boolean)
193 */
194 public static CompressedTableHDU fromBinaryTableHDU(BinaryTableHDU binaryTableHDU, int tileRows,
195 String... columnCompressionAlgorithms) throws FitsException {
196
197 Header header = new Header();
198
199 CompressedTableData compressedData = new CompressedTableData();
200 compressedData.setColumnCompressionAlgorithms(columnCompressionAlgorithms);
201
202 int rowsPerTile = tileRows > 0 ? tileRows : binaryTableHDU.getData().getNRows();
203 compressedData.setRowsPerTile(rowsPerTile);
204
205 Cursor<String, HeaderCard> headerIterator = header.iterator();
206 Cursor<String, HeaderCard> imageIterator = binaryTableHDU.getHeader().iterator();
207 while (imageIterator.hasNext()) {
208 HeaderCard card = imageIterator.next();
209 CompressedCard.restore(card, headerIterator);
210 }
211
212 CompressedTableHDU compressedHDU = new CompressedTableHDU(header, compressedData);
213 compressedData.prepareUncompressedData(binaryTableHDU.getData());
214 compressedData.fillHeader(header);
215
216 return compressedHDU;
217 }
218
219 /**
220 * Check that this HDU has a valid header for this type.
221 *
222 * @deprecated (<i>for internal use</i>) Will reduce visibility in the future
223 *
224 * @param hdr header to check
225 *
226 * @return <CODE>true</CODE> if this HDU has a valid header.
227 */
228 @Deprecated
229 @SuppressFBWarnings(value = "HSM_HIDING_METHOD", justification = "deprecated existing method, kept for compatibility")
230 public static boolean isHeader(Header hdr) {
231 return hdr.getBooleanValue(ZTABLE, false);
232 }
233
234 /**
235 * @deprecated (<i>for internal use</i>) Will reduce visibility in the future
236 *
237 * @param hdr the header that describes the compressed HDU
238 *
239 * @return a new blank data object created from the header description
240 */
241 @Deprecated
242 public static CompressedTableData manufactureData(Header hdr) throws FitsException {
243 return new CompressedTableData(hdr);
244 }
245
246 /**
247 * Creates an new compressed table HDU with the specified header and compressed data.
248 *
249 * @param hdr the header
250 * @param datum the compressed table data. The data may not be actually compressed at this point, int which case you
251 * may need to call {@link #compress()} before writing the new compressed HDU to a stream.
252 *
253 * @see #compress()
254 */
255 public CompressedTableHDU(Header hdr, CompressedTableData datum) {
256 super(hdr, datum);
257 }
258
259 /**
260 * Restores the original binary table HDU by decompressing the data contained in this compresed table HDU.
261 *
262 * @return The uncompressed binary table HDU.
263 *
264 * @throws FitsException If there was an issue with the decompression.
265 *
266 * @see #asBinaryTableHDU(int, int)
267 * @see #fromBinaryTableHDU(BinaryTableHDU, int, String...)
268 * @see #useOldStandardVLAIndexing(boolean)
269 */
270 public BinaryTableHDU asBinaryTableHDU() throws FitsException {
271 return asBinaryTableHDU(0, getTileCount());
272 // Header header = getTableHeader();
273 // BinaryTable data = BinaryTableHDU.manufactureData(header);
274 // BinaryTableHDU tableHDU = new BinaryTableHDU(header, data);
275 // getData().asBinaryTable(data, getHeader(), header);
276 // return tableHDU;
277 }
278
279 /**
280 * Returns the number of table rows that are compressed in each table tile. This may be useful for figuring out what
281 * tiles to decompress, e.g. via {@link #asBinaryTableHDU(int, int)}, when wanting to access select table rows only.
282 * This value is stored under the FITS keyword ZTILELEN in the compressed header. Thus, this method simply provides
283 * a user-friendly way to access it. Note that the last tile may contain fewer rows than the value indicated by this
284 *
285 * @return the number of table rows that are compressed into a tile.
286 *
287 * @throws FitsException if the compressed header does not contain the required ZTILELEN keyword, or it is <= 0.
288 *
289 * @see #asBinaryTableHDU(int, int)
290 *
291 * @since 1.19
292 */
293 public int getTileRows() throws FitsException {
294 int n = getHeader().getIntValue(Compression.ZTILELEN, -1);
295 if (n <= 0) {
296 throw new FitsException("imnvalid or missing ZTILELEN header keyword");
297 }
298 return n;
299 }
300
301 /**
302 * Returns the number of compressed tiles contained in this HDU.
303 *
304 * @return the number of compressed tiles in this table. It is the same as the NAXIS2 value of the header, which is
305 * also returned by {@link #getNRows()} for this compressed table.
306 *
307 * @see #getTileRows()
308 *
309 * @since 1.19
310 */
311 public int getTileCount() {
312 return getNRows();
313 }
314
315 /**
316 * Restores a section of the original binary table HDU by decompressing a selected range of compressed table tiles.
317 * The returned section will start at row index <code>fromTile * getTileRows()</code> of the full table.
318 *
319 * @param fromTile Java index of first tile to decompress
320 * @param toTile Java index of last tile to decompress
321 *
322 * @return The uncompressed binary table HDU from the selected compressed tiles.
323 *
324 * @throws IllegalArgumentException If the tile range is out of bounds
325 * @throws FitsException If there was an issue with the decompression.
326 *
327 * @see #getTileRows()
328 * @see #getTileCount()
329 * @see #asBinaryTableHDU()
330 * @see #fromBinaryTableHDU(BinaryTableHDU, int, String...)
331 * @see #useOldStandardVLAIndexing(boolean)
332 */
333 public BinaryTableHDU asBinaryTableHDU(int fromTile, int toTile) throws FitsException, IllegalArgumentException {
334 Header header = getTableHeader();
335 int tileSize = getTileRows();
336 getData().setRowsPerTile(tileSize);
337
338 if (fromTile < 0 || toTile > getTileCount() || toTile <= fromTile) {
339 throw new IllegalArgumentException(
340 "illegal tile range [" + fromTile + ", " + toTile + "] for " + getTileCount() + " tiles");
341 }
342
343 // Set the correct number of rows
344 int rows = getHeader().getIntValue(Compression.ZNAXISn.n(2));
345 header.addValue(Standard.NAXIS2, Integer.min(rows, toTile * tileSize) - fromTile * tileSize);
346
347 BinaryTable data = BinaryTableHDU.manufactureData(header);
348 BinaryTableHDU tableHDU = new BinaryTableHDU(header, data);
349 getData().asBinaryTable(data, getHeader(), header, fromTile);
350
351 return tableHDU;
352 }
353
354 /**
355 * Restores a section of the original binary table HDU by decompressing a single compressed table tile. The returned
356 * section will start at row index <code>tile * getTileRows()</code> of the full table.
357 *
358 * @param tile Java index of the table tile to decompress
359 *
360 * @return The uncompressed binary table HDU from the selected compressed tile.
361 *
362 * @throws IllegalArgumentException If the tile index is out of bounds
363 * @throws FitsException If there was an issue with the decompression.
364 *
365 * @see #asBinaryTableHDU(int, int)
366 * @see #getTileRows()
367 * @see #getTileCount()
368 * @see #useOldStandardVLAIndexing(boolean)
369 */
370 public final BinaryTableHDU asBinaryTableHDU(int tile) throws FitsException, IllegalArgumentException {
371 return asBinaryTableHDU(tile, tile + 1);
372 }
373
374 /**
375 * Returns a particular section of a decompressed data column.
376 *
377 * @param col the Java column index
378 *
379 * @return The uncompressed column data as an array.
380 *
381 * @throws IllegalArgumentException If the tile range is out of bounds
382 * @throws FitsException If there was an issue with the decompression.
383 *
384 * @see #getColumnData(int, int, int)
385 * @see #asBinaryTableHDU()
386 * @see #fromBinaryTableHDU(BinaryTableHDU, int, String...)
387 */
388 public Object getColumnData(int col) throws FitsException, IllegalArgumentException {
389 return getColumnData(col, 0, getTileCount());
390 }
391
392 /**
393 * Returns a particular section of a decompressed data column.
394 *
395 * @param col the Java column index
396 * @param fromTile the Java index of first tile to decompress
397 * @param toTile the Java index of last tile to decompress
398 *
399 * @return The uncompressed column data segment as an array.
400 *
401 * @throws IllegalArgumentException If the tile range is out of bounds
402 * @throws FitsException If there was an issue with the decompression.
403 *
404 * @see #getColumnData(int)
405 * @see #asBinaryTableHDU()
406 * @see #fromBinaryTableHDU(BinaryTableHDU, int, String...)
407 */
408 public Object getColumnData(int col, int fromTile, int toTile) throws FitsException, IllegalArgumentException {
409 getData().setRowsPerTile(getTileRows());
410 return getData().getColumnData(col, fromTile, toTile, getHeader(), getTableHeader());
411 }
412
413 /**
414 * Performs the actual compression with the selected algorithm(s) and options. When creating a compressed table HDU,
415 * e.g using the {@link #fromBinaryTableHDU(BinaryTableHDU, int, String...)} method, the HDU is merely prepared but
416 * without actually performing the compression, and this method will have to be called to actually perform the
417 * compression. The design would allow for setting options between creation and compressing, but in this case there
418 * is really nothing of the sort.
419 *
420 * @return itself
421 *
422 * @throws FitsException if the compression could not be performed
423 *
424 * @see #fromBinaryTableHDU(BinaryTableHDU, int, String...)
425 * @see #useOldStandardVLAIndexing(boolean)
426 */
427 public CompressedTableHDU compress() throws FitsException {
428 getData().compress(getHeader());
429 return this;
430 }
431
432 /**
433 * Obtain a header representative of a decompressed TableHDU.
434 *
435 * @return Header with decompressed cards.
436 *
437 * @throws HeaderCardException if the card could not be copied
438 *
439 * @since 1.18
440 */
441 public Header getTableHeader() throws HeaderCardException {
442 Header header = new Header();
443
444 header.addValue(Standard.XTENSION, Standard.XTENSION_BINTABLE);
445 header.addValue(Standard.BITPIX, ElementType.BYTE.bitPix());
446 header.addValue(Standard.NAXIS, 2);
447
448 Cursor<String, HeaderCard> tableIterator = header.iterator();
449 Cursor<String, HeaderCard> iterator = getHeader().iterator();
450
451 while (iterator.hasNext()) {
452 CompressedCard.backup(iterator.next(), tableIterator);
453 }
454
455 return header;
456 }
457
458 @Override
459 public CompressedTableData getData() {
460 return (CompressedTableData) super.getData();
461 }
462
463 }