1 package nom.tam.image.compression.hdu;
2
3 import nom.tam.fits.BinaryTable;
4 import nom.tam.fits.BinaryTableHDU;
5 import nom.tam.fits.FitsException;
6 import nom.tam.fits.Header;
7 import nom.tam.fits.HeaderCard;
8 import nom.tam.fits.HeaderCardException;
9 import nom.tam.fits.header.Compression;
10 import nom.tam.fits.header.Standard;
11 import nom.tam.util.Cursor;
12 import nom.tam.util.type.ElementType;
13
14 /*
15 * #%L
16 * nom.tam FITS library
17 * %%
18 * Copyright (C) 1996 - 2024 nom-tam-fits
19 * %%
20 * This is free and unencumbered software released into the public domain.
21 *
22 * Anyone is free to copy, modify, publish, use, compile, sell, or
23 * distribute this software, either in source code form or as a compiled
24 * binary, for any purpose, commercial or non-commercial, and by any
25 * means.
26 *
27 * In jurisdictions that recognize copyright laws, the author or authors
28 * of this software dedicate any and all copyright interest in the
29 * software to the public domain. We make this dedication for the benefit
30 * of the public at large and to the detriment of our heirs and
31 * successors. We intend this dedication to be an overt act of
32 * relinquishment in perpetuity of all present and future rights to this
33 * software under copyright law.
34 *
35 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
36 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
37 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
38 * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
39 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
40 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
41 * OTHER DEALINGS IN THE SOFTWARE.
42 * #L%
43 */
44
45 import static nom.tam.fits.header.Compression.ZTABLE;
46
47 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
48
49 /**
50 * A header-data unit (HDU) containing a compressed binary table. A compressed table is still a binary table but with
51 * some additional constraints. The original table is divided into groups of rows (tiles) and each tile is compressed on
52 * its own. The compressed data is then stored in the 3 data columns of this binary table (compressed, gzipped and
53 * uncompressed) depending on the compression type used in the tile. Additional data columns may contain specific
54 * compression options for each tile (i.e. compressed table row) individually. Table keywords, which conflict with those
55 * in the original table are 'saved' under standard alternative names, so they may be restored with the original table
56 * as appropriate.
57 * <p>
58 * Compressing a table HDU is typically a two-step process:
59 * </p>
60 * <ol>
61 * <li>Create a <code>CompressedTableHDU</code>, e.g. with {@link #fromBinaryTableHDU(BinaryTableHDU, int, String...)},
62 * using the specified number of table rows per compressed block, and compression algorithm(s)</li>
63 * <li>Perform the compression via {@link #compress()}</li>
64 * </ol>
65 * <p>
66 * For example to compress a binary table:
67 * </p>
68 *
69 * <pre>
70 * BinaryTableHDU table = ...
71 *
72 * // 1. Create compressed HDU with the
73 * CompressedTableHDU compressed = CompressedTableHDU.fromBinaryTableHDU(table, 4, Compression.ZCMPTYPE_RICE_1);
74 *
75 * // 2. Perform the compression.
76 * compressed.compress();
77 * </pre>
78 * <p>
79 * which of course you can compact into a single line as:
80 * </p>
81 *
82 * <pre>
83 * CompressedTableHDU compressed = CompressedTableHDU.fromBinaryTableHDU(table, 4, Compression.ZCMPTYPE_RICE_1).compress();
84 * </pre>
85 * <p>
86 * The two step process (as opposed to a single-step one) was probably chosen because it mimics that of
87 * {@link CompressedImageHDU}, where further configuration steps may be inserted in-between. After the compression, the
88 * compressed table HDU can be handled just like any other HDU, and written to a file or stream, for example.
89 * </p>
90 * <p>
91 * The reverse process is simply via the {@link #asBinaryTableHDU()} method. E.g.:
92 * </p>
93 *
94 * <pre>
95 * CompressedTableHDU compressed = ...
96 * BinaryTableHDU table = compressed.asBinaryTableHDU();
97 * </pre>
98 *
99 * @see CompressedTableData
100 */
101 @SuppressWarnings("deprecation")
102 public class CompressedTableHDU extends BinaryTableHDU {
103
104 private static boolean reversedVLAIndices = false;
105
106 /**
107 * Enables or disables using reversed compressed/uncompressed heap indices for when decompressing variable-length
108 * arrays (VLAs), as was prescribed by the original FITS 4.0 standard and the Pence et al. 2013 convention.
109 * <p>
110 * The <a href="https://fits.gsfc.nasa.gov/standard40/fits_standard40aa-le.pdf">FITS 4.0 standard</a> defines the
111 * convention of compressing variable-length columns in binary tables. On page 50 it writes:
112 * </p>
113 * <blockquote> 2. Append the VLA descriptors from the uncompressed table (which may be either Q-type or P-type) to
114 * the temporary array of VLA descriptors for the compressed table. </blockquote>
115 * <p>
116 * And the original <a href="https://fits.gsfc.nasa.gov/registry/tiletablecompression/tiletable2.0.pdf">Tiled-table
117 * convention</a> by W. Pence et al. also writes:
118 * </p>
119 * <blockquote> [...] we concatenate the array of descriptors from the uncompressed table onto the end of the
120 * temporary array of descriptors (to the compressed VLAs in the compressed table) before the 2 combined arrays of
121 * descriptors are compressed and written into the heap in the compressed table. </blockquote> However, it appears
122 * that the commonly used tools <code>fpack</code> / <code>funpack</code> provided in CFITSIO do just the reverse of
123 * this presciption. They store the uncopressed pointer <i>before</i> the compressed pointers. Because these tools
124 * are widely used we want to support files created or consumed by these tools. As such we allow to deviate from the
125 * FITS standard, and swap the order of the stored VLA indices inside compressed tables.
126 *
127 * @param value <code>true</code> if we should use VLA indices in compressed tables that follow the format described
128 * in the original Pence et al. 2013 convention, and also the FITS 4.0 standard as of 2024 Mar 1;
129 * otherwise <code>false</code>. These original prescriptions define the reverse of what is
130 * actually implemented by CFITSIO and its tools <code>fpack</code> and <code>funpack</code>. (Our
131 * default is to conform to CFITSIO, and the expected revision of the standard to match).
132 *
133 * @see #hasOldStandardVLAIndexing()
134 * @see #asBinaryTableHDU()
135 *
136 * @since 1.19.1
137 */
138 public static void useOldStandardVLAIndexing(boolean value) {
139 reversedVLAIndices = value;
140 }
141
142 /**
143 * Checks if we should use reversed compressed/uncompressed heap indices for when decompressing variable-length
144 * arrays (VLAs).
145 *
146 * @return value <code>true</code> if we will assime VLA indices in compressed tables that follow the format
147 * described in the original Pence et al. 2013 convention, and also the FITS 4.0 standard as of 2024 Mar
148 * 1; otherwise <code>false</code>. These original prescriptions define the reverse of what is actually
149 * implemented by CFITSIO and its tools <code>fpack</code> and <code>funpack</code>. (Our default is to
150 * conform to CFITSIO, and the expcted revision of the standard to match).
151 *
152 * @see #useOldStandardVLAIndexing(boolean)
153 *
154 * @since 1.19.1
155 */
156 public static boolean hasOldStandardVLAIndexing() {
157 return reversedVLAIndices;
158 }
159
160 /**
161 * Prepare a compressed binary table HDU for the specified binary table. When the tile row size is specified with
162 * -1, the value will be set ti the number of rows in the table. The table will be compressed in "rows" that are
163 * defined by the tile size. Next step would be to set the compression options into the HDU and then compress it.
164 *
165 * @param binaryTableHDU the binary table to compress
166 * @param tileRows the number of rows that should be compressed per tile.
167 * @param columnCompressionAlgorithms the compression algorithms to use for the columns (optional default
168 * compression will be used if a column has no compression specified). You
169 * should typically use one or more of the enum values defined in
170 * {@link Compression}. The FITS standard currently allows only the lossless
171 * GZIP_1, GZIP_2, RICE_1, or NOCOMPRESS (e.g.
172 * {@link Compression#ZCMPTYPE_GZIP_1} or
173 * {@link Compression#ZCMPTYPE_NOCOMPRESS}.)
174 *
175 * @return the prepared compressed binary table HDU.
176 *
177 * @throws IllegalArgumentException if any of the listed compression algorithms are not approved for use with
178 * tables.
179 * @throws FitsException if the binary table could not be used to create a compressed binary table.
180 *
181 * @see Compression
182 * @see Compression#ZCMPTYPE_GZIP_1
183 * @see Compression#ZCMPTYPE_GZIP_2
184 * @see Compression#ZCMPTYPE_RICE_1
185 * @see Compression#ZCMPTYPE_NOCOMPRESS
186 * @see #asBinaryTableHDU()
187 * @see #useOldStandardVLAIndexing(boolean)
188 */
189 public static CompressedTableHDU fromBinaryTableHDU(BinaryTableHDU binaryTableHDU, int tileRows,
190 String... columnCompressionAlgorithms) throws FitsException {
191
192 Header header = new Header();
193
194 CompressedTableData compressedData = new CompressedTableData();
195 compressedData.setColumnCompressionAlgorithms(columnCompressionAlgorithms);
196
197 int rowsPerTile = tileRows > 0 ? tileRows : binaryTableHDU.getData().getNRows();
198 compressedData.setRowsPerTile(rowsPerTile);
199
200 Cursor<String, HeaderCard> headerIterator = header.iterator();
201 Cursor<String, HeaderCard> imageIterator = binaryTableHDU.getHeader().iterator();
202 while (imageIterator.hasNext()) {
203 HeaderCard card = imageIterator.next();
204 CompressedCard.restore(card, headerIterator);
205 }
206
207 CompressedTableHDU compressedHDU = new CompressedTableHDU(header, compressedData);
208 compressedData.prepareUncompressedData(binaryTableHDU.getData());
209 compressedData.fillHeader(header);
210
211 return compressedHDU;
212 }
213
214 /**
215 * Check that this HDU has a valid header for this type.
216 *
217 * @deprecated (<i>for internal use</i>) Will reduce visibility in the future
218 *
219 * @param hdr header to check
220 *
221 * @return <CODE>true</CODE> if this HDU has a valid header.
222 */
223 @Deprecated
224 @SuppressFBWarnings(value = "HSM_HIDING_METHOD", justification = "deprecated existing method, kept for compatibility")
225 public static boolean isHeader(Header hdr) {
226 return hdr.getBooleanValue(ZTABLE, false);
227 }
228
229 /**
230 * @deprecated (<i>for internal use</i>) Will reduce visibility in the future
231 *
232 * @param hdr the header that describes the compressed HDU
233 *
234 * @return a new blank data object created from the header description
235 */
236 @Deprecated
237 public static CompressedTableData manufactureData(Header hdr) throws FitsException {
238 return new CompressedTableData(hdr);
239 }
240
241 /**
242 * Creates an new compressed table HDU with the specified header and compressed data.
243 *
244 * @param hdr the header
245 * @param datum the compressed table data. The data may not be actually compressed at this point, int which case you
246 * may need to call {@link #compress()} before writing the new compressed HDU to a stream.
247 *
248 * @see #compress()
249 */
250 public CompressedTableHDU(Header hdr, CompressedTableData datum) {
251 super(hdr, datum);
252 }
253
254 /**
255 * Restores the original binary table HDU by decompressing the data contained in this compresed table HDU.
256 *
257 * @return The uncompressed binary table HDU.
258 *
259 * @throws FitsException If there was an issue with the decompression.
260 *
261 * @see #asBinaryTableHDU(int, int)
262 * @see #fromBinaryTableHDU(BinaryTableHDU, int, String...)
263 * @see #useOldStandardVLAIndexing(boolean)
264 */
265 public BinaryTableHDU asBinaryTableHDU() throws FitsException {
266 return asBinaryTableHDU(0, getTileCount());
267 // Header header = getTableHeader();
268 // BinaryTable data = BinaryTableHDU.manufactureData(header);
269 // BinaryTableHDU tableHDU = new BinaryTableHDU(header, data);
270 // getData().asBinaryTable(data, getHeader(), header);
271 // return tableHDU;
272 }
273
274 /**
275 * Returns the number of table rows that are compressed in each table tile. This may be useful for figuring out what
276 * tiles to decompress, e.g. via {@link #asBinaryTableHDU(int, int)}, when wanting to access select table rows only.
277 * This value is stored under the FITS keyword ZTILELEN in the compressed header. Thus, this method simply provides
278 * a user-friendly way to access it. Note that the last tile may contain fewer rows than the value indicated by this
279 *
280 * @return the number of table rows that are compressed into a tile.
281 *
282 * @throws FitsException if the compressed header does not contain the required ZTILELEN keyword, or it is <= 0.
283 *
284 * @see #asBinaryTableHDU(int, int)
285 *
286 * @since 1.19
287 */
288 public int getTileRows() throws FitsException {
289 int n = getHeader().getIntValue(Compression.ZTILELEN, -1);
290 if (n <= 0) {
291 throw new FitsException("imnvalid or missing ZTILELEN header keyword");
292 }
293 return n;
294 }
295
296 /**
297 * Returns the number of compressed tiles contained in this HDU.
298 *
299 * @return the number of compressed tiles in this table. It is the same as the NAXIS2 value of the header, which is
300 * also returned by {@link #getNRows()} for this compressed table.
301 *
302 * @see #getTileRows()
303 *
304 * @since 1.19
305 */
306 public int getTileCount() {
307 return getNRows();
308 }
309
310 /**
311 * Restores a section of the original binary table HDU by decompressing a selected range of compressed table tiles.
312 * The returned section will start at row index <code>fromTile * getTileRows()</code> of the full table.
313 *
314 * @param fromTile Java index of first tile to decompress
315 * @param toTile Java index of last tile to decompress
316 *
317 * @return The uncompressed binary table HDU from the selected compressed tiles.
318 *
319 * @throws IllegalArgumentException If the tile range is out of bounds
320 * @throws FitsException If there was an issue with the decompression.
321 *
322 * @see #getTileRows()
323 * @see #getTileCount()
324 * @see #asBinaryTableHDU()
325 * @see #fromBinaryTableHDU(BinaryTableHDU, int, String...)
326 * @see #useOldStandardVLAIndexing(boolean)
327 */
328 public BinaryTableHDU asBinaryTableHDU(int fromTile, int toTile) throws FitsException, IllegalArgumentException {
329 Header header = getTableHeader();
330 int tileSize = getTileRows();
331 getData().setRowsPerTile(tileSize);
332
333 if (fromTile < 0 || toTile > getTileCount() || toTile <= fromTile) {
334 throw new IllegalArgumentException(
335 "illegal tile range [" + fromTile + ", " + toTile + "] for " + getTileCount() + " tiles");
336 }
337
338 // Set the correct number of rows
339 int rows = getHeader().getIntValue(Compression.ZNAXISn.n(2));
340 header.addValue(Standard.NAXIS2, Integer.min(rows, toTile * tileSize) - fromTile * tileSize);
341
342 BinaryTable data = BinaryTableHDU.manufactureData(header);
343 BinaryTableHDU tableHDU = new BinaryTableHDU(header, data);
344 getData().asBinaryTable(data, getHeader(), header, fromTile);
345
346 return tableHDU;
347 }
348
349 /**
350 * Restores a section of the original binary table HDU by decompressing a single compressed table tile. The returned
351 * section will start at row index <code>tile * getTileRows()</code> of the full table.
352 *
353 * @param tile Java index of the table tile to decompress
354 *
355 * @return The uncompressed binary table HDU from the selected compressed tile.
356 *
357 * @throws IllegalArgumentException If the tile index is out of bounds
358 * @throws FitsException If there was an issue with the decompression.
359 *
360 * @see #asBinaryTableHDU(int, int)
361 * @see #getTileRows()
362 * @see #getTileCount()
363 * @see #useOldStandardVLAIndexing(boolean)
364 */
365 public final BinaryTableHDU asBinaryTableHDU(int tile) throws FitsException, IllegalArgumentException {
366 return asBinaryTableHDU(tile, tile + 1);
367 }
368
369 /**
370 * Returns a particular section of a decompressed data column.
371 *
372 * @param col the Java column index
373 *
374 * @return The uncompressed column data as an array.
375 *
376 * @throws IllegalArgumentException If the tile range is out of bounds
377 * @throws FitsException If there was an issue with the decompression.
378 *
379 * @see #getColumnData(int, int, int)
380 * @see #asBinaryTableHDU()
381 * @see #fromBinaryTableHDU(BinaryTableHDU, int, String...)
382 */
383 public Object getColumnData(int col) throws FitsException, IllegalArgumentException {
384 return getColumnData(col, 0, getTileCount());
385 }
386
387 /**
388 * Returns a particular section of a decompressed data column.
389 *
390 * @param col the Java column index
391 * @param fromTile the Java index of first tile to decompress
392 * @param toTile the Java index of last tile to decompress
393 *
394 * @return The uncompressed column data segment as an array.
395 *
396 * @throws IllegalArgumentException If the tile range is out of bounds
397 * @throws FitsException If there was an issue with the decompression.
398 *
399 * @see #getColumnData(int)
400 * @see #asBinaryTableHDU()
401 * @see #fromBinaryTableHDU(BinaryTableHDU, int, String...)
402 */
403 public Object getColumnData(int col, int fromTile, int toTile) throws FitsException, IllegalArgumentException {
404 getData().setRowsPerTile(getTileRows());
405 return getData().getColumnData(col, fromTile, toTile, getHeader(), getTableHeader());
406 }
407
408 /**
409 * Performs the actual compression with the selected algorithm(s) and options. When creating a compressed table HDU,
410 * e.g using the {@link #fromBinaryTableHDU(BinaryTableHDU, int, String...)} method, the HDU is merely prepared but
411 * without actually performing the compression, and this method will have to be called to actually perform the
412 * compression. The design would allow for setting options between creation and compressing, but in this case there
413 * is really nothing of the sort.
414 *
415 * @return itself
416 *
417 * @throws FitsException if the compression could not be performed
418 *
419 * @see #fromBinaryTableHDU(BinaryTableHDU, int, String...)
420 * @see #useOldStandardVLAIndexing(boolean)
421 */
422 public CompressedTableHDU compress() throws FitsException {
423 getData().compress(getHeader());
424 return this;
425 }
426
427 /**
428 * Obtain a header representative of a decompressed TableHDU.
429 *
430 * @return Header with decompressed cards.
431 *
432 * @throws HeaderCardException if the card could not be copied
433 *
434 * @since 1.18
435 */
436 public Header getTableHeader() throws HeaderCardException {
437 Header header = new Header();
438
439 header.addValue(Standard.XTENSION, Standard.XTENSION_BINTABLE);
440 header.addValue(Standard.BITPIX, ElementType.BYTE.bitPix());
441 header.addValue(Standard.NAXIS, 2);
442
443 Cursor<String, HeaderCard> tableIterator = header.iterator();
444 Cursor<String, HeaderCard> iterator = getHeader().iterator();
445
446 while (iterator.hasNext()) {
447 CompressedCard.backup(iterator.next(), tableIterator);
448 }
449
450 return header;
451 }
452
453 @Override
454 public CompressedTableData getData() {
455 return (CompressedTableData) super.getData();
456 }
457
458 }