View Javadoc
1   package nom.tam.image.compression.hdu;
2   
3   import java.util.ArrayList;
4   import java.util.Arrays;
5   import java.util.List;
6   import java.util.Locale;
7   
8   import nom.tam.fits.BinaryTable;
9   import nom.tam.fits.FitsException;
10  import nom.tam.fits.FitsFactory;
11  import nom.tam.fits.Header;
12  import nom.tam.fits.header.Compression;
13  import nom.tam.fits.header.Standard;
14  import nom.tam.image.compression.bintable.BinaryTableTile;
15  import nom.tam.image.compression.bintable.BinaryTableTileCompressor;
16  import nom.tam.image.compression.bintable.BinaryTableTileDecompressor;
17  import nom.tam.image.compression.bintable.BinaryTableTileDescription;
18  import nom.tam.util.ColumnTable;
19  
20  /*
21   * #%L
22   * nom.tam FITS library
23   * %%
24   * Copyright (C) 1996 - 2024 nom-tam-fits
25   * %%
26   * This is free and unencumbered software released into the public domain.
27   *
28   * Anyone is free to copy, modify, publish, use, compile, sell, or
29   * distribute this software, either in source code form or as a compiled
30   * binary, for any purpose, commercial or non-commercial, and by any
31   * means.
32   *
33   * In jurisdictions that recognize copyright laws, the author or authors
34   * of this software dedicate any and all copyright interest in the
35   * software to the public domain. We make this dedication for the benefit
36   * of the public at large and to the detriment of our heirs and
37   * successors. We intend this dedication to be an overt act of
38   * relinquishment in perpetuity of all present and future rights to this
39   * software under copyright law.
40   *
41   * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
42   * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
43   * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
44   * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
45   * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
46   * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
47   * OTHER DEALINGS IN THE SOFTWARE.
48   * #L%
49   */
50  
51  import static nom.tam.fits.header.Standard.TFIELDS;
52  import static nom.tam.image.compression.bintable.BinaryTableTileDescription.tile;
53  
54  /**
55   * FITS representation of a compressed binary table. It itself is a binary table, but one in which each row represents
56   * the compressed image of one or more rows of the original table.
57   * 
58   * @see CompressedTableHDU
59   */
60  @SuppressWarnings("deprecation")
61  public class CompressedTableData extends BinaryTable {
62  
63      private static final List<String> ALLOWED_ALGORITHMS = Arrays.asList(Compression.ZCMPTYPE_GZIP_1,
64              Compression.ZCMPTYPE_GZIP_2, Compression.ZCMPTYPE_RICE_1, Compression.ZCMPTYPE_NOCOMPRESS);
65  
66      private int rowsPerTile;
67  
68      private List<BinaryTableTile> tiles;
69  
70      private BinaryTable orig;
71  
72      /** Only add new var-length column in the preparation step once */
73      private boolean isPrepped;
74  
75      private String[] colAlgorithm;
76  
77      /**
78       * Creates a new empty compressed table data to be initialized at a later point
79       */
80      public CompressedTableData() {
81      }
82  
83      /**
84       * Creates a new compressed table data based on the prescription of the supplied header.
85       * 
86       * @param  header        The header that describes the compressed table
87       * 
88       * @throws FitsException If the header is invalid or could not be accessed.
89       */
90      public CompressedTableData(Header header) throws FitsException {
91          super(header);
92          rowsPerTile = header.getIntValue(Compression.ZTILELEN, header.getIntValue(Standard.NAXIS2));
93          setColumnCompressionAlgorithms(header);
94      }
95  
96      /**
97       * (<i>for internal use</i>) This should only be called by {@link CompressedTableHDU}, and should have reduced
98       * visibility accordingly.
99       */
100     @SuppressWarnings("javadoc")
101     public void compress(Header header) throws FitsException {
102         discardVLAs();
103 
104         // If table has only fixed-length data, we can compress in parallel, and defragment after.
105         for (BinaryTableTile tile : tiles) {
106             tile.execute(FitsFactory.threadPool());
107         }
108 
109         for (BinaryTableTile tile : tiles) {
110             tile.waitForResult();
111         }
112     }
113 
114     @Override
115     public synchronized long defragment() throws FitsException {
116         if (orig != null && orig.containsHeap()) {
117             // Don't defragment if the original had VLAs, since these are stored on the heap
118             // with a dual-set of descriptors, includeing compressed ones on the heap itself
119             // which are not trivial to de-fragment.
120             return 0L;
121         }
122         return super.defragment();
123     }
124 
125     @Override
126     public void fillHeader(Header h) throws FitsException {
127         super.fillHeader(h);
128 
129         h.setNaxis(2, getData().getNRows());
130         h.addValue(Compression.ZTABLE, true);
131         h.addValue(Compression.ZTILELEN, getRowsPerTile());
132 
133         for (int i = 0; i < getNCols(); i++) {
134             h.findCard(Compression.ZFORMn.n(i + 1));
135             h.addValue(Compression.ZCTYPn.n(i + 1), getAlgorithm(i));
136         }
137 
138         h.deleteKey(Compression.ZIMAGE);
139     }
140 
141     void prepareUncompressedData(BinaryTable fromTable) throws FitsException {
142         orig = fromTable;
143         prepareUncompressedData(orig.getData());
144     }
145 
146     /**
147      * @deprecated (<i>for internal use</i>) This should only be called by {@link CompressedTableHDU}, and its
148      *                 visibility will be reduced accordingly in the future, not to mention that it should take a
149      *                 BinaryTable as its argument.
150      */
151     @SuppressWarnings("javadoc")
152     public void prepareUncompressedData(ColumnTable<?> data) throws FitsException {
153         tiles = new ArrayList<>();
154 
155         int nrows = data.getNRows();
156         int ncols = data.getNCols();
157 
158         if (!isPrepped) {
159             // Create compressed columns...
160             for (int column = 0; column < ncols; column++) {
161                 addColumn(BinaryTable.ColumnDesc.createForVariableSize(byte.class));
162             }
163 
164             // Initialized compressed rows...
165             for (int rowStart = 0; rowStart < nrows; rowStart += getRowsPerTile()) {
166                 addRow(new byte[ncols][0]);
167             }
168         }
169 
170         // Changed tile-order in 1.19.1 to be in row-major table order.
171         for (int column = 0; column < ncols; column++) {
172             for (int tileIndex = 0, rowStart = 0; rowStart < nrows; tileIndex++, rowStart += getRowsPerTile()) {
173 
174                 BinaryTableTileDescription td = tile()//
175                         .rowStart(rowStart)//
176                         .rowEnd(Math.min(nrows, rowStart + getRowsPerTile()))//
177                         .column(column)//
178                         .tileIndex(tileIndex + 1)//
179                         .compressionAlgorithm(getAlgorithm(column));
180 
181                 BinaryTableTileCompressor tile = (orig == null) ? new BinaryTableTileCompressor(this, data, td) :
182                         new BinaryTableTileCompressor(this, orig, td);
183 
184                 tiles.add(tile);
185             }
186         }
187 
188         isPrepped = true;
189     }
190 
191     /**
192      * (<i>for internal use</i>) No longer used, and it may be removed in the future.
193      */
194     @SuppressWarnings("javadoc")
195     protected BinaryTable asBinaryTable(BinaryTable toTable, Header compressedHeader, Header targetHeader)
196             throws FitsException {
197         return asBinaryTable(toTable, compressedHeader, targetHeader, 0);
198     }
199 
200     BinaryTable asBinaryTable(BinaryTable toTable, Header compressedHeader, Header targetHeader, int fromTile)
201             throws FitsException {
202         int nrows = targetHeader.getIntValue(Standard.NAXIS2);
203         int ncols = compressedHeader.getIntValue(TFIELDS);
204         int tileSize = compressedHeader.getIntValue(Compression.ZTILELEN, nrows);
205 
206         ensureData();
207         setColumnCompressionAlgorithms(compressedHeader);
208 
209         BinaryTable.createColumnDataFor(toTable);
210 
211         List<BinaryTableTile> tileList = new ArrayList<>();
212 
213         for (int tileIndex = fromTile, rowStart = 0; rowStart < nrows; tileIndex++, rowStart += tileSize) {
214             for (int column = 0; column < ncols; column++) {
215                 BinaryTableTileDecompressor tile = new BinaryTableTileDecompressor(this, toTable, tile()//
216                         .rowStart(rowStart)//
217                         .rowEnd(Math.min(nrows, rowStart + tileSize))//
218                         .column(column)//
219                         .tileIndex(tileIndex + 1)//
220                         .compressionAlgorithm(getAlgorithm(column)));
221                 tileList.add(tile);
222 
223                 tile.execute(FitsFactory.threadPool());
224             }
225         }
226 
227         for (BinaryTableTile tile : tileList) {
228             tile.waitForResult();
229         }
230 
231         return toTable;
232     }
233 
234     Object getColumnData(int col, int fromTile, int toTile, Header compressedHeader, Header targetHeader)
235             throws FitsException {
236 
237         if (fromTile < 0 || fromTile >= getNRows()) {
238             throw new IllegalArgumentException("start tile " + fromTile + " is outof bounds for " + getNRows() + " tiles.");
239         }
240 
241         if (toTile > getNRows()) {
242             throw new IllegalArgumentException("end tile " + toTile + " is outof bounds for " + getNRows() + " tiles.");
243         }
244 
245         if (toTile <= fromTile) {
246             return null;
247         }
248 
249         setColumnCompressionAlgorithms(compressedHeader);
250 
251         int nr = targetHeader.getIntValue(Standard.NAXIS2);
252 
253         int tileSize = compressedHeader.getIntValue(Compression.ZTILELEN, nr);
254         int nRows = (toTile - fromTile) * tileSize;
255 
256         if (nRows > nr) {
257             nRows = nr;
258         }
259 
260         ColumnDesc c = getDescriptor(targetHeader, col);
261         class UncompressedTable extends BinaryTable {
262             @Override
263             public void createTable(int nRows) throws FitsException {
264                 super.createTable(nRows);
265             }
266         }
267 
268         UncompressedTable data = new UncompressedTable();
269         data.addColumn(c);
270         data.createTable(nRows);
271 
272         List<BinaryTableTile> tileList = new ArrayList<>();
273 
274         String algorithm = compressedHeader.getStringValue(Compression.ZCTYPn.n(col + 1));
275 
276         for (int tileIndex = fromTile, rowStart = 0; rowStart < nRows; tileIndex++, rowStart += tileSize) {
277             BinaryTableTileDecompressor tile = new BinaryTableTileDecompressor(this, data, tile()//
278                     .rowStart(rowStart)//
279                     .rowEnd(Math.min(nr, rowStart + tileSize))//
280                     .column(col)//
281                     .tileIndex(tileIndex + 1)//
282                     .compressionAlgorithm(algorithm));
283             tile.decompressToColumn(0);
284             tileList.add(tile);
285 
286             tile.execute(FitsFactory.threadPool());
287         }
288 
289         for (BinaryTableTile tile : tileList) {
290             tile.waitForResult();
291         }
292 
293         return data.getColumn(0);
294     }
295 
296     /**
297      * Returns the number of original (uncompressed) table rows that are compressed as a block into a single compressed
298      * table row.
299      * 
300      * @return the number of table rows compressed together as a block.
301      */
302     protected final int getRowsPerTile() {
303         return rowsPerTile;
304     }
305 
306     private String getAlgorithm(int column) {
307         if (colAlgorithm != null && column < colAlgorithm.length && colAlgorithm[column] != null) {
308             return colAlgorithm[column];
309         }
310         return Compression.ZCMPTYPE_GZIP_2;
311     }
312 
313     /**
314      * (<i>for internal use</i>) Visibility may be reduced to the package level. This should only be called by
315      * {@link CompressedTableHDU}.
316      */
317     @SuppressWarnings("javadoc")
318     protected void setColumnCompressionAlgorithms(String[] columnCompressionAlgorithms) {
319         for (String algo : columnCompressionAlgorithms) {
320             if (!ALLOWED_ALGORITHMS.contains(algo.toUpperCase(Locale.US))) {
321                 throw new IllegalArgumentException(algo + " cannot be used to compress tables.");
322             }
323         }
324 
325         this.colAlgorithm = columnCompressionAlgorithms;
326     }
327 
328     private void setColumnCompressionAlgorithms(Header header) {
329         int ncols = header.getIntValue(TFIELDS);
330 
331         // Default compression algorithm, unless specified...
332         colAlgorithm = new String[ncols];
333 
334         // Set the compression algorithms specified.
335         for (int column = 0; column < ncols; column++) {
336             colAlgorithm[column] = header.getStringValue(Compression.ZCTYPn.n(column + 1));
337         }
338     }
339 
340     /**
341      * (<i>for internal use</i>) Visibility may be reduced to the package level. This should only be called by
342      * {@link CompressedTableHDU}.
343      */
344     @SuppressWarnings("javadoc")
345     protected CompressedTableData setRowsPerTile(int value) {
346         rowsPerTile = value;
347         return this;
348     }
349 }