View Javadoc
1   package nom.tam.fits.compression.algorithm.rice;
2   
3   import java.nio.Buffer;
4   import java.nio.ByteBuffer;
5   import java.nio.IntBuffer;
6   import java.nio.ShortBuffer;
7   import java.util.logging.Logger;
8   
9   import nom.tam.fits.compression.algorithm.api.ICompressor;
10  import nom.tam.fits.compression.algorithm.quant.QuantizeProcessor.DoubleQuantCompressor;
11  import nom.tam.fits.compression.algorithm.quant.QuantizeProcessor.FloatQuantCompressor;
12  import nom.tam.util.FitsIO;
13  import nom.tam.util.type.ElementType;
14  
15  /*
16   * #%L
17   * nom.tam FITS library
18   * %%
19   * Copyright (C) 1996 - 2024 nom-tam-fits
20   * %%
21   * This is free and unencumbered software released into the public domain.
22   *
23   * Anyone is free to copy, modify, publish, use, compile, sell, or
24   * distribute this software, either in source code form or as a compiled
25   * binary, for any purpose, commercial or non-commercial, and by any
26   * means.
27   *
28   * In jurisdictions that recognize copyright laws, the author or authors
29   * of this software dedicate any and all copyright interest in the
30   * software to the public domain. We make this dedication for the benefit
31   * of the public at large and to the detriment of our heirs and
32   * successors. We intend this dedication to be an overt act of
33   * relinquishment in perpetuity of all present and future rights to this
34   * software under copyright law.
35   *
36   * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
37   * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
38   * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
39   * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
40   * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
41   * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
42   * OTHER DEALINGS IN THE SOFTWARE.
43   * #L%
44   */
45  
46  /**
47   * (<i>for internal use</i>) The Rice compression algorithm. The original compression was designed by Rice, Yeh, and
48   * Miller the code was written by Richard White at STScI at the STScI and included (ported to c and adapted) in cfitsio
49   * by William Pence, NASA/GSFC. That code was then ported to java by R. van Nieuwenhoven. Later it was massively
50   * refactored to harmonize the different compression algorithms and reduce the duplicate code pieces without obscuring
51   * the algorithm itself as far as possible.
52   *
53   * @author     Richard White
54   * @author     William Pence
55   * @author     Richard van Nieuwenhoven
56   *
57   * @param  <T> the genetic type of NIO buffer on which this compressor operates.
58   */
59  @SuppressWarnings({"deprecation", "javadoc"})
60  public abstract class RiceCompressor<T extends Buffer> implements ICompressor<T> {
61  
62      public static class ByteRiceCompressor extends RiceCompressor<ByteBuffer> {
63  
64          private ByteBuffer pixelBuffer;
65  
66          /**
67           * Rice compression of byte streams with the default block size of 32.
68           * 
69           * @since 1.19.1
70           */
71          public ByteRiceCompressor() {
72              this(new RiceCompressOption());
73          }
74  
75          public ByteRiceCompressor(RiceCompressOption option) {
76              super(option);
77          }
78  
79          @Override
80          public boolean compress(ByteBuffer buffer, ByteBuffer writeBuffer) {
81              pixelBuffer = buffer;
82              super.compress(buffer.limit(), pixelBuffer.get(pixelBuffer.position()), new BitBuffer(writeBuffer));
83              return true;
84          }
85  
86          @Override
87          public void decompress(ByteBuffer readBuffer, ByteBuffer buffer) {
88              pixelBuffer = buffer;
89              super.decompressBuffer(readBuffer, buffer.limit());
90          }
91  
92          @Override
93          protected int nextPixel() {
94              return pixelBuffer.get();
95          }
96  
97          @Override
98          protected void nextPixel(int pixel) {
99              pixelBuffer.put((byte) pixel);
100         }
101     }
102 
103     public static class DoubleRiceCompressor extends DoubleQuantCompressor {
104 
105         public DoubleRiceCompressor(RiceQuantizeCompressOption options) throws ClassCastException {
106             super(options, new IntRiceCompressor((RiceCompressOption) options.getCompressOption()));
107         }
108     }
109 
110     public static class FloatRiceCompressor extends FloatQuantCompressor {
111 
112         public FloatRiceCompressor(RiceQuantizeCompressOption options) throws ClassCastException {
113             super(options, new IntRiceCompressor((RiceCompressOption) options.getCompressOption()));
114         }
115     }
116 
117     public static class IntRiceCompressor extends RiceCompressor<IntBuffer> {
118 
119         private IntBuffer pixelBuffer;
120 
121         /**
122          * Rice compression of 32-bit integer streams with the default block size of 32.
123          * 
124          * @since 1.19.1
125          */
126         public IntRiceCompressor() {
127             this(new RiceCompressOption());
128         }
129 
130         public IntRiceCompressor(RiceCompressOption option) {
131             super(option);
132         }
133 
134         @Override
135         public boolean compress(IntBuffer buffer, ByteBuffer writeBuffer) {
136             pixelBuffer = buffer;
137             super.compress(buffer.limit(), pixelBuffer.get(pixelBuffer.position()), new BitBuffer(writeBuffer));
138             return true;
139         }
140 
141         @Override
142         public void decompress(ByteBuffer readBuffer, IntBuffer buffer) {
143             pixelBuffer = buffer;
144             super.decompressBuffer(readBuffer, buffer.limit());
145         }
146 
147         @Override
148         protected int nextPixel() {
149             return pixelBuffer.get();
150         }
151 
152         @Override
153         protected void nextPixel(int pixel) {
154             pixelBuffer.put(pixel);
155         }
156     }
157 
158     public static class ShortRiceCompressor extends RiceCompressor<ShortBuffer> {
159 
160         private ShortBuffer pixelBuffer;
161 
162         /**
163          * Rice compression of 16-bit integer streams with the default block size of 32.
164          * 
165          * @since 1.19.1
166          */
167         public ShortRiceCompressor() {
168             this(new RiceCompressOption());
169         }
170 
171         public ShortRiceCompressor(RiceCompressOption option) {
172             super(option);
173         }
174 
175         @Override
176         public boolean compress(ShortBuffer buffer, ByteBuffer writeBuffer) {
177             pixelBuffer = buffer;
178             super.compress(buffer.limit(), pixelBuffer.get(pixelBuffer.position()), new BitBuffer(writeBuffer));
179             return true;
180         }
181 
182         @Override
183         public void decompress(ByteBuffer readBuffer, ShortBuffer buffer) {
184             pixelBuffer = buffer;
185             super.decompressBuffer(readBuffer, buffer.limit());
186         }
187 
188         @Override
189         protected int nextPixel() {
190             return pixelBuffer.get();
191         }
192 
193         @Override
194         protected void nextPixel(int pixel) {
195             pixelBuffer.put((short) pixel);
196         }
197     }
198 
199     /**
200      * mask to convert a "unsigned" byte to a long.
201      */
202     private static final long UNSIGNED_BYTE_MASK = 0xFFL;
203 
204     /**
205      * mask to convert a "unsigned" short to a long.
206      */
207     private static final long UNSIGNED_SHORT_MASK = 0xFFFFL;
208 
209     /**
210      * mask to convert a "unsigned" int to a long.
211      */
212     private static final long UNSIGNED_INTEGER_MASK = 0xFFFFFFFFL;
213 
214     /**
215      * logger to log to.
216      */
217     private static final Logger LOG = Logger.getLogger(RiceCompressor.class.getName());
218 
219     private static final int BITS_OF_1_BYTE = 8;
220 
221     private static final int BITS_PER_BYTE = 8;
222 
223     private static final int BYTE_MASK = 0xff;
224 
225     private static final int FS_BITS_FOR_BYTE = 3;
226 
227     private static final int FS_BITS_FOR_INT = 5;
228 
229     private static final int FS_BITS_FOR_SHORT = 4;
230 
231     private static final int FS_MAX_FOR_BYTE = 6;
232 
233     private static final int FS_MAX_FOR_INT = 25;
234 
235     private static final int FS_MAX_FOR_SHORT = 14;
236 
237     /*
238      * nonzero_count is lookup table giving number of bits in 8-bit values not including leading zeros used in
239      * fits_rdecomp, fits_rdecomp_short and fits_rdecomp_byte.
240      *
241      * @formatter:off
242      */
243     private static final int[] NONZERO_COUNT = {0, 1, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5,
244             5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
245             6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
246             7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8,
247             8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,
248             8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,
249             8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,
250             8, 8, 8, 8, 8, 8, 8, 8, 8};
251     // @formatter:on
252 
253     private final int bBits;
254 
255     private final int bitsPerPixel;
256 
257     private final int blockSize;
258 
259     private final int fsBits;
260 
261     private final int fsMax;
262 
263     private RiceCompressor(RiceCompressOption option) throws UnsupportedOperationException {
264         blockSize = option.getBlockSize();
265         if (option.getBytePix() == ElementType.BYTE.size()) {
266             fsBits = FS_BITS_FOR_BYTE;
267             fsMax = FS_MAX_FOR_BYTE;
268             bitsPerPixel = FitsIO.BITS_OF_1_BYTE;
269         } else if (option.getBytePix() == ElementType.SHORT.size()) {
270             fsBits = FS_BITS_FOR_SHORT;
271             fsMax = FS_MAX_FOR_SHORT;
272             bitsPerPixel = FitsIO.BITS_OF_2_BYTES;
273         } else if (option.getBytePix() == ElementType.INT.size()) {
274             fsBits = FS_BITS_FOR_INT;
275             fsMax = FS_MAX_FOR_INT;
276             bitsPerPixel = FitsIO.BITS_OF_4_BYTES;
277         } else {
278             throw new UnsupportedOperationException("Implemented for 1/2/4 bytes only");
279         }
280         /*
281          * From bsize derive: FSBITS = # bits required to store FS FSMAX = maximum value for FS BBITS = bits/pixel for
282          * direct coding
283          */
284         bBits = 1 << fsBits;
285     }
286 
287     /**
288      * <p>
289      * undo mapping and differencing Note that some of these operations will overflow the unsigned int arithmetic --
290      * that's OK, it all works out to give the right answers in the output file.
291      * </p>
292      * <p>
293      * In java this is more complicated because of the missing unsigned integers. trying to simulate the behavior
294      * </p>
295      *
296      * @param  lastpix the current last pix value
297      * @param  diff    the difference to "add"
298      *
299      * @return         return the new lastpiy value
300      */
301     private long undoMappingAndDifferencing(long lastpix, long diff) {
302         diff &= UNSIGNED_INTEGER_MASK;
303         if ((diff & 1) == 0) {
304             diff = diff >>> 1;
305         } else {
306             diff = diff >>> 1 ^ UNSIGNED_INTEGER_MASK;
307         }
308         lastpix = diff + lastpix & UNSIGNED_INTEGER_MASK;
309         nextPixel((int) lastpix);
310         return lastpix;
311     }
312 
313     /**
314      * compress the integer tiledImageOperation on a rise compressed byte buffer.
315      *
316      * @param dataLength length of the data to compress
317      * @param firstPixel the value of the first pixel
318      * @param buffer     the buffer to write to
319      */
320     protected void compress(final int dataLength, int firstPixel, BitBuffer buffer) {
321         /* the first difference will always be zero */
322         int lastpix = firstPixel;
323         /* write out first int value to the first 4 bytes of the buffer */
324         buffer.putInt(firstPixel, bitsPerPixel);
325         int thisblock = blockSize;
326         for (int i = 0; i < dataLength; i += blockSize) {
327             /* last block may be shorter */
328             if (dataLength - i < blockSize) {
329                 thisblock = dataLength - i;
330             }
331             /*
332              * Compute differences of adjacent pixels and map them to unsigned values. Note that this may overflow the
333              * integer variables -- that's OK, because we can recover when decompressing. If we were compressing shorts
334              * or bytes, would want to do this arithmetic with short/byte working variables (though diff will still be
335              * passed as an int.) compute sum of mapped pixel values at same time use double precision for sum to allow
336              * 32-bit integer inputs
337              */
338             long[] diff = new long[blockSize];
339             double pixelsum = 0.0;
340             int nextpix;
341             /*
342              * tiledImageOperation for differences mapped to non-negative values
343              */
344             for (int j = 0; j < thisblock; j++) {
345                 nextpix = nextPixel();
346                 long pdiff = (nextpix - lastpix);
347                 diff[j] = (pdiff < 0 ? (pdiff << 1) ^ UNSIGNED_INTEGER_MASK : pdiff << 1) & UNSIGNED_INTEGER_MASK;
348                 pixelsum += diff[j];
349                 lastpix = nextpix;
350             }
351 
352             /*
353              * compute number of bits to split from sum
354              */
355             double dpsum = (pixelsum - thisblock / 2d - 1d) / thisblock;
356             if (dpsum < 0) {
357                 dpsum = 0.0;
358             }
359             long psum = (long) dpsum >> 1;
360             int fs;
361             for (fs = 0; psum > 0; fs++) { // NOSONAR
362                 psum >>= 1;
363             }
364 
365             /*
366              * write the codes fsbits ID bits used to indicate split level
367              */
368             if (fs >= fsMax) {
369                 /*
370                  * Special high entropy case when FS >= fsmax Just write pixel difference values directly, no Rice
371                  * coding at all.
372                  */
373                 buffer.putInt(fsMax + 1, fsBits);
374                 for (int j = 0; j < thisblock; j++) {
375                     buffer.putLong(diff[j], bBits);
376                 }
377             } else if (fs == 0 && pixelsum == 0) { // NOSONAR
378                 /*
379                  * special low entropy case when FS = 0 and pixelsum=0 (all pixels in block are zero.) Output a 0 and
380                  * return
381                  */
382                 buffer.putInt(0, fsBits);
383             } else {
384                 /* normal case: not either very high or very low entropy */
385                 buffer.putInt(fs + 1, fsBits);
386                 int fsmask = (1 << fs) - 1;
387                 /*
388                  * local copies of bit buffer to improve optimization
389                  */
390                 int bitsToGo = buffer.missingBitsInCurrentByte();
391                 int bitBuffer = buffer.bitbuffer() >> bitsToGo;
392                 buffer.movePosition(bitsToGo - BITS_OF_1_BYTE);
393                 for (int j = 0; j < thisblock; j++) {
394                     int v = (int) diff[j];
395                     int top = v >> fs;
396                     /*
397                      * top is coded by top zeros + 1
398                      */
399                     if (bitsToGo >= top + 1) {
400                         bitBuffer <<= top + 1;
401                         bitBuffer |= 1;
402                         bitsToGo -= top + 1;
403                     } else {
404                         bitBuffer <<= bitsToGo;
405                         buffer.putByte((byte) (bitBuffer & BYTE_MASK));
406                         for (top -= bitsToGo; top >= BITS_OF_1_BYTE; top -= BITS_OF_1_BYTE) {
407                             buffer.putByte((byte) 0);
408                         }
409                         bitBuffer = 1;
410                         bitsToGo = BITS_OF_1_BYTE - 1 - top;
411                     }
412                     /*
413                      * bottom FS bits are written without coding code is output_nbits, moved into this routine to reduce
414                      * overheads This code potentially breaks if FS>24, so I am limiting FS to 24 by choice of FSMAX
415                      * above.
416                      */
417                     if (fs > 0) {
418                         bitBuffer <<= fs;
419                         bitBuffer |= v & fsmask;
420                         bitsToGo -= fs;
421                         while (bitsToGo <= 0) {
422                             buffer.putByte((byte) (bitBuffer >> -bitsToGo & BYTE_MASK));
423                             bitsToGo += BITS_OF_1_BYTE;
424                         }
425                     }
426                 }
427                 buffer.putByte((byte) (bitBuffer & BYTE_MASK), BITS_OF_1_BYTE - bitsToGo);
428             }
429         }
430         buffer.close();
431     }
432 
433     /**
434      * decompress the readbuffer and fill the pixelarray.
435      *
436      * @param readBuffer input buffer
437      * @param nx         the number of pixel to uncompress
438      */
439     protected void decompressBuffer(final ByteBuffer readBuffer, final int nx) {
440         /* first x bytes of input buffer contain the value of the first */
441         /* x byte integer value, without any encoding */
442         long lastpix = 0L;
443         if (bitsPerPixel == ElementType.BYTE.bitPix()) {
444             lastpix = readBuffer.get() & UNSIGNED_BYTE_MASK;
445         } else if (bitsPerPixel == ElementType.SHORT.bitPix()) {
446             lastpix = readBuffer.getShort() & UNSIGNED_SHORT_MASK;
447         } else {
448             // Must be (this.bitsPerPixel == ElementType.INT.bitPix())
449             lastpix = readBuffer.getInt() & UNSIGNED_INTEGER_MASK;
450         }
451         long b = readBuffer.get() & BYTE_MASK; /* bit buffer */
452         int nbits = BITS_PER_BYTE; /* number of bits remaining in b */
453         for (int i = 0; i < nx;) {
454             /* get the FS value from first fsbits */
455             nbits -= fsBits;
456             while (nbits < 0) {
457                 b = b << BITS_PER_BYTE | readBuffer.get() & BYTE_MASK;
458                 nbits += BITS_PER_BYTE;
459             }
460             long fs = (b >>> nbits) - 1L;
461 
462             b &= (1 << nbits) - 1;
463             /* loop over the next block */
464             int imax = i + blockSize;
465             if (imax > nx) {
466                 imax = nx;
467             }
468             if (fs < 0) {
469                 /* low-entropy case, all zero differences */
470                 for (; i < imax; i++) {
471                     nextPixel((int) lastpix);
472                 }
473             } else if (fs == fsMax) {
474                 /* high-entropy case, directly coded pixel values */
475                 for (; i < imax; i++) {
476                     int k = bBits - nbits;
477                     long diff = b << k;
478                     for (k -= BITS_PER_BYTE; k >= 0; k -= BITS_PER_BYTE) {
479                         b = readBuffer.get() & BYTE_MASK;
480                         diff |= b << k;
481                     }
482                     if (nbits > 0) {
483                         b = readBuffer.get() & BYTE_MASK;
484                         diff |= b >>> -k;
485                         b &= (1 << nbits) - 1L;
486                     } else {
487                         b = 0;
488                     }
489                     lastpix = undoMappingAndDifferencing(lastpix, diff);
490                 }
491             } else {
492                 /* normal case, Rice coding */
493                 for (; i < imax; i++) {
494                     /* count number of leading zeros */
495                     while (b == 0) {
496                         nbits += BITS_PER_BYTE;
497                         b = readBuffer.get() & BYTE_MASK;
498                     }
499                     long nzero = nbits - NONZERO_COUNT[(int) (b & BYTE_MASK)];
500                     nbits -= nzero + 1;
501                     /* flip the leading one-bit */
502                     b ^= 1 << nbits;
503                     /* get the FS trailing bits */
504                     nbits -= fs;
505                     while (nbits < 0) {
506                         b = b << BITS_PER_BYTE | readBuffer.get() & BYTE_MASK;
507                         nbits += BITS_PER_BYTE;
508                     }
509                     long diff = nzero << fs | b >> nbits;
510                     b &= (1 << nbits) - 1L;
511 
512                     lastpix = undoMappingAndDifferencing(lastpix, diff);
513                 }
514             }
515         }
516         if (readBuffer.limit() > readBuffer.position()) {
517             LOG.warning("decompressing left over some extra bytes got: " + readBuffer.limit() + " but needed only "
518                     + readBuffer.position());
519         }
520 
521     }
522 
523     protected abstract int nextPixel();
524 
525     protected abstract void nextPixel(int pixel);
526 
527 }