View Javadoc
1   package nom.tam.fits.compression.algorithm.quant;
2   
3   import java.nio.Buffer;
4   import java.nio.DoubleBuffer;
5   import java.nio.FloatBuffer;
6   
7   /*
8    * #%L
9    * nom.tam FITS library
10   * %%
11   * Copyright (C) 1996 - 2024 nom-tam-fits
12   * %%
13   * This is free and unencumbered software released into the public domain.
14   *
15   * Anyone is free to copy, modify, publish, use, compile, sell, or
16   * distribute this software, either in source code form or as a compiled
17   * binary, for any purpose, commercial or non-commercial, and by any
18   * means.
19   *
20   * In jurisdictions that recognize copyright laws, the author or authors
21   * of this software dedicate any and all copyright interest in the
22   * software to the public domain. We make this dedication for the benefit
23   * of the public at large and to the detriment of our heirs and
24   * successors. We intend this dedication to be an overt act of
25   * relinquishment in perpetuity of all present and future rights to this
26   * software under copyright law.
27   *
28   * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
29   * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
30   * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
31   * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
32   * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
33   * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
34   * OTHER DEALINGS IN THE SOFTWARE.
35   * #L%
36   */
37  
38  import java.util.Arrays;
39  
40  /**
41   * (<i>for internal use</i>) Determines the optimal quantization to use for floating-point data. It estimates the noise
42   * level in the data to determine qhat quantization should be use to lose no information above the noise level.
43   * 
44   * @deprecated (<i>for internal use</i>) This class sohuld have visibility reduced to the package level
45   */
46  @Deprecated
47  @SuppressWarnings("javadoc")
48  public class Quantize {
49  
50      private static final double DEFAULT_QUANT_LEVEL = 4.;
51  
52      private static final int MINIMUM_PIXEL_WIDTH = 9;
53  
54      private static final int N4 = 4;
55  
56      private static final int N6 = 6;
57  
58      private static final double NOISE_2_MULTIPLICATOR = 1.0483579;
59  
60      private static final double NOISE_3_MULTIPLICATOR = 0.6052697;
61  
62      private static final double NOISE_5_MULTIPLICATOR = 0.1772048;
63  
64      private final QuantizeOption parameter;
65  
66      /**
67       * maximum non-null value
68       */
69      private double maxValue;
70  
71      /**
72       * minimum non-null value
73       */
74      private double minValue;
75  
76      /**
77       * number of good, non-null pixels?
78       */
79      private long ngood;
80  
81      /**
82       * returned 2nd order MAD of all non-null pixels
83       */
84      private double noise2;
85  
86      /**
87       * returned 3rd order MAD of all non-null pixels
88       */
89      private double noise3;
90  
91      /**
92       * returned 5th order MAD of all non-null pixels
93       */
94      private double noise5;
95  
96      private double xmaxval = Double.NEGATIVE_INFINITY;
97  
98      private double xminval = Double.POSITIVE_INFINITY;
99  
100     private double xnoise2;
101 
102     private double xnoise3;
103 
104     private double xnoise5;
105 
106     @Deprecated
107     public Quantize(QuantizeOption quantizeOption) {
108         parameter = quantizeOption;
109     }
110 
111     private void checkDataRange(Buffer in) throws IllegalArgumentException {
112         int n = parameter.getTileWidth() * parameter.getTileHeight();
113         int origpos = in.position();
114 
115         xminval = Double.POSITIVE_INFINITY;
116         xmaxval = Double.NEGATIVE_INFINITY;
117         xnoise2 = 0.0;
118         xnoise3 = 0.0;
119         xnoise5 = 0.0;
120 
121         int nval = 0;
122 
123         FloatBuffer fin = (in instanceof FloatBuffer) ? (FloatBuffer) in : null;
124         DoubleBuffer din = (in instanceof DoubleBuffer) ? (DoubleBuffer) in : null;
125 
126         if (fin == null && din == null) {
127             throw new IllegalArgumentException("input buffer of type " + in.getClass().getName() + " is unsupported.");
128         }
129 
130         for (int i = 0; i < n; i++) {
131             double x = fin == null ? din.get() : fin.get();
132 
133             if (!parameter.isRegular(x)) {
134                 continue;
135             }
136 
137             if (x < xminval) {
138                 xminval = x;
139             }
140 
141             if (x > xmaxval) {
142                 xmaxval = x;
143             }
144 
145             nval++;
146         }
147 
148         in.position(origpos);
149 
150         setNoiseResult(nval);
151     }
152 
153     /**
154      * Estimate the median and background noise in the input image using 2nd, 3rd and 5th order Median Absolute
155      * Differences. The noise in the background of the image is calculated using the MAD algorithms developed for
156      * deriving the signal to noise ratio in spectra (see issue #42 of the ST-ECF newsletter,
157      * http://www.stecf.org/documents/newsletter/) 3rd order: noise = 1.482602 / sqrt(6) * median (abs(2*flux(i) -
158      * flux(i-2) - flux(i+2))) The returned estimates are the median of the values that are computed for each row of the
159      * image.
160      * 
161      * @param  in                       a FloatBuffer or a DoubleBuffer instance. It is rewinded at return.
162      * 
163      * @throws IllegalArgumentException if the input buffer is not a FloatBuffer or DoubleBuffer instance.
164      */
165     @SuppressWarnings("null")
166     private void calculateNoise(Buffer in) throws IllegalArgumentException {
167         int origPos = in.position();
168         initializeNoise();
169 
170         int nx = parameter.getTileWidth();
171         int ny = parameter.getTileHeight();
172 
173         if (nx * ny < MINIMUM_PIXEL_WIDTH) {
174             calculateNoiseShortRow(in);
175             return;
176         }
177 
178         FloatBuffer fin = (in instanceof FloatBuffer) ? (FloatBuffer) in : null;
179         DoubleBuffer din = (in instanceof DoubleBuffer) ? (DoubleBuffer) in : null;
180 
181         if (fin == null && din == null) {
182             throw new IllegalArgumentException("input buffer of type " + in.getClass().getName() + " is unsupported.");
183         }
184 
185         int nrows = 0, nrows2 = 0;
186         long ngoodpix = 0;
187 
188         /* allocate arrays used to compute the median and noise estimates */
189         double[] differences2 = new double[nx];
190         double[] differences3 = new double[nx];
191         double[] differences5 = new double[nx];
192         double[] diffs2 = new double[ny];
193         double[] diffs3 = new double[ny];
194         double[] diffs5 = new double[ny];
195 
196         /* loop over each row of the image */
197         for (int jj = 0; jj < ny; jj++) {
198             int nvals = 0;
199             int nvals2 = 0;
200             double[] v = new double[9];
201 
202             for (int ii = 0, k = 0; ii < nx; ii++) {
203                 v[k] = fin == null ? din.get() : fin.get();
204 
205                 if (!parameter.isRegular(v[k])) {
206                     continue;
207                 }
208 
209                 if (v[k] < xminval) {
210                     xminval = v[k];
211                 }
212 
213                 if (v[k] > xmaxval) {
214                     xmaxval = v[k];
215                 }
216 
217                 ngoodpix++;
218 
219                 if (k + 1 < v.length) {
220                     k++;
221                     continue; // Wait until first 8 elements are filled before processing...
222                 }
223 
224                 /* construct tiledImageOperation of absolute differences */
225                 if (!(v[4] == v[5] && v[5] == v[6])) {
226                     differences2[nvals2] = Math.abs(v[4] - v[6]);
227                     nvals2++;
228                 }
229                 if (!(v[2] == v[3] && v[3] == v[4] && v[4] == v[5] && v[5] == v[6])) {
230                     differences3[nvals] = Math.abs(2 * v[4] - v[2] - v[6]);
231                     differences5[nvals] = Math.abs(N6 * v[4] - N4 * v[2] - N4 * v[6] + v[0] + v[8]);
232                     nvals++;
233                 } else {
234                     /* ignore constant background regions */
235                     ngoodpix++;
236                 }
237 
238                 /* shift over 1 pixel */
239                 System.arraycopy(v, 1, v, 0, v.length - 1);
240             } /* end of loop over pixels in the row */
241 
242             // compute the median diffs Note that there are 8 more pixel values
243             // than there are diffs values.
244             ngoodpix += nvals;
245 
246             if (nvals == 0) {
247                 continue; /* cannot compute medians on this row */
248             }
249 
250             if (nvals == 1) {
251                 if (nvals2 == 1) {
252                     diffs2[nrows2] = differences2[0];
253                     nrows2++;
254                 }
255                 diffs3[nrows] = differences3[0];
256                 diffs5[nrows] = differences5[0];
257             } else {
258                 /* quick_select returns the median MUCH faster than using qsort */
259                 if (nvals2 > 1) {
260                     diffs2[nrows2] = quickSelect(differences2, nvals);
261                     nrows2++;
262                 }
263                 diffs3[nrows] = quickSelect(differences3, nvals);
264                 diffs5[nrows] = quickSelect(differences5, nvals);
265             }
266 
267             nrows++;
268         } /* end of loop over rows */
269 
270         in.position(origPos);
271 
272         computeMedianOfValuesEachRow(nrows, nrows2, diffs2, diffs3, diffs5);
273         setNoiseResult(ngoodpix);
274     }
275 
276     @SuppressWarnings("null")
277     private void calculateNoiseShortRow(Buffer in) throws IllegalArgumentException {
278         int origPos = in.position();
279 
280         FloatBuffer fin = (in instanceof FloatBuffer) ? (FloatBuffer) in : null;
281         DoubleBuffer din = (in instanceof DoubleBuffer) ? (DoubleBuffer) in : null;
282 
283         if (fin == null && din == null) {
284             throw new IllegalArgumentException("input buffer of type " + in.getClass().getName() + " is unsupported.");
285         }
286 
287         int n = parameter.getTileWidth() * parameter.getTileHeight();
288         int ngoodpix = 0;
289         for (int index = 0; index < n; index++) {
290             double x = fin == null ? din.get() : fin.get();
291 
292             if (isNull(x)) {
293                 continue;
294             }
295 
296             if (x < xminval) {
297                 xminval = x;
298             }
299             if (x > xmaxval) {
300                 xmaxval = x;
301             }
302 
303             ngoodpix++;
304         }
305 
306         in.position(origPos);
307 
308         setNoiseResult(ngoodpix);
309     }
310 
311     @Deprecated
312     protected void computeMedianOfValuesEachRow(int nrows, int nrows2, double[] diffs2, double[] diffs3, double[] diffs5) {
313         // compute median of the values for each row.
314         if (nrows == 0) {
315             xnoise3 = 0;
316             xnoise5 = 0;
317         } else if (nrows == 1) {
318             xnoise3 = diffs3[0];
319             xnoise5 = diffs5[0];
320         } else {
321             Arrays.sort(diffs3, 0, nrows);
322             Arrays.sort(diffs5, 0, nrows);
323             xnoise3 = (diffs3[(nrows - 1) / 2] + diffs3[nrows / 2]) / 2.;
324             xnoise5 = (diffs5[(nrows - 1) / 2] + diffs5[nrows / 2]) / 2.;
325         }
326         if (nrows2 == 0) {
327             xnoise2 = 0;
328         } else if (nrows2 == 1) {
329             xnoise2 = diffs2[0];
330         } else {
331             Arrays.sort(diffs2, 0, nrows2);
332             xnoise2 = (diffs2[(nrows2 - 1) / 2] + diffs2[nrows2 / 2]) / 2.;
333         }
334     }
335 
336     @Deprecated
337     protected double getNoise2() {
338         return noise2;
339     }
340 
341     @Deprecated
342     protected double getNoise3() {
343         return noise3;
344     }
345 
346     @Deprecated
347     protected double getNoise5() {
348         return noise5;
349     }
350 
351     private void initializeNoise() {
352         xnoise2 = 0;
353         xnoise3 = 0;
354         xnoise5 = 0;
355         xminval = Double.POSITIVE_INFINITY;
356         xmaxval = Double.NEGATIVE_INFINITY;
357     }
358 
359     @Deprecated
360     protected boolean isNull(double d) {
361         return !parameter.isRegular(d);
362     }
363 
364     /**
365      * arguments: long row i: tile number = row number in the binary table double fdata[] i: tiledImageOperation of
366      * image pixels to be compressed long nxpix i: number of pixels in each row of fdata long nypix i: number of rows in
367      * fdata nullcheck i: check for nullvalues in fdata? double in_null_value i: value used to represent undefined
368      * pixels in fdata float qlevel i: quantization level int dither_method i; which dithering method to use int idata[]
369      * o: values of fdata after applying bzero and bscale double bscale o: scale factor double bzero o: zero offset int
370      * iminval o: minimum quantized value that is returned int imaxval o: maximum quantized value that is returned The
371      * function value will be one if the input fdata were copied to idata; in this case the parameters bscale and bzero
372      * can be used to convert back to nearly the original floating point values: fdata ~= idata * bscale + bzero. If the
373      * function value is zero, the data were not copied to idata.
374      * <p>
375      * In earlier implementations of the compression code, we only used the noise3 value as the most reliable estimate
376      * of the background noise in an image. If it is not possible to compute a noise3 value, then this serves as a red
377      * flag to indicate that quantizing the image could cause a loss of significant information in the image.
378      * </p>
379      * <p>
380      * At some later date, we decided to take the more conservative approach of using the minimum of all three of the
381      * noise values (while still requiring that noise3 has a defined value) as the best estimate of the noise. Note that
382      * if an image contains pure Gaussian distributed noise, then noise2, noise3, and noise5 will have exactly the same
383      * value (within statistical measurement errors).
384      * </p>
385      * 
386      * @param  fdata the data to quantinize
387      * @param  nxpix (unused) the image width -- the tile width of the initializing option is used instead.
388      * @param  nypix (unused) the image hight -- the tile height of the initializing option is used instead.
389      * 
390      * @return       <code>true</code> if the quantification was possible, or else <code>false</code> if the tile cannot
391      *                   be quantized
392      */
393     @Deprecated
394     public boolean quantize(double[] fdata, int nxpix, int nypix) {
395         DoubleBuffer buf = DoubleBuffer.wrap(fdata);
396         return guessQuantization(buf);
397     }
398 
399     private double getCharacteristicRMS() {
400         // use the minimum of noise2, noise3, and noise5 as the best
401         // noise value
402         double stdev = noise3;
403         if (noise2 != 0.0 && noise2 < stdev) {
404             stdev = noise2;
405         }
406         if (noise5 != 0.0 && noise5 < stdev) {
407             stdev = noise5;
408         }
409         return stdev;
410     }
411 
412     /**
413      * Guesses the quantization scaling and zero offset parameters based on the noise distribution in the data.
414      * 
415      * @param  fdata                    Input FloatBuffer or DoubleBuffer instance containing the floating-point data.
416      *                                      On return the buffer is restored to its initial position.
417      * 
418      * @return                          <code>true</code> if the quantization was successful, otherwise
419      *                                      <code>false</code>.
420      * 
421      * @throws IllegalArgumentException if the input buffer is not a FloatBuffer or DoubleBuffer instance.
422      * 
423      * @since                           1.23
424      */
425     boolean guessQuantization(Buffer fdata) throws IllegalArgumentException {
426 
427         // AK: defaults
428         parameter.setBScale(1.0);
429         parameter.setBZero(0.0);
430 
431         // estimate background noise using MAD pixel differences
432         if (parameter.getQLevel() >= 0.0) {
433             calculateNoise(fdata);
434         } else {
435             checkDataRange(fdata);
436         }
437 
438         if (ngood > 0) {
439             if (parameter.getQLevel() >= 0.0) {
440                 parameter.setBScale(getCharacteristicRMS()
441                         / (parameter.getQLevel() == 0.0 ? DEFAULT_QUANT_LEVEL : parameter.getQLevel()));
442             } else {
443                 // negative Q value represents the absolute quantization level
444                 parameter.setBScale(-parameter.getQLevel());
445             }
446 
447             parameter.setMinValue(minValue);
448             parameter.setMaxValue(maxValue);
449         } else {
450             /* set parameters to dummy values, which are not used */
451             parameter.setMinValue(0.0);
452             parameter.setMaxValue(0.0);
453         }
454 
455         /* check that the number of quantized levels does not exceed the range of regular integer values */
456         if (Math.ceil((maxValue - minValue) / parameter.getBScale()) >= 2.0 * Integer.MAX_VALUE) {
457             return false; /* don't quantize */
458         }
459 
460         parameter.updateBZeroAndIntLimits();
461 
462         return true; /* yes, data have been quantized */
463     }
464 
465     private double quickSelect(double[] arr, int n) {
466         int low, high;
467         int median;
468         int middle, ll, hh;
469 
470         low = 0;
471         high = n - 1;
472         median = low + high >>> 1; // was (low + high) / 2;
473         for (;;) {
474             if (high <= low) {
475                 return arr[median];
476             }
477 
478             if (high == low + 1) { /* Two elements only */
479                 if (arr[low] > arr[high]) {
480                     swapElements(arr, low, high);
481                 }
482                 return arr[median];
483             }
484 
485             /* Find median of low, middle and high items; swap into position low */
486             middle = low + high >>> 1; // was (low + high) / 2;
487             if (arr[middle] > arr[high]) {
488                 swapElements(arr, middle, high);
489             }
490             if (arr[low] > arr[high]) {
491                 swapElements(arr, low, high);
492             }
493             if (arr[middle] > arr[low]) {
494                 swapElements(arr, middle, low);
495             }
496 
497             /* Swap low item (now in position middle) into position (low+1) */
498             swapElements(arr, middle, low + 1);
499 
500             /* Nibble from each end towards middle, swapping items when stuck */
501             ll = low + 1;
502             hh = high;
503             for (;;) {
504                 do {
505                     ll++;
506                 } while (arr[low] > arr[ll]);
507                 do {
508                     hh--;
509                 } while (arr[hh] > arr[low]);
510 
511                 if (hh < ll) {
512                     break;
513                 }
514 
515                 swapElements(arr, ll, hh);
516             }
517 
518             /* Swap middle item (in position low) back into correct position */
519             swapElements(arr, low, hh);
520 
521             /* Re-set active partition */
522             if (hh <= median) {
523                 low = ll;
524             }
525             if (hh >= median) {
526                 high = hh - 1;
527             }
528         }
529     }
530 
531     private void setNoiseResult(long ngoodpix) {
532         minValue = Double.isFinite(xminval) ? xminval : 0.0;
533         maxValue = Double.isFinite(xmaxval) ? xmaxval : 0.0;
534         ngood = ngoodpix;
535         noise2 = NOISE_2_MULTIPLICATOR * xnoise2;
536         noise3 = NOISE_3_MULTIPLICATOR * xnoise3;
537         noise5 = NOISE_5_MULTIPLICATOR * xnoise5;
538     }
539 
540     private void swapElements(double[] array, int i, int j) {
541         double value = array[i];
542         array[i] = array[j];
543         array[j] = value;
544     }
545 
546 }