1 package nom.tam.fits.compression;
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34 import java.awt.BorderLayout;
35 import java.awt.Color;
36 import java.awt.Graphics2D;
37 import java.awt.image.BufferedImage;
38 import java.io.BufferedInputStream;
39 import java.io.ByteArrayOutputStream;
40 import java.io.File;
41 import java.io.FileInputStream;
42 import java.io.FileNotFoundException;
43 import java.io.FileOutputStream;
44 import java.io.IOException;
45 import java.io.InputStream;
46 import java.lang.reflect.Array;
47 import java.math.BigDecimal;
48 import java.math.BigInteger;
49 import java.nio.ByteBuffer;
50 import java.nio.DoubleBuffer;
51 import java.nio.FloatBuffer;
52 import java.nio.IntBuffer;
53 import java.nio.ShortBuffer;
54 import java.nio.file.Path;
55 import java.util.Iterator;
56 import java.util.Random;
57 import java.util.stream.IntStream;
58
59 import javax.swing.ImageIcon;
60 import javax.swing.JFrame;
61 import javax.swing.JLabel;
62 import javax.swing.WindowConstants;
63
64 import org.junit.jupiter.api.Assertions;
65 import org.junit.jupiter.api.BeforeEach;
66 import org.junit.jupiter.api.Test;
67
68 import nom.tam.fits.BasicHDU;
69 import nom.tam.fits.Fits;
70 import nom.tam.fits.FitsException;
71 import nom.tam.fits.FitsFactory;
72 import nom.tam.fits.Header;
73 import nom.tam.fits.HeaderCard;
74 import nom.tam.fits.HeaderCardException;
75 import nom.tam.fits.ImageData;
76 import nom.tam.fits.ImageHDU;
77 import nom.tam.fits.compression.algorithm.hcompress.HCompressorOption;
78 import nom.tam.fits.compression.algorithm.quant.QuantizeOption;
79 import nom.tam.fits.compression.algorithm.rice.RiceCompressOption;
80 import nom.tam.fits.header.Compression;
81 import nom.tam.fits.header.DataDescription;
82 import nom.tam.fits.header.Standard;
83 import nom.tam.fits.util.BlackBoxImages;
84 import nom.tam.image.StandardImageTiler;
85 import nom.tam.image.StreamingTileImageData;
86 import nom.tam.image.compression.CompressedImageTiler;
87 import nom.tam.image.compression.hdu.CompressedImageHDU;
88 import nom.tam.util.ArrayDataOutput;
89 import nom.tam.util.ArrayFuncs;
90 import nom.tam.util.FitsInputStream;
91 import nom.tam.util.FitsOutputStream;
92 import nom.tam.util.SafeClose;
93
94 @SuppressWarnings({"deprecation"})
95 public class ReadWriteProvidedCompressedImageTest {
96
97 private ImageHDU m13;
98
99 private short[][] m13_data;
100
101 private float[][] m13_data_real;
102
103 private ImageHDU m13real;
104
105 private final boolean showImage = false;
106
107 private static int assertionCounter = 1;
108
109 @BeforeEach
110 public void setDefaults() {
111 FitsFactory.setDefaults();
112 }
113
114 private void assert_float_image(float[][] actual, float[][] expected, float delta) {
115 Assertions.assertEquals(expected.length, actual.length);
116 IntStream.range(0, expected.length).parallel()
117 .forEach(axis0 -> Assertions.assertArrayEquals(expected[axis0], actual[axis0], delta));
118 }
119
120 private void assert_int_image(int[][] actual, int[][] expected) {
121 Assertions.assertEquals(expected.length, actual.length);
122 IntStream.range(0, expected.length).parallel()
123 .forEach(axis0 -> Assertions.assertArrayEquals(expected[axis0], actual[axis0]));
124 }
125
126 private void assert_short_image(short[][] actual, short[][] expected) {
127 Assertions.assertEquals(expected.length, actual.length);
128 IntStream.range(0, expected.length).parallel()
129 .forEach(axis0 -> Assertions.assertArrayEquals(expected[axis0], actual[axis0]));
130 }
131
132
133
134
135
136 private void assertArrayEquals(double[] expected, double[] actual, double delta, boolean checkNaN) {
137 Assertions.assertEquals(expected.length, actual.length);
138 for (int index = 0; index < actual.length; index++) {
139 double d1 = expected[index];
140 double d2 = actual[index];
141 if (Double.isNaN(d1) || Double.isNaN(d2)) {
142 if (checkNaN) {
143 Assertions.assertEquals(Double.isNaN(d1), Double.isNaN(d2));
144 }
145 } else {
146 Assertions.assertEquals(d1, d2, delta);
147 }
148 }
149 }
150
151 private void assertArrayEquals(float[][] expected, float[][] actual, float delta) {
152 Assertions.assertEquals(expected.length, actual.length);
153 for (int index = 0; index < actual.length; index++) {
154 Assertions.assertArrayEquals(expected[index], actual[index], delta, "index " + index);
155 }
156 }
157
158
159
160
161 @SuppressWarnings("unchecked")
162 private <T>
163
164 void assertCompressedToUncompressedImage(String fileName, String unCompfileName, Class<T> clazz,
165 IHDUAsserter<T> reader) throws Exception {
166
167
168
169 try (Fits f = new Fits(fileName); Fits unFits = new Fits(unCompfileName)) {
170 ImageHDU hdu = readHDU(unFits, ImageHDU.class);
171 unFits.deleteHDU(0);
172 CompressedImageHDU uncompHdu = readHDU(f, CompressedImageHDU.class);
173
174 f.deleteHDU(0);
175 while (hdu != null && uncompHdu != null) {
176 T compressedData = (T) uncompHdu.asImageHDU().getData().getData();
177 T orgData = (T) hdu.getData().getData();
178
179
180 System.out.println("Asserting image data! " + (assertionCounter++));
181 reader.assertData(orgData, compressedData);
182
183 hdu = readHDU(unFits, ImageHDU.class);
184 if (hdu != null) {
185 unFits.deleteHDU(0);
186 }
187 uncompHdu = readHDU(f, CompressedImageHDU.class);
188 if (uncompHdu != null) {
189 f.deleteHDU(0);
190 }
191 }
192 Assertions.assertFalse(hdu != null || uncompHdu != null);
193 }
194
195
196 }
197
198 private void assertData(float[][] data) {
199 for (int x = 0; x < 300; x++) {
200 for (int y = 0; y < 300; y++) {
201 Assertions.assertEquals(m13_data_real[x][y], data[x][y], 1f);
202 }
203 }
204 }
205
206 private void assertData(short[][] data) {
207 for (int x = 0; x < 300; x++) {
208 for (int y = 0; y < 300; y++) {
209 Assertions.assertEquals(m13_data[x][y], data[x][y]);
210 }
211 }
212 }
213
214 private void assertFloatImage(FloatBuffer result, float[][] expected, float delta) {
215 float[] real = new float[expected[0].length];
216 for (float[] expectedPart : expected) {
217 result.get(real);
218 Assertions.assertEquals(expectedPart.length, real.length);
219 for (int subindex = 0; subindex < expectedPart.length; subindex++) {
220 float expectedFloat = expectedPart[subindex];
221 float realFloat = real[subindex];
222 if (!Float.isNaN(expectedFloat) && !Float.isNaN(realFloat)) {
223 Assertions.assertEquals(expectedFloat, realFloat, delta);
224 }
225 }
226 }
227 }
228
229 private void assertIntImage(IntBuffer result, int[][] expected) {
230 int[] real = new int[expected[0].length];
231 for (int index = 0; index < real.length; index++) {
232 int[] expectedPart = expected[index];
233 result.get(real);
234 Assertions.assertEquals(expectedPart.length, real.length);
235 for (int subindex = 0; subindex < expectedPart.length; subindex++) {
236 int expectedFloat = expectedPart[subindex];
237 int realFloat = real[subindex];
238 Assertions.assertEquals(expectedFloat, realFloat, "pixel differes at " + subindex + "x" + index);
239 }
240 }
241 }
242
243 @Test
244 public void blackboxTest_c4s_060126_182642_zri() throws Exception {
245 assertCompressedToUncompressedImage(resolveLocalOrRemoteFileName("c4s_060126_182642_zri.fits.fz"),
246 resolveLocalOrRemoteFileName("c4s_060126_182642_zri.fits"), short[][].class,
247 (IHDUAsserter<short[][]>) (expected, actual) -> assert_short_image(actual, expected));
248 }
249
250 @Test
251 public void blackboxTest_c4s_060127_070751_cri() throws Exception {
252 assertCompressedToUncompressedImage(resolveLocalOrRemoteFileName("c4s_060127_070751_cri.fits.fz"),
253 resolveLocalOrRemoteFileName("c4s_060127_070751_cri.fits"), short[][].class,
254 (IHDUAsserter<short[][]>) (expected, actual) -> assert_short_image(actual, expected));
255 }
256
257 @Test
258 public void blackboxTest_kwi_041217_212603_fri() throws Exception {
259 assertCompressedToUncompressedImage(resolveLocalOrRemoteFileName("kwi_041217_212603_fri.fits.fz"),
260 resolveLocalOrRemoteFileName("kwi_041217_212603_fri.fits"), short[][].class,
261 (IHDUAsserter<short[][]>) (expected, actual) -> assert_short_image(actual, expected));
262 }
263
264 @Test
265 public void blackboxTest_kwi_041217_213100_fri() throws Exception {
266 assertCompressedToUncompressedImage(resolveLocalOrRemoteFileName("kwi_041217_213100_fri.fits.fz"),
267 resolveLocalOrRemoteFileName("kwi_041217_213100_fri.fits"), short[][].class,
268 (IHDUAsserter<short[][]>) (expected, actual) -> assert_short_image(actual, expected));
269 }
270
271 @Test
272 public void blackboxTest_psa_140305_191552_zri() throws Exception {
273 assertCompressedToUncompressedImage(resolveLocalOrRemoteFileName("psa_140305_191552_zri.fits.fz"),
274 resolveLocalOrRemoteFileName("psa_140305_191552_zri.fits"), short[][].class,
275 (IHDUAsserter<short[][]>) (expected, actual) -> assert_short_image(actual, expected));
276 }
277
278 @Test
279 public void blackboxTest_psa_140305_194520_fri() throws Exception {
280 assertCompressedToUncompressedImage(resolveLocalOrRemoteFileName("psa_140305_194520_fri.fits.fz"),
281 resolveLocalOrRemoteFileName("psa_140305_194520_fri.fits"), short[][].class,
282 (IHDUAsserter<short[][]>) (expected, actual) -> assert_short_image(actual, expected));
283 }
284
285 @Test
286 public void blackboxTest_tu1134529() throws Exception {
287 assertCompressedToUncompressedImage(resolveLocalOrRemoteFileName("tu1134529.fits.fz"),
288 resolveLocalOrRemoteFileName("tu1134529.fits"), int[][].class,
289 (IHDUAsserter<int[][]>) (expected, actual) -> assert_int_image(actual, expected));
290
291 }
292
293 @Test
294 public void blackboxTest_tu1134531() throws Exception {
295 assertCompressedToUncompressedImage(resolveLocalOrRemoteFileName("tu1134531.fits.fz"),
296 resolveLocalOrRemoteFileName("tu1134531.fits"), float[][].class,
297 (IHDUAsserter<float[][]>) (expected, actual) -> assert_float_image(actual, expected, 15f));
298 }
299
300 @Test
301 public void blackboxTest1_1() throws Exception {
302 FloatBuffer result = (FloatBuffer) readAll(resolveLocalOrRemoteFileName("DECam_00149774_40_DESX0332-2742.fits.fz"),
303 1);
304 float[][] expected = (float[][]) readAll(resolveLocalOrRemoteFileName("DECam_00149774_40_DESX0332-2742.fits"), 0);
305 result.rewind();
306 assertFloatImage(result, expected, 6f);
307 }
308
309 @Test
310 public void blackboxTest1_2() throws Exception {
311 IntBuffer result = (IntBuffer) readAll(resolveLocalOrRemoteFileName("DECam_00149774_40_DESX0332-2742.fits.fz"), 2);
312 int[][] expected = (int[][]) readAll(resolveLocalOrRemoteFileName("DECam_00149774_40_DESX0332-2742.fits"), 1);
313 result.rewind();
314 assertIntImage(result, expected);
315 }
316
317 @Test
318 public void blackboxTest1_3() throws Exception {
319 FloatBuffer result = (FloatBuffer) readAll(resolveLocalOrRemoteFileName("DECam_00149774_40_DESX0332-2742.fits.fz"),
320 3);
321 float[][] expected = (float[][]) readAll(resolveLocalOrRemoteFileName("DECam_00149774_40_DESX0332-2742.fits"), 2);
322 result.rewind();
323 assertFloatImage(result, expected, 0.0005f);
324 }
325
326 @Test
327 public void blackboxTest2() throws Exception {
328 FloatBuffer result = (FloatBuffer) readAll(resolveLocalOrRemoteFileName("unpack_vlos_mag.fits.fz"), 1);
329 float[][] expected = (float[][]) readAll(resolveLocalOrRemoteFileName("unpack_vlos_mag.fits"), 0);
330 result.rewind();
331 assertFloatImage(result, expected, 6500f);
332 }
333
334 @Test
335 public void blackboxTest3() throws Exception {
336 int lastpix = 0;
337 byte bytevalue = (byte) 0x80;
338 lastpix = lastpix | (bytevalue << 24);
339
340 IntBuffer result = (IntBuffer) readAll(resolveLocalOrRemoteFileName("hmi.fits.fz"), 1);
341 int[][] expected = (int[][]) readAll(resolveLocalOrRemoteFileName("hmi.fits"), 1);
342 result.rewind();
343 assertIntImage(result, expected);
344
345 }
346
347 private void dispayImage(short[][] data) {
348 JFrame frame = new JFrame("FrameDemo");
349 frame.setDefaultCloseOperation(WindowConstants.EXIT_ON_CLOSE);
350 frame.pack();
351 frame.setVisible(true);
352
353 BufferedImage img = (BufferedImage) frame.createImage(300, 300);
354
355 Graphics2D g = img.createGraphics();
356
357 for (int y = 0; y < 300; y++) {
358 for (int x = 0; x < 300; x++) {
359 float grayScale = data[x][y] / 4000f;
360 g.setColor(new Color(grayScale, grayScale, grayScale));
361 g.drawRect(x, y, 1, 1);
362 }
363 }
364 ImageIcon icon = new ImageIcon(img);
365 JLabel label = new JLabel(icon);
366 frame.getContentPane().add(label, BorderLayout.CENTER);
367 frame.repaint();
368 frame.pack();
369 frame.setVisible(true);
370 }
371
372 private HeaderCard findCompressOption(Header header, String key) {
373 int index = 1;
374 while (true) {
375 HeaderCard card = header.getCard(Compression.ZNAMEn.n(index));
376 if (card.getValue().equals(key)) {
377 return header.findCard(Compression.ZVALn.n(index));
378 }
379 index++;
380 }
381 }
382
383 private boolean isEmptyImage(BasicHDU<?> result) throws FitsException {
384 return result instanceof ImageHDU && ((ImageHDU) result).getData().getData() == null;
385 }
386
387 private Object readAll(String fileName, int index) throws Exception {
388 Object data = null;
389
390 try (Fits f = new Fits(fileName)) {
391 BasicHDU<?> hdu = f.getHDU(index);
392 if (hdu instanceof CompressedImageHDU) {
393 CompressedImageHDU bhdu = (CompressedImageHDU) hdu;
394 data = bhdu.getUncompressedData();
395 }
396 if (hdu instanceof ImageHDU) {
397 ImageHDU bhdu = (ImageHDU) hdu;
398 data = bhdu.getData().getData();
399 }
400 }
401 return data;
402 }
403
404 private ImageHDU readCompressedHdu(String fileName, int index) throws Exception {
405 try (Fits f = new Fits(fileName)) {
406 BasicHDU<?> hdu = f.getHDU(index);
407 if (hdu instanceof CompressedImageHDU) {
408 CompressedImageHDU bhdu = (CompressedImageHDU) hdu;
409 return bhdu.asImageHDU();
410 }
411 }
412 return null;
413 }
414
415 @Test
416 public void readGzip() throws Exception {
417 Object result = readAll("src/test/resources/nom/tam/image/provided/m13_gzip.fits", 1);
418
419 short[][] data = new short[300][300];
420 ArrayFuncs.copyInto(((ShortBuffer) result).array(), data);
421 assertData(data);
422 if (showImage) {
423 dispayImage(data);
424 }
425 }
426
427 @Test
428 public void readHCompressed() throws Exception {
429 Object result = readAll("src/test/resources/nom/tam/image/provided/m13_hcomp.fits", 1);
430 short[][] data = new short[300][300];
431 ArrayFuncs.copyInto(((ShortBuffer) result).array(), data);
432 assertData(data);
433 if (showImage) {
434 dispayImage(data);
435 }
436 }
437
438 @SuppressWarnings("unchecked")
439 private <T> T readHDU(Fits fits, Class<T> clazz) throws FitsException, IOException {
440 BasicHDU<?> result = fits.readHDU();
441 while (result != null && (isEmptyImage(result) || !clazz.isAssignableFrom(result.getClass()))) {
442 result = fits.readHDU();
443 }
444 return (T) result;
445 }
446
447 @Test
448 public void readPLIO() throws Exception {
449 Object result = readAll("src/test/resources/nom/tam/image/provided/m13_plio.fits", 1);
450 short[][] data = new short[300][300];
451 ArrayFuncs.copyInto(((ShortBuffer) result).array(), data);
452 assertData(data);
453 if (showImage) {
454 dispayImage(data);
455 }
456 }
457
458 @Test
459 public void readReal() throws Exception {
460 Object result = readAll("src/test/resources/nom/tam/image/provided/m13real_rice.fits", 1);
461 float[][] data = new float[300][300];
462 ArrayFuncs.copyInto(((FloatBuffer) result).array(), data);
463 assertData(data);
464 }
465
466 @Test
467 public void readRice() throws Exception {
468 Object result = readAll("src/test/resources/nom/tam/image/provided/m13_rice.fits", 1);
469
470 short[][] data = new short[300][300];
471 ArrayFuncs.copyInto(((ShortBuffer) result).array(), data);
472 assertData(data);
473 if (showImage) {
474 dispayImage(data);
475 }
476 }
477
478 @Test
479 public void readRiceAsImageHDU() throws Exception {
480 ImageHDU image = readCompressedHdu("src/test/resources/nom/tam/image/provided/m13_rice.fits", 1);
481
482 short[][] data = (short[][]) image.getData().getData();
483 if (showImage) {
484 dispayImage(data);
485 }
486 "to set a breakpoint ;-)".toString();
487 doTile("readRiceAsImageHDU", data, image.getTiler(), 0, 0, 20, 20);
488 }
489
490 private void doTile(String test, Object data, StandardImageTiler t, int x, int y, int nx, int ny) throws Exception {
491 Class<?> baseClass = ArrayFuncs.getBaseClass(data);
492 Object tile = Array.newInstance(baseClass, nx * ny);
493 t.getTile(tile, new int[] {y, x}, new int[] {ny, nx});
494
495 float sum0 = 0;
496 float sum1 = 0;
497 int length = Array.getLength(tile);
498 for (int i = 0; i < nx; i++) {
499 for (int j = 0; j < ny; j++) {
500 int tileOffset = i + j * nx;
501 if (tileOffset >= length) {
502 Assertions.fail("tile offset wrong");
503 }
504 sum0 += ((Number) Array.get(tile, tileOffset)).doubleValue();
505 sum1 += ((Number) Array.get(Array.get(data, j + y), i + x)).doubleValue();
506 }
507 }
508 Assertions.assertEquals(sum0, sum1, 0, "Tiler" + test);
509 }
510
511 private String resolveLocalOrRemoteFileName(String fileName) {
512 return BlackBoxImages.getBlackBoxImage(fileName);
513 }
514
515 @BeforeEach
516 public void setup() throws Exception {
517 try (Fits f = new Fits("src/test/resources/nom/tam/image/provided/m13.fits")) {
518 m13 = (ImageHDU) f.getHDU(0);
519 m13_data = (short[][]) m13.getData().getData();
520 }
521 try (Fits f = new Fits("src/test/resources/nom/tam/image/provided/m13real.fits")) {
522 m13real = (ImageHDU) f.getHDU(0);
523 m13_data_real = (float[][]) m13real.getData().getData();
524 }
525 }
526
527 @Test
528 public void testBlanksInCompressedFloatImage() throws Exception {
529 try {
530 new File("target/testBlanksInCompressedFloatImage.fits").delete();
531 new File("target/testBlanksInCompressedFloatImage.fits.fz").delete();
532 } catch (Exception e) {
533
534 }
535
536 double[][] data = new double[100][100];
537 double value = -1000.0d;
538 int blanks = 0;
539 for (int index1 = 0; index1 < 100; index1++) {
540 for (int index2 = 0; index2 < 100; index2++) {
541 value += 0.12345d;
542 if (blanks++ % 20 == 0) {
543 data[index1][index2] = Double.NaN;
544 } else {
545 data[index1][index2] = value;
546 }
547 }
548 }
549
550 try (Fits f = new Fits();
551 FitsOutputStream bdos = new FitsOutputStream(
552 new FileOutputStream("target/testBlanksInCompressedFloatImage.fits"))) {
553 ImageData imageData = new ImageData(data);
554 ImageHDU hdu = new ImageHDU(ImageHDU.manufactureHeader(imageData), imageData);
555 f.addHDU(hdu);
556 f.write(bdos);
557 }
558
559 try (Fits f = new Fits();
560 FitsOutputStream bdos = new FitsOutputStream(
561 new FileOutputStream("target/testBlanksInCompressedFloatImage.fits.fz"))) {
562 ImageData imageData = new ImageData(data);
563 ImageHDU hdu = new ImageHDU(ImageHDU.manufactureHeader(imageData), imageData);
564 CompressedImageHDU compressedHdu = CompressedImageHDU.fromImageHDU(hdu, 100, 15);
565 compressedHdu.setCompressAlgorithm(Compression.ZCMPTYPE_HCOMPRESS_1)
566 .setQuantAlgorithm(Compression.ZQUANTIZ_SUBTRACTIVE_DITHER_2)
567 .getCompressOption(HCompressorOption.class)
568 .setScale(4);
569 compressedHdu
570 .getCompressOption(QuantizeOption.class)
571 .setQlevel(1.0).setCheckNull(true);
572 compressedHdu.compress();
573 f.addHDU(compressedHdu);
574 f.write(bdos);
575 }
576
577 try (Fits f = new Fits("target/testBlanksInCompressedFloatImage.fits.fz")) {
578 f.readHDU();
579 CompressedImageHDU hdu = (CompressedImageHDU) f.readHDU();
580 double[][] actual = (double[][]) hdu.asImageHDU().getData().getData();
581 for (int index = 0; index < actual.length; index++) {
582 Assertions.assertArrayEquals(data[index], actual[index], 0f);
583 }
584 }
585 }
586
587 @Test
588 public void testGzipFallbackCompressed() throws Exception {
589 short[][] array = new short[][] {{1000, -2000, 3000, -3000, 4000}, {1000, -2000, 3000, -3000, 4000},
590 {1000, -2000, 3000, -3000, 4000}, {1000, -2000, 3000, -3000, 4000}, {1000, -2000, 3000, -3000, 4000}};
591
592 try (Fits f = new Fits()) {
593 ImageHDU image = (ImageHDU) Fits.makeHDU(array);
594 CompressedImageHDU compressedHdu = CompressedImageHDU.fromImageHDU(image, 5, 1);
595 compressedHdu.setCompressAlgorithm(Compression.ZCMPTYPE_RICE_1)
596 .getCompressOption(RiceCompressOption.class)
597 .setBlockSize(32);
598 compressedHdu.compress();
599 f.addHDU(compressedHdu);
600 FitsOutputStream bdos = null;
601 try {
602 bdos = new FitsOutputStream(new FileOutputStream("target/write_fallback.fits.fz"));
603 f.write(bdos);
604 } finally {
605 SafeClose.close(bdos);
606 }
607 }
608
609 try (Fits f = new Fits("target/write_fallback.fits.fz")) {
610 f.readHDU();
611 CompressedImageHDU hdu = (CompressedImageHDU) f.readHDU();
612 short[][] actualShortArray = (short[][]) hdu.asImageHDU().getData().getData();
613 Assertions.assertArrayEquals(array, actualShortArray);
614 }
615 }
616
617 @Test
618 public void testSomeBlanksInCompressedFloatImage() throws Exception {
619 double[][] data = newTestImageWithSomeBlanks("");
620
621 try (Fits f = new Fits();
622 FitsOutputStream bdos = new FitsOutputStream(
623 new FileOutputStream("target/testSomeBlanksInCompressedFloatImage.fits.fz"))) {
624 ImageData imageData = new ImageData(data);
625 ImageHDU hdu = new ImageHDU(ImageHDU.manufactureHeader(imageData), imageData);
626 CompressedImageHDU compressedHdu = CompressedImageHDU.fromImageHDU(hdu, 100, 15);
627 compressedHdu.setCompressAlgorithm(Compression.ZCMPTYPE_HCOMPRESS_1)
628 .setQuantAlgorithm(Compression.ZQUANTIZ_SUBTRACTIVE_DITHER_2)
629 .getCompressOption(HCompressorOption.class)
630 .setScale(4);
631 compressedHdu
632 .getCompressOption(QuantizeOption.class)
633 .setQlevel(1.0).setCheckNull(true);
634 compressedHdu.compress();
635 f.addHDU(compressedHdu);
636 f.write(bdos);
637 Assertions.assertEquals(Compression.COMPRESSED_DATA_COLUMN,
638 compressedHdu.getHeader().findCard(Standard.TTYPEn.n(1)).getValue());
639 }
640 try (Fits f = new Fits("target/testSomeBlanksInCompressedFloatImage.fits.fz")) {
641 f.readHDU();
642 CompressedImageHDU hdu = (CompressedImageHDU) f.readHDU();
643 double[][] actual = (double[][]) hdu.asImageHDU().getData().getData();
644 for (int index = 0; index < actual.length; index++) {
645 assertArrayEquals(data[index], actual[index], 1d, false);
646 }
647 }
648 }
649
650 @Test
651 public void testUnknownCompression() throws Exception {
652 try (Fits f = new Fits()) {
653 CompressedImageHDU compressedHdu = CompressedImageHDU.fromImageHDU(m13, -1, 15);
654 Assertions.assertThrows(IllegalStateException.class, () -> compressedHdu.setCompressAlgorithm("NIX"));
655 }
656 }
657
658 @Test
659 public void testMissingTileSize() throws Exception {
660
661 try (Fits f = new Fits()) {
662 CompressedImageHDU compressedHdu = CompressedImageHDU.fromImageHDU(m13, -1, 15);
663 compressedHdu.setCompressAlgorithm(Compression.ZCMPTYPE_HCOMPRESS_1)
664 .setQuantAlgorithm((String) null)
665 .getCompressOption(HCompressorOption.class)
666 .setScale(1);
667 compressedHdu.compress();
668 f.addHDU(compressedHdu);
669 Assertions.assertTrue(compressedHdu.isHeader());
670 compressedHdu.getHeader().deleteKey(Compression.ZTILEn.n(1));
671 FitsOutputStream bdos = null;
672 try {
673 bdos = new FitsOutputStream(new FileOutputStream("target/write_m13_own_h.fits.fz"));
674 f.write(bdos);
675 } finally {
676 SafeClose.close(bdos);
677 }
678 }
679
680 try (Fits f = new Fits("target/write_m13_own_h.fits.fz")) {
681 f.readHDU();
682 CompressedImageHDU hdu = (CompressedImageHDU) f.readHDU();
683 short[][] actualShortArray = (short[][]) hdu.asImageHDU().getData().getData();
684 Assertions.assertArrayEquals(m13_data, actualShortArray);
685 }
686 }
687
688 @Test
689 public void writeHcompress() throws Exception {
690
691 try (Fits f = new Fits()) {
692 CompressedImageHDU compressedHdu = CompressedImageHDU.fromImageHDU(m13, 300, 15);
693 compressedHdu.setCompressAlgorithm(Compression.ZCMPTYPE_HCOMPRESS_1)
694 .setQuantAlgorithm(null)
695 .getCompressOption(HCompressorOption.class)
696 .setScale(1);
697 compressedHdu.compress();
698 f.addHDU(compressedHdu);
699 compressedHdu = CompressedImageHDU.fromImageHDU(m13, 300, 1);
700 compressedHdu.setCompressAlgorithm(Compression.ZCMPTYPE_HCOMPRESS_1)
701 .setQuantAlgorithm(null)
702 .getCompressOption(HCompressorOption.class)
703 .setScale(1);
704 compressedHdu.compress();
705 f.addHDU(compressedHdu);
706 compressedHdu = CompressedImageHDU.fromImageHDU(m13, 100, 300);
707 compressedHdu.setCompressAlgorithm(Compression.ZCMPTYPE_HCOMPRESS_1)
708 .setQuantAlgorithm(null)
709 .getCompressOption(HCompressorOption.class)
710 .setSmooth(true)
711 .setScale(1);
712 compressedHdu.compress();
713 f.addHDU(compressedHdu);
714 FitsOutputStream bdos = null;
715 try {
716 bdos = new FitsOutputStream(new FileOutputStream("target/write_m13_own_h.fits.fz"));
717 f.write(bdos);
718 } finally {
719 SafeClose.close(bdos);
720 }
721 }
722
723 try (Fits f = new Fits("target/write_m13_own_h.fits.fz")) {
724 f.readHDU();
725 CompressedImageHDU hdu = (CompressedImageHDU) f.readHDU();
726 short[][] actualShortArray = (short[][]) hdu.asImageHDU().getData().getData();
727 Assertions.assertArrayEquals(m13_data, actualShortArray);
728 hdu = (CompressedImageHDU) f.readHDU();
729 actualShortArray = (short[][]) hdu.asImageHDU().getData().getData();
730 Assertions.assertArrayEquals(m13_data, actualShortArray);
731 hdu = (CompressedImageHDU) f.readHDU();
732 actualShortArray = (short[][]) hdu.asImageHDU().getData().getData();
733 Assertions.assertArrayEquals(m13_data, actualShortArray);
734 }
735 }
736
737 @Test
738 public void writeHcompressFloat() throws Exception {
739
740 try (Fits f = new Fits()) {
741 CompressedImageHDU compressedHdu = CompressedImageHDU.fromImageHDU(m13real, 300, 15);
742 HCompressorOption option = compressedHdu.setCompressAlgorithm(Compression.ZCMPTYPE_HCOMPRESS_1)
743 .setQuantAlgorithm(Compression.ZQUANTIZ_SUBTRACTIVE_DITHER_2)
744 .getCompressOption(QuantizeOption.class)
745 .setQlevel(1.0)
746 .getCompressOption(HCompressorOption.class)
747 .setScale(1);
748
749 Assertions.assertSame(compressedHdu.getCompressOption(HCompressorOption.class), option);
750 Assertions.assertSame(compressedHdu.getCompressOption(QuantizeOption.class),
751 compressedHdu.getCompressOption(QuantizeOption.class));
752 compressedHdu.compress();
753 f.addHDU(compressedHdu);
754 compressedHdu = CompressedImageHDU.fromImageHDU(m13real, 300, 1);
755 compressedHdu.setCompressAlgorithm(Compression.ZCMPTYPE_HCOMPRESS_1)
756 .setQuantAlgorithm(Compression.ZQUANTIZ_SUBTRACTIVE_DITHER_2)
757 .getCompressOption(HCompressorOption.class)
758 .setScale(1);
759 compressedHdu.getCompressOption(QuantizeOption.class).setQlevel(1.0);
760 compressedHdu.compress();
761 f.addHDU(compressedHdu);
762 compressedHdu = CompressedImageHDU.fromImageHDU(m13real, 100, 300);
763 compressedHdu.setCompressAlgorithm(Compression.ZCMPTYPE_HCOMPRESS_1)
764 .setQuantAlgorithm(Compression.ZQUANTIZ_SUBTRACTIVE_DITHER_2)
765 .getCompressOption(HCompressorOption.class)
766 .setSmooth(true)
767 .setScale(1);
768 compressedHdu.getCompressOption(QuantizeOption.class).setQlevel(1.0);
769 compressedHdu.compress();
770 f.addHDU(compressedHdu);
771 FitsOutputStream bdos = null;
772 try {
773 bdos = new FitsOutputStream(new FileOutputStream("target/write_m13real_own_h.fits.fz"));
774 f.write(bdos);
775 } finally {
776 SafeClose.close(bdos);
777 }
778 }
779
780 try (Fits f = new Fits("target/write_m13real_own_h.fits.fz")) {
781 f.readHDU();
782 CompressedImageHDU hdu = (CompressedImageHDU) f.readHDU();
783 float[][] actualShortArray = (float[][]) hdu.asImageHDU().getData().getData();
784 assertArrayEquals(m13_data_real, actualShortArray, 9f);
785 hdu = (CompressedImageHDU) f.readHDU();
786 actualShortArray = (float[][]) hdu.asImageHDU().getData().getData();
787 assertArrayEquals(m13_data_real, actualShortArray, .000000000001f);
788 hdu = (CompressedImageHDU) f.readHDU();
789 actualShortArray = (float[][]) hdu.asImageHDU().getData().getData();
790 assertArrayEquals(m13_data_real, actualShortArray, 6f);
791 }
792 }
793
794 @Test
795 public void writeRice() throws Exception {
796
797 try (Fits f = new Fits()) {
798 CompressedImageHDU compressedHdu = CompressedImageHDU.fromImageHDU(m13, 300, 15);
799 compressedHdu.setCompressAlgorithm(Compression.ZCMPTYPE_RICE_1)
800 .setQuantAlgorithm((String) null)
801 .getCompressOption(RiceCompressOption.class)
802 .setBlockSize(32)
803 .setBytePix(2);
804 compressedHdu.compress();
805 f.addHDU(compressedHdu);
806 compressedHdu = CompressedImageHDU.fromImageHDU(m13, 300, 1);
807 compressedHdu.setCompressAlgorithm(Compression.ZCMPTYPE_RICE_1)
808 .setQuantAlgorithm((String) null)
809 .getCompressOption(RiceCompressOption.class)
810 .setBlockSize(32)
811 .setBytePix(2);
812 compressedHdu.compress();
813 f.addHDU(compressedHdu);
814 compressedHdu = CompressedImageHDU.fromImageHDU(m13, 100, 300);
815 compressedHdu.setCompressAlgorithm(Compression.ZCMPTYPE_RICE_1)
816 .setQuantAlgorithm((String) null)
817 .getCompressOption(RiceCompressOption.class)
818 .setBlockSize(32)
819 .setBytePix(2);
820 compressedHdu.compress();
821 f.addHDU(compressedHdu);
822 FitsOutputStream bdos = null;
823 try {
824 bdos = new FitsOutputStream(new FileOutputStream("target/write_m13_own.fits.fz"));
825 f.write(bdos);
826 } finally {
827 SafeClose.close(bdos);
828 }
829 }
830
831 try (Fits f = new Fits("target/write_m13_own.fits.fz")) {
832 f.readHDU();
833
834 CompressedImageHDU hdu = (CompressedImageHDU) f.readHDU();
835 Assertions.assertEquals("2", findCompressOption(hdu.getHeader(), Compression.BYTEPIX).getValue());
836 short[][] actualShortArray = (short[][]) hdu.asImageHDU().getData().getData();
837 Assertions.assertArrayEquals(m13_data, actualShortArray);
838
839 hdu = (CompressedImageHDU) f.readHDU();
840 Assertions.assertEquals("2", findCompressOption(hdu.getHeader(), Compression.BYTEPIX).getValue());
841 actualShortArray = (short[][]) hdu.asImageHDU().getData().getData();
842 Assertions.assertArrayEquals(m13_data, actualShortArray);
843
844 hdu = (CompressedImageHDU) f.readHDU();
845 Assertions.assertEquals("2", findCompressOption(hdu.getHeader(), Compression.BYTEPIX).getValue());
846 actualShortArray = (short[][]) hdu.asImageHDU().getData().getData();
847 Assertions.assertArrayEquals(m13_data, actualShortArray);
848
849 }
850 }
851
852 @Test
853 public void writeRiceSpeciaIntOverflow() throws Exception {
854 ImageHDU uncompressed = null;
855 int[][] expectedIntData;
856
857 try (Fits f = new Fits(resolveLocalOrRemoteFileName("hmi.fits"))) {
858 uncompressed = (ImageHDU) f.getHDU(1);
859 expectedIntData = (int[][]) uncompressed.getData().getData();
860 f.close();
861 }
862
863 try (Fits f = new Fits()) {
864 CompressedImageHDU compressedHdu = CompressedImageHDU.fromImageHDU(uncompressed, uncompressed.getAxes()[0], 4);
865 compressedHdu.setCompressAlgorithm(Compression.ZCMPTYPE_RICE_1)
866 .setQuantAlgorithm((String) null)
867 .getCompressOption(RiceCompressOption.class)
868 .setBlockSize(32)
869 .setBytePix(4);
870 compressedHdu.compress();
871 f.addHDU(compressedHdu);
872
873 try (FitsOutputStream bdos = new FitsOutputStream(new FileOutputStream("target/hmi.fits.fz"))) {
874 f.write(bdos);
875 }
876 }
877
878 try (Fits f = new Fits("target/hmi.fits.fz")) {
879 f.readHDU();
880 CompressedImageHDU hdu = (CompressedImageHDU) f.readHDU();
881 Assertions.assertEquals("4", findCompressOption(hdu.getHeader(), Compression.BYTEPIX).getValue());
882 ImageHDU asImageHDU = hdu.asImageHDU();
883 int[][] actualIntArray = (int[][]) asImageHDU.getData().getData();
884
885 Assertions.assertArrayEquals(expectedIntData, actualIntArray);
886 Assertions.assertNull(asImageHDU.getHeader().getStringValue(Standard.TFIELDS.key()));
887 Assertions.assertNull(asImageHDU.getHeader().getStringValue(Standard.TTYPEn.n(1).key()));
888 }
889 }
890
891 @Test
892 public void writeRiceFloat() throws Exception {
893 try (Fits f = new Fits()) {
894 CompressedImageHDU compressedHdu = CompressedImageHDU.fromImageHDU(m13real, 300, 15);
895 compressedHdu.setCompressAlgorithm(Compression.ZCMPTYPE_RICE_1)
896 .setQuantAlgorithm(Compression.ZQUANTIZ_SUBTRACTIVE_DITHER_2)
897 .getCompressOption(QuantizeOption.class)
898 .setQlevel(1.0)
899 .getCompressOption(RiceCompressOption.class)
900 .setBlockSize(32);
901 compressedHdu.compress();
902 f.addHDU(compressedHdu);
903 compressedHdu = CompressedImageHDU.fromImageHDU(m13real, 300, 1);
904 compressedHdu.setCompressAlgorithm(Compression.ZCMPTYPE_RICE_1)
905 .setQuantAlgorithm(Compression.ZQUANTIZ_SUBTRACTIVE_DITHER_2)
906 .getCompressOption(RiceCompressOption.class)
907 .setBlockSize(32);
908 compressedHdu.getCompressOption(QuantizeOption.class).setQlevel(1.0);
909 compressedHdu.compress();
910 f.addHDU(compressedHdu);
911 compressedHdu = CompressedImageHDU.fromImageHDU(m13real, 100, 300);
912 compressedHdu.setCompressAlgorithm(Compression.ZCMPTYPE_RICE_1)
913 .setQuantAlgorithm(Compression.ZQUANTIZ_SUBTRACTIVE_DITHER_2)
914 .getCompressOption(RiceCompressOption.class)
915 .setBlockSize(32);
916 compressedHdu.getCompressOption(QuantizeOption.class).setQlevel(1.0);
917 compressedHdu.compress();
918 f.addHDU(compressedHdu);
919 FitsOutputStream bdos = null;
920 try {
921 bdos = new FitsOutputStream(new FileOutputStream("target/write_m13real_own.fits.fz"));
922 f.write(bdos);
923 } finally {
924 SafeClose.close(bdos);
925 }
926 }
927
928 try (Fits f = new Fits("target/write_m13real_own.fits.fz")) {
929 f.readHDU();
930 CompressedImageHDU hdu = (CompressedImageHDU) f.readHDU();
931 float[][] actualShortArray = (float[][]) hdu.asImageHDU().getData().getData();
932 assertArrayEquals(m13_data_real, actualShortArray, 9f);
933 hdu = (CompressedImageHDU) f.readHDU();
934 actualShortArray = (float[][]) hdu.asImageHDU().getData().getData();
935 assertArrayEquals(m13_data_real, actualShortArray, 15f);
936 hdu = (CompressedImageHDU) f.readHDU();
937 actualShortArray = (float[][]) hdu.asImageHDU().getData().getData();
938 assertArrayEquals(m13_data_real, actualShortArray, 6f);
939 }
940 }
941
942 @Test
943 public void writeRiceFloatWithForceNoLoss() throws Exception {
944
945 try (Fits f = new Fits()) {
946 CompressedImageHDU compressedHdu = CompressedImageHDU.fromImageHDU(m13real, 300, 15);
947 compressedHdu.setCompressAlgorithm(Compression.ZCMPTYPE_RICE_1)
948 .setQuantAlgorithm(Compression.ZQUANTIZ_SUBTRACTIVE_DITHER_2)
949 .forceNoLoss(140, 140, 20, 20)
950 .getCompressOption(QuantizeOption.class)
951 .setQlevel(1.0)
952 .getCompressOption(RiceCompressOption.class)
953 .setBlockSize(32);
954 compressedHdu.compress();
955 f.addHDU(compressedHdu);
956
957 try (FitsOutputStream bdos = new FitsOutputStream(
958 new FileOutputStream("target/write_m13real_own_noloss.fits.fz"))) {
959 f.write(bdos);
960 }
961 }
962
963 try (Fits f = new Fits("target/write_m13real_own_noloss.fits.fz")) {
964 f.readHDU();
965 CompressedImageHDU hdu = (CompressedImageHDU) f.readHDU();
966 float[][] actualShortArray = (float[][]) hdu.asImageHDU().getData().getData();
967 assertArrayEquals(m13_data_real, actualShortArray, 3.5f);
968
969 for (int x = 140; x < 160; x++) {
970 for (int y = 140; y < 160; y++) {
971 Assertions.assertEquals(actualShortArray[x][y], m13_data_real[x][y], 0.0f);
972 }
973 }
974 }
975 }
976
977
978
979
980
981
982
983 @Test
984 public void writeRiceFloatWithForceNoLossTileLargerThanGzipBuffer() throws Exception {
985 Random random = new Random(42);
986 float[][] noise = new float[300][300];
987 for (float[] row : noise) {
988 for (int x = 0; x < row.length; x++) {
989 row[x] = random.nextFloat() * 1000.0f;
990 }
991 }
992 ImageData imageData = ImageHDU.encapsulate(noise);
993 ImageHDU noiseHdu = new ImageHDU(ImageHDU.manufactureHeader(imageData), imageData);
994
995 try (Fits f = new Fits()) {
996
997 CompressedImageHDU compressedHdu = CompressedImageHDU.fromImageHDU(noiseHdu, 300, 150);
998 compressedHdu.setCompressAlgorithm(Compression.ZCMPTYPE_RICE_1)
999 .setQuantAlgorithm(Compression.ZQUANTIZ_SUBTRACTIVE_DITHER_2)
1000 .forceNoLoss(0, 0, 300, 300)
1001 .getCompressOption(QuantizeOption.class)
1002 .setQlevel(1.0)
1003 .getCompressOption(RiceCompressOption.class)
1004 .setBlockSize(32);
1005 compressedHdu.compress();
1006 f.addHDU(compressedHdu);
1007
1008 try (FitsOutputStream bdos = new FitsOutputStream(
1009 new FileOutputStream("target/write_noise_own_noloss_large_tile.fits.fz"))) {
1010 f.write(bdos);
1011 }
1012 }
1013
1014 try (Fits f = new Fits("target/write_noise_own_noloss_large_tile.fits.fz")) {
1015 f.readHDU();
1016 CompressedImageHDU hdu = (CompressedImageHDU) f.readHDU();
1017
1018 int gzipColumn = hdu.findColumn(Compression.GZIP_COMPRESSED_DATA_COLUMN);
1019 Assertions.assertTrue(gzipColumn >= 0, "expected a GZIP_COMPRESSED_DATA column");
1020 for (int tile = 0; tile < hdu.getData().getNRows(); tile++) {
1021 Assertions.assertTrue(((byte[]) hdu.getData().get(tile, gzipColumn)).length > 0,
1022 "tile " + tile + " should have been stored as gzipped floats");
1023 }
1024
1025
1026 float[][] actual = (float[][]) hdu.asImageHDU().getData().getData();
1027 for (int y = 0; y < noise.length; y++) {
1028 Assertions.assertArrayEquals(noise[y], actual[y], 0.0f, "row " + y);
1029 }
1030 }
1031 }
1032
1033 @Test
1034 public void writeRiceDouble() throws Exception {
1035 double[][] m13_double_data = (double[][]) ArrayFuncs.convertArray(m13_data_real, double.class);
1036 ImageData imagedata = ImageHDU.encapsulate(m13_double_data);
1037 ImageHDU m13_double = new ImageHDU(ImageHDU.manufactureHeader(imagedata), imagedata);
1038
1039 try (Fits f = new Fits()) {
1040 CompressedImageHDU compressedHdu = CompressedImageHDU.fromImageHDU(m13_double, 300, 15);
1041 compressedHdu.setCompressAlgorithm(Compression.ZCMPTYPE_RICE_1)
1042 .setQuantAlgorithm(Compression.ZQUANTIZ_SUBTRACTIVE_DITHER_2)
1043 .getCompressOption(QuantizeOption.class)
1044 .setQlevel(1.0)
1045 .getCompressOption(RiceCompressOption.class)
1046 .setBlockSize(32);
1047 compressedHdu.compress();
1048 f.addHDU(compressedHdu);
1049
1050 try (FitsOutputStream bdos = new FitsOutputStream(new FileOutputStream("target/write_m13double.fits.fz"))) {
1051 f.write(bdos);
1052 }
1053 }
1054
1055 try (Fits f = new Fits("target/write_m13double.fits.fz")) {
1056 f.readHDU();
1057 CompressedImageHDU hdu = (CompressedImageHDU) f.readHDU();
1058 double[][] actualShortArray = (double[][]) hdu.asImageHDU().getData().getData();
1059 for (int index = 0; index < actualShortArray.length; index++) {
1060 assertArrayEquals(m13_double_data[index], actualShortArray[index], 3.5d, false);
1061 }
1062 }
1063 }
1064
1065 @Test
1066 public void writeRiceFloatWithNullPixelMask() throws Exception {
1067 double[][] data = newTestImageWithSomeBlanks("ForNull");
1068
1069 try (Fits f = new Fits();
1070 FitsOutputStream bdos = new FitsOutputStream(
1071 new FileOutputStream("target/testSomeBlanksInCompressedFloatImage.fits.fz"))) {
1072 ImageData imageData = new ImageData(data);
1073 ImageHDU hdu = new ImageHDU(ImageHDU.manufactureHeader(imageData), imageData);
1074 CompressedImageHDU compressedHdu = CompressedImageHDU.fromImageHDU(hdu, 100, 15);
1075 compressedHdu.setCompressAlgorithm(Compression.ZCMPTYPE_HCOMPRESS_1)
1076 .setQuantAlgorithm(Compression.ZQUANTIZ_SUBTRACTIVE_DITHER_2)
1077 .preserveNulls(Compression.ZCMPTYPE_RICE_1)
1078 .getCompressOption(QuantizeOption.class)
1079 .setQlevel(1.0).setCheckNull(true)
1080 .getCompressOption(HCompressorOption.class)
1081 .setScale(4);
1082 compressedHdu.compress();
1083 f.addHDU(compressedHdu);
1084 f.write(bdos);
1085 Assertions.assertEquals(Compression.COMPRESSED_DATA_COLUMN,
1086 compressedHdu.getHeader().findCard(Standard.TTYPEn.n(1)).getValue());
1087 }
1088
1089 try (Fits f = new Fits("target/testSomeBlanksInCompressedFloatImage.fits.fz")) {
1090 f.readHDU();
1091 CompressedImageHDU hdu = (CompressedImageHDU) f.readHDU();
1092 double[][] actual = (double[][]) hdu.asImageHDU().getData().getData();
1093 for (int index = 0; index < actual.length; index++) {
1094 assertArrayEquals(data[index], actual[index], 1d, true);
1095
1096
1097
1098 }
1099 }
1100 }
1101
1102 protected double[][] newTestImageWithSomeBlanks(String suffix)
1103 throws FitsException, IOException, FileNotFoundException {
1104 try {
1105 new File("target/testSomeBlanksInCompressedFloatImage" + suffix + ".fits").delete();
1106 new File("target/testSomeBlanksInCompressedFloatImage" + suffix + ".fits.fz").delete();
1107 } catch (Exception e) {
1108
1109 }
1110
1111 double[][] data = new double[100][100];
1112 double value = -1000.0d;
1113 int blanks = 0;
1114 Random random = new Random();
1115 for (int index1 = 0; index1 < 100; index1++) {
1116 for (int index2 = 0; index2 < 100; index2++) {
1117 data[index1][index2] = value;
1118 if (blanks++ % 20 == 0) {
1119 data[index1][index2] = Double.NaN;
1120 } else {
1121 data[index1][index2] = value;
1122 value += 0.12345d + random.nextDouble() / 1000d;
1123 }
1124 }
1125 }
1126 Fits f = null;
1127 FitsOutputStream bdos = null;
1128 try {
1129 f = new Fits();
1130 bdos = new FitsOutputStream(
1131 new FileOutputStream("target/testSomeBlanksInCompressedFloatImage" + suffix + ".fits"));
1132 ImageData imageData = new ImageData(data);
1133 ImageHDU hdu = new ImageHDU(ImageHDU.manufactureHeader(imageData), imageData);
1134 f.addHDU(hdu);
1135 f.write(bdos);
1136 } finally {
1137 SafeClose.close(bdos);
1138 SafeClose.close(f);
1139 }
1140 return data;
1141 }
1142
1143 @Test
1144 public void testImagePlusCompressedImage1() throws Exception {
1145
1146 testImagePlusCompressedImage(
1147 resolveLocalOrRemoteFileName("03h-80dec--C_CVT_2013-12-29-MW1-03h_Light_600SecISO200_000042.fit"));
1148
1149 }
1150
1151 @Test
1152 public void testImagePlusCompressedImage2() throws Exception {
1153
1154 testImagePlusCompressedImage(
1155 resolveLocalOrRemoteFileName("17h-75dec--BINT_C_CVT_2014-06-25-MW1-17h_Light_600SecISO200_000031.fit"));
1156
1157 }
1158
1159 private void testImagePlusCompressedImage(String imageFile) throws Exception {
1160 BasicHDU<?>[] fitsFileHDU = null;
1161 try (InputStream fileStream = new BufferedInputStream(new FileInputStream(imageFile));
1162 Fits fitsFile = new Fits(fileStream)) {
1163 fitsFileHDU = fitsFile.read();
1164 for (BasicHDU<?> element : fitsFileHDU) {
1165 if (element.getHeader().containsKey("ZIMAGE")) {
1166 if (element.getHeader().getBooleanValue("ZIMAGE")) {
1167 CompressedImageHDU hdu = (CompressedImageHDU) element;
1168 hdu.asImageHDU();
1169 }
1170 }
1171 }
1172 }
1173 }
1174
1175 @Test
1176 public void testDitherDecompress() throws Exception {
1177 final ImageHDU im;
1178
1179 try (Fits fits = new Fits("../blackbox-images/fpack.fits.fz")) {
1180 fits.readHDU();
1181 im = ((CompressedImageHDU) fits.readHDU()).asImageHDU();
1182 }
1183
1184 try (Fits fits = new Fits("../blackbox-images/funpack.fits")) {
1185 ImageHDU ref = (ImageHDU) fits.readHDU();
1186
1187 assertArrayEquals((float[][]) ref.getData().getData(), (float[][]) im.getData().getData(), 0.01f);
1188 }
1189 }
1190
1191 @Test
1192 public void testFireflyRiceQuant() throws Exception {
1193 try (Fits fits = new Fits("../blackbox-images/firefly_zblank_nan_test.fits")) {
1194 fits.readHDU();
1195 ImageHDU orig = (ImageHDU) fits.readHDU();
1196 ImageHDU compressed = ((CompressedImageHDU) fits.readHDU()).asImageHDU();
1197
1198 Fits f = new Fits();
1199 f.addHDU(compressed);
1200 f.write("target/firefly1.fits");
1201
1202 assertArrayEquals((float[][]) orig.getData().getData(), (float[][]) compressed.getData().getData(), 3.0f);
1203 }
1204 }
1205
1206 @Test
1207 public void writeCutoutFromCompressed() throws Exception {
1208 final File tempFile = File.createTempFile("temp", ".fits");
1209
1210 final Path filePath = Path.of(resolveLocalOrRemoteFileName("tu1134531.fits.fz"));
1211 final int[] corners = new int[] {1, 230};
1212 final int[] lengths = new int[] {464, 225};
1213 byte[] expectedCutout;
1214
1215 try (final FitsInputStream fitsInputStream = new FitsInputStream(new FileInputStream(filePath.toFile()));
1216 final Fits fits = new Fits(fitsInputStream);
1217 final Fits fitsOutput = new Fits(tempFile);
1218 final FitsOutputStream fitsOutputStream = new FitsOutputStream(new FileOutputStream(tempFile))) {
1219
1220 final BasicHDU<?> hdu = fits.getHDU(5);
1221 Assertions.assertInstanceOf(CompressedImageHDU.class, hdu, "Incorrect type of HDU at index 5");
1222 final CompressedImageHDU compressedImageHDU = (CompressedImageHDU) hdu;
1223 expectedCutout = getCutoutBytes(compressedImageHDU, corners, lengths);
1224 final Header header = ReadWriteProvidedCompressedImageTest.copyHeader(compressedImageHDU.getHeader());
1225 ReadWriteProvidedCompressedImageTest.adjustHeader(header);
1226 ReadWriteProvidedCompressedImageTest.addCutoutFromCompressed(compressedImageHDU, header, fitsOutput, corners,
1227 lengths);
1228
1229 fitsOutput.write(fitsOutputStream);
1230 }
1231
1232 try {
1233 assertWrittenCutoutMatches(tempFile, 0, expectedCutout, lengths);
1234 } finally {
1235 tempFile.delete();
1236 }
1237 }
1238
1239 static void addCutoutFromCompressed(final CompressedImageHDU compressedImageHDU, final Header adjustedHeader,
1240 final Fits fitsOutput, final int[] corners, final int[] lengths) throws HeaderCardException {
1241 ReadWriteProvidedCompressedImageTest.adjustCutoutHeader(adjustedHeader, lengths);
1242 final CompressedImageTiler compressedImageTiler = new CompressedImageTiler(compressedImageHDU);
1243 final StreamingTileImageData streamingTileImageData = new StreamingTileImageData(adjustedHeader,
1244 compressedImageTiler, corners, lengths, new int[] {1, 1});
1245 fitsOutput.addHDU(new ImageHDU(adjustedHeader, streamingTileImageData));
1246 }
1247
1248 static void adjustCutoutHeader(final Header header, final int[] lengths) throws HeaderCardException {
1249 if (header.containsKey(Compression.ZBITPIX.key())) {
1250 header.setBitpix(header.getIntValue(Compression.ZBITPIX));
1251 }
1252 header.setNaxes(2);
1253
1254 header.setNaxis(1, lengths[1]);
1255 header.setNaxis(2, lengths[0]);
1256 removeCompressedImageKeys(header);
1257 }
1258
1259 private static byte[] getCutoutBytes(final CompressedImageHDU compressedImageHDU, final int[] corners,
1260 final int[] lengths) throws FitsException, IOException {
1261 final CompressedImageTiler tiler = new CompressedImageTiler(compressedImageHDU);
1262 final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
1263 try (final ArrayDataOutput arrayDataOutput = new FitsOutputStream(byteArrayOutputStream)) {
1264 tiler.getTile(arrayDataOutput, corners, lengths);
1265 arrayDataOutput.flush();
1266 }
1267 return byteArrayOutputStream.toByteArray();
1268 }
1269
1270 private static void assertWrittenCutoutMatches(final File outputFile, final int hduIndex, final byte[] expected,
1271 final int[] lengths) throws Exception {
1272 final int[] expectedAxes = new int[] {lengths[0], lengths[1]};
1273 try (final Fits fits = new Fits(outputFile)) {
1274 final ImageHDU imageHdu = (ImageHDU) fits.getHDU(hduIndex);
1275 final Header header = imageHdu.getHeader();
1276 Assertions.assertEquals(lengths[1], header.getIntValue(Standard.NAXISn.n(1)), "NAXIS1 at HDU " + hduIndex);
1277 Assertions.assertEquals(lengths[0], header.getIntValue(Standard.NAXISn.n(2)), "NAXIS2 at HDU " + hduIndex);
1278 Assertions.assertArrayEquals(expectedAxes, imageHdu.getAxes(), "cutout NAXIS dimensions at HDU " + hduIndex);
1279 Assertions.assertArrayEquals(expectedAxes, ArrayFuncs.getDimensions(imageHdu.getData().getData()),
1280 "Java array dimensions at HDU " + hduIndex);
1281 final Class<?> arrayType = ArrayFuncs.getBaseClass(imageHdu.getData().getData());
1282 final ByteBuffer expectedPixels = java.nio.ByteBuffer.wrap(expected);
1283 ReadWriteProvidedCompressedImageTest.writeExpectedCutoutBytes(expectedPixels, imageHdu, lengths, hduIndex,
1284 arrayType);
1285 }
1286 }
1287
1288 private static void writeExpectedCutoutBytes(final ByteBuffer expectedPixels, final ImageHDU imageHdu,
1289 final int[] lengths, final int hduIndex, final Class<?> arrayType) {
1290 if (arrayType == short.class) {
1291 final ShortBuffer expectedShortPixels = expectedPixels.asShortBuffer();
1292 final short[][] data = (short[][]) imageHdu.getData().getData();
1293 int index = 0;
1294 for (int axis2 = 0; axis2 < lengths[0]; axis2++) {
1295 for (int axis1 = 0; axis1 < lengths[1]; axis1++) {
1296 Assertions.assertEquals(expectedShortPixels.get(index), data[axis2][axis1],
1297 "pixel differed at cutout index " + index + " in HDU " + hduIndex);
1298 index++;
1299 }
1300 }
1301 } else if (arrayType == float.class) {
1302 final FloatBuffer expectedFloatPixels = expectedPixels.asFloatBuffer();
1303 final float[][] data = (float[][]) imageHdu.getData().getData();
1304 int index = 0;
1305 for (int axis2 = 0; axis2 < lengths[0]; axis2++) {
1306 for (int axis1 = 0; axis1 < lengths[1]; axis1++) {
1307 Assertions.assertEquals(expectedFloatPixels.get(index), data[axis2][axis1],
1308 "pixel differed at cutout index " + index + " in HDU " + hduIndex);
1309 index++;
1310 }
1311 }
1312 } else if (arrayType == double.class) {
1313 final DoubleBuffer expectedDoublePixels = expectedPixels.asDoubleBuffer();
1314 final double[][] data = (double[][]) imageHdu.getData().getData();
1315 int index = 0;
1316 for (int axis2 = 0; axis2 < lengths[0]; axis2++) {
1317 for (int axis1 = 0; axis1 < lengths[1]; axis1++) {
1318 Assertions.assertEquals(expectedDoublePixels.get(index), data[axis2][axis1],
1319 "pixel differed at cutout index " + index + " in HDU " + hduIndex);
1320 index++;
1321 }
1322 }
1323 } else if (arrayType == int.class) {
1324 final IntBuffer expectedIntPixels = expectedPixels.asIntBuffer();
1325 final int[][] data = (int[][]) imageHdu.getData().getData();
1326 int index = 0;
1327 for (int axis2 = 0; axis2 < lengths[0]; axis2++) {
1328 for (int axis1 = 0; axis1 < lengths[1]; axis1++) {
1329 Assertions.assertEquals(expectedIntPixels.get(index), data[axis2][axis1],
1330 "pixel differed at cutout index " + index + " in HDU " + hduIndex);
1331 index++;
1332 }
1333 }
1334 }
1335 }
1336
1337 private static void removeCompressedImageKeys(final Header header) {
1338 header.deleteKey(Compression.ZTABLE);
1339 header.deleteKey(Compression.ZIMAGE);
1340 header.deleteKey(Compression.ZCMPTYPE);
1341 header.deleteKey(Compression.ZBITPIX);
1342 header.deleteKey(Compression.ZNAXIS);
1343 header.deleteKey(Compression.ZQUANTIZ);
1344 header.deleteKey(Compression.ZMASKCMP);
1345 header.deleteKey(Compression.ZSIMPLE);
1346 header.deleteKey(Compression.ZTENSION);
1347 header.deleteKey(Compression.ZEXTEND);
1348 header.deleteKey(Compression.ZBLOCKED);
1349 header.deleteKey(Compression.ZPCOUNT);
1350 header.deleteKey(Compression.ZGCOUNT);
1351 header.deleteKey(Compression.ZHECKSUM);
1352 header.deleteKey(Compression.ZDATASUM);
1353 header.deleteKey(Compression.ZDITHER0);
1354 header.deleteKey(Compression.ZBLANK);
1355 header.deleteKey(Compression.ZTHEAP);
1356 header.deleteKey(Compression.ZTILELEN);
1357 for (int n = 1; n <= 9; n++) {
1358 header.deleteKey(Compression.ZNAXISn.n(n));
1359 header.deleteKey(Compression.ZTILEn.n(n));
1360 header.deleteKey(Compression.ZNAMEn.n(n));
1361 header.deleteKey(Compression.ZVALn.n(n));
1362 header.deleteKey(Compression.ZFORMn.n(n));
1363 header.deleteKey(Compression.ZCTYPn.n(n));
1364 }
1365 }
1366
1367 static void adjustHeader(final Header header) {
1368 header.deleteKey(Standard.SIMPLE);
1369 header.deleteKey(Standard.XTENSION);
1370 header.deleteKey(Standard.EXTEND);
1371 header.deleteKey(Standard.TFIELDS);
1372
1373 header.addValue(Standard.XTENSION, Standard.XTENSION_IMAGE);
1374
1375
1376 header.addValue(Standard.PCOUNT, 0);
1377 header.addValue(Standard.GCOUNT, 1);
1378 }
1379
1380 private static Header copyHeader(final Header source) throws NumberFormatException, HeaderCardException {
1381 final Header destination = new Header();
1382 for (final Iterator<HeaderCard> headerCardIterator = source.iterator(); headerCardIterator.hasNext();) {
1383 final HeaderCard headerCard = headerCardIterator.next();
1384 final String headerCardKey = headerCard.getKey();
1385 final Class<?> valueType = headerCard.valueType();
1386
1387
1388 if (headerCardKey == null || headerCardKey.trim().isEmpty()) {
1389 destination.addValue(headerCardKey, (String) null, headerCard.getComment());
1390 } else if (Standard.COMMENT.key().equals(headerCardKey)) {
1391 destination.insertComment(headerCard.getComment());
1392 } else if (Standard.HISTORY.key().equals(headerCardKey)) {
1393 destination.insertHistory(headerCard.getComment());
1394 } else {
1395 if (valueType == String.class || valueType == null) {
1396 destination.addValue(headerCardKey, headerCard.getValue(), headerCard.getComment());
1397 } else if (valueType == Boolean.class) {
1398 destination.addValue(headerCardKey, Boolean.parseBoolean(headerCard.getValue()),
1399 headerCard.getComment());
1400 } else if (valueType == Integer.class || valueType == BigInteger.class) {
1401 destination.addValue(headerCardKey, new BigInteger(headerCard.getValue()), headerCard.getComment());
1402 } else if (valueType == Long.class) {
1403 destination.addValue(headerCardKey, Long.parseLong(headerCard.getValue()), headerCard.getComment());
1404 } else if (valueType == Double.class) {
1405 destination.addValue(headerCardKey, Double.parseDouble(headerCard.getValue()), headerCard.getComment());
1406 } else if (valueType == BigDecimal.class) {
1407 destination.addValue(headerCardKey, new BigDecimal(headerCard.getValue()), headerCard.getComment());
1408 } else if (valueType == Float.class) {
1409 destination.addValue(headerCardKey, Float.parseFloat(headerCard.getValue()), headerCard.getComment());
1410 }
1411 }
1412 }
1413
1414 return destination;
1415 }
1416
1417 private static void setupPrimaryHeader(final Header header, final int nextEndSize) throws HeaderCardException {
1418 final HeaderCard nextEndCard = header.findCard(DataDescription.NEXTEND);
1419
1420
1421 if (nextEndCard == null) {
1422 header.addValue(DataDescription.NEXTEND, nextEndSize);
1423 } else {
1424 nextEndCard.setValue(nextEndSize);
1425 }
1426
1427 final HeaderCard extendFlagCard = header.findCard(Standard.EXTEND);
1428
1429
1430 if (extendFlagCard == null) {
1431 header.addValue(Standard.EXTEND, true);
1432 } else {
1433 extendFlagCard.setValue(true);
1434 }
1435 }
1436 }