1 package nom.tam.image.compression.bintable;
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34 import java.io.IOException;
35 import java.nio.Buffer;
36 import java.nio.ByteBuffer;
37 import java.util.Arrays;
38
39 import nom.tam.fits.BinaryTable;
40 import nom.tam.fits.compression.algorithm.api.ICompressorControl;
41 import nom.tam.image.compression.hdu.CompressedTableData;
42 import nom.tam.util.ArrayOutputStream;
43 import nom.tam.util.ByteBufferOutputStream;
44 import nom.tam.util.ColumnTable;
45 import nom.tam.util.FitsOutputStream;
46 import nom.tam.util.type.ElementType;
47
48
49
50
51 @SuppressWarnings("javadoc")
52 public class BinaryTableTileCompressor extends BinaryTableTile {
53
54 private static final double NORMAL_OVERHEAD = 1.2;
55
56 private static final int MINIMUM_EXTRA_SPACE = 1024;
57
58 private final CompressedTableData compressed;
59
60
61 private BinaryTable orig;
62
63
64 private byte[][] compressedBytes;
65 private long[][] cdesc;
66 private Object udesc;
67
68
69
70
71
72
73 public BinaryTableTileCompressor(CompressedTableData compressedTable, ColumnTable<?> columnTable,
74 BinaryTableTileDescription description) {
75 super(columnTable, description);
76 this.compressed = compressedTable;
77 }
78
79
80
81
82
83
84
85
86 public BinaryTableTileCompressor(CompressedTableData compressedTable, BinaryTable table,
87 BinaryTableTileDescription description) {
88 this(compressedTable, table.getData(), description);
89 this.orig = table;
90 }
91
92 private int getCushion(int size, double factor) throws IllegalStateException {
93 long lsize = (long) Math.ceil(size * factor + MINIMUM_EXTRA_SPACE);
94 return (lsize > Integer.MAX_VALUE) ? Integer.MAX_VALUE : (int) lsize;
95 }
96
97 private byte[] getCompressedBytes(ByteBuffer buffer, ElementType<?> t, ICompressorControl compressor) {
98 buffer.flip();
99
100
101 int need = getCushion(getUncompressedSizeInBytes(), NORMAL_OVERHEAD);
102 ByteBuffer cbuf = ByteBuffer.allocateDirect(need);
103
104 Buffer tb = t.asTypedBuffer(buffer);
105
106 if (!compressor.compress(tb, cbuf, null)) {
107 throw new IllegalStateException("Compression error");
108 }
109
110 cbuf.flip();
111 byte[] cdata = new byte[cbuf.limit()];
112 cbuf.get(cdata);
113
114 buffer.clear();
115
116 return cdata;
117 }
118
119 private void compressRegular() throws IOException {
120 compressedBytes = new byte[1][];
121
122 ByteBuffer buffer = ByteBuffer.allocateDirect(getUncompressedSizeInBytes());
123 try (FitsOutputStream os = new FitsOutputStream(new ByteBufferOutputStream(buffer))) {
124 data.write(os, rowStart, rowEnd, column);
125 }
126
127 compressedBytes[0] = getCompressedBytes(buffer, type, getCompressorControl());
128 }
129
130 private void compressVariable() throws IOException {
131 int nRows = rowEnd - rowStart;
132 boolean longPointers = orig.getDescriptor(column).hasLongPointers();
133 long max = 0;
134
135 udesc = longPointers ? new long[nRows][] : new int[nRows][2];
136
137
138 for (int r = 0; r < nRows; r++) {
139 Object desc = data.getElement(rowStart + r, column);
140 long n = 0;
141
142 if (longPointers) {
143 ((long[][]) udesc)[r] = (long[]) desc;
144 n = ((long[]) desc)[0];
145 } else {
146 ((int[][]) udesc)[r] = (int[]) desc;
147 n = ((int[]) desc)[0];
148 }
149
150 if (n > max) {
151 max = n;
152 }
153 }
154
155 max *= type.size();
156
157
158 if (max > Integer.MAX_VALUE) {
159 throw new IllegalStateException("Uncompressed data too large for Java arrays: max=" + max);
160 }
161
162
163 ByteBuffer buffer = ByteBuffer.allocateDirect((int) max);
164 ElementType<?> dataType = ElementType.forClass(orig.getDescriptor(column).getElementClass());
165
166 ICompressorControl compressor = getCompressorControl(dataType.primitiveClass());
167 compressedBytes = new byte[nRows][];
168
169 for (int r = 0; r < nRows; r++) {
170 try (FitsOutputStream os = new FitsOutputStream(new ByteBufferOutputStream(buffer))) {
171
172 Object entry = orig.get(rowStart + r, column);
173 os.writeArray(entry);
174 }
175
176 compressedBytes[r] = getCompressedBytes(buffer, dataType, compressor);
177 }
178
179 }
180
181 @Override
182 public void run() {
183 try {
184 if (orig != null && orig.getDescriptor(column).isVariableSize()) {
185
186 compressVariable();
187 } else {
188
189 compressRegular();
190 }
191 } catch (IOException e) {
192 throw new IllegalStateException(e.getMessage(), e);
193 }
194 }
195
196 private Object setCompressedData(byte[] data) {
197 synchronized (compressed) {
198
199 Object p = compressed.getData().getElement(getTileIndex(), column);
200 if (p instanceof long[]) {
201 Arrays.fill((long[]) p, 0L);
202 } else {
203 Arrays.fill((int[]) p, 0);
204 }
205 compressed.getData().setElement(getTileIndex(), column, p);
206
207
208 compressed.setElement(getTileIndex(), column, data);
209
210
211
212 return compressed.getData().getElement(getTileIndex(), column);
213 }
214
215 }
216
217 private void setRegularData() {
218 setCompressedData(compressedBytes[0]);
219
220
221 compressedBytes = null;
222 }
223
224 private void setVariableData() throws IOException {
225 int nRows = compressedBytes.length;
226 boolean longPointers = orig.getDescriptor(column).hasLongPointers();
227
228 cdesc = new long[nRows][2];
229
230 ByteBuffer buffer = ByteBuffer
231 .allocateDirect((nRows * 2) * (Long.SIZE + (longPointers ? Long.BYTES : Integer.BYTES)));
232
233 for (int r = 0; r < nRows; r++) {
234
235 Object cp = setCompressedData(compressedBytes[r]);
236
237 if (cp instanceof long[]) {
238 cdesc[r] = (long[]) cp;
239 } else {
240
241 cdesc[r][0] = ((int[]) cp)[0];
242 cdesc[r][1] = ((int[]) cp)[1];
243 }
244 }
245
246 try (ArrayOutputStream os = new FitsOutputStream(new ByteBufferOutputStream(buffer))) {
247
248
249 os.writeArray(udesc);
250
251 os.writeArray(cdesc);
252 }
253
254
255
256 setCompressedData(getCompressedBytes(buffer, ElementType.BYTE, getGZipCompressorControl()));
257
258
259 compressedBytes = null;
260 cdesc = null;
261 udesc = null;
262 }
263
264 @Override
265 public void waitForResult() {
266 super.waitForResult();
267
268 if (orig != null && orig.getDescriptor(column).isVariableSize()) {
269 try {
270 setVariableData();
271 } catch (IOException e) {
272 throw new IllegalStateException(e.getMessage(), e);
273 }
274 } else {
275 setRegularData();
276 }
277
278 }
279
280 }