Line data Source code
1 : /*
2 : * Copyright (c) 2002-2012, California Institute of Technology.
3 : * All rights reserved. Based on Government Sponsored Research under contracts
4 : * NAS7-1407 and/or NAS7-03001.
5 : *
6 : * Redistribution and use in source and binary forms, with or without
7 : * modification, are permitted provided that the following conditions are met:
8 : * 1. Redistributions of source code must retain the above copyright notice,
9 : * this list of conditions and the following disclaimer.
10 : * 2. Redistributions in binary form must reproduce the above copyright
11 : * notice, this list of conditions and the following disclaimer in the
12 : * documentation and/or other materials provided with the distribution.
13 : * 3. Neither the name of the California Institute of Technology (Caltech),
14 : * its operating division the Jet Propulsion Laboratory (JPL), the National
15 : * Aeronautics and Space Administration (NASA), nor the names of its
16 : * contributors may be used to endorse or promote products derived from this
17 : * software without specific prior written permission.
18 : *
19 : * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20 : * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 : * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 : * ARE DISCLAIMED. IN NO EVENT SHALL THE CALIFORNIA INSTITUTE OF TECHNOLOGY BE
23 : * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 : * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 : * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 : * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 : * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 : * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 : * POSSIBILITY OF SUCH DAMAGE.
30 : *
31 : * Copyright 2014-2021 Esri
32 : *
33 : * Licensed under the Apache License, Version 2.0 (the "License");
34 : * you may not use this file except in compliance with the License.
35 : * You may obtain a copy of the License at
36 : *
37 : * http://www.apache.org/licenses/LICENSE-2.0
38 : *
39 : * Unless required by applicable law or agreed to in writing, software
40 : * distributed under the License is distributed on an "AS IS" BASIS,
41 : * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
42 : * See the License for the specific language governing permissions and
43 : * limitations under the License.
44 : */
45 :
46 : /******************************************************************************
47 : *
48 : * Project: Meta Raster File Format Driver Implementation, RasterBand
49 : * Purpose: Implementation of MRF band
50 : *
51 : * Author: Lucian Plesea, Lucian.Plesea jpl.nasa.gov, lplesea esri.com
52 : *
53 : ****************************************************************************/
54 :
55 : #include "marfa.h"
56 : #include "gdal_priv.h"
57 : #include "ogr_srs_api.h"
58 : #include "ogr_spatialref.h"
59 :
60 : #include <vector>
61 : #include <cassert>
62 : #include <zlib.h>
63 : #if defined(ZSTD_SUPPORT)
64 : #include <zstd.h>
65 : #endif
66 :
67 : using namespace std::chrono;
68 :
69 : NAMESPACE_MRF_START
70 :
71 : // packs a block of a given type, with a stride
72 : // Count is the number of items that need to be copied
73 : // These are separate to allow for optimization
74 :
75 : template <typename T>
76 24 : static void cpy_stride_in(void *dst, void *src, int c, int stride)
77 : {
78 24 : T *s = reinterpret_cast<T *>(src);
79 24 : T *d = reinterpret_cast<T *>(dst);
80 :
81 6291480 : while (c--)
82 : {
83 6291460 : *d++ = *s;
84 6291460 : s += stride;
85 : }
86 24 : }
87 :
88 : template <typename T>
89 33 : static void cpy_stride_out(void *dst, void *src, int c, int stride)
90 : {
91 33 : T *s = reinterpret_cast<T *>(src);
92 33 : T *d = reinterpret_cast<T *>(dst);
93 :
94 8650780 : while (c--)
95 : {
96 8650750 : *d = *s++;
97 8650750 : d += stride;
98 : }
99 33 : }
100 :
101 : // Does every value in the buffer have the same value, using strict comparison
102 : template <typename T>
103 5105 : inline int isAllVal(const T *b, size_t bytecount, double ndv)
104 : {
105 5105 : T val = static_cast<T>(ndv);
106 5105 : size_t count = bytecount / sizeof(T);
107 1843727 : for (; count; --count)
108 : {
109 1843727 : if (*(b++) != val)
110 : {
111 5098 : return FALSE;
112 : }
113 : }
114 7 : return TRUE;
115 : }
116 :
117 : // Dispatcher based on gdal data type
118 5105 : static int isAllVal(GDALDataType gt, void *b, size_t bytecount, double ndv)
119 : {
120 : // Test to see if it has data
121 5105 : int isempty = false;
122 :
123 : // A case branch in a temporary macro, conversion from gdal enum to type
124 : #define TEST_T(GType, T) \
125 : case GType: \
126 : isempty = isAllVal(reinterpret_cast<T *>(b), bytecount, ndv); \
127 : break
128 :
129 5105 : switch (gt)
130 : {
131 4942 : TEST_T(GDT_Byte, GByte);
132 0 : TEST_T(GDT_Int8, GInt8);
133 28 : TEST_T(GDT_UInt16, GUInt16);
134 27 : TEST_T(GDT_Int16, GInt16);
135 26 : TEST_T(GDT_UInt32, GUInt32);
136 26 : TEST_T(GDT_Int32, GInt32);
137 0 : TEST_T(GDT_UInt64, GUInt64);
138 4 : TEST_T(GDT_Int64, GInt64);
139 26 : TEST_T(GDT_Float32, float);
140 26 : TEST_T(GDT_Float64, double);
141 0 : default:
142 0 : break;
143 : }
144 : #undef TEST_T
145 :
146 5105 : return isempty;
147 : }
148 :
149 : // Swap bytes in place, unconditional
150 : // cppcheck-suppress constParameterReference
151 0 : static void swab_buff(buf_mgr &src, const ILImage &img)
152 : {
153 : size_t i;
154 0 : switch (GDALGetDataTypeSize(img.dt))
155 : {
156 0 : case 16:
157 : {
158 0 : short int *b = (short int *)src.buffer;
159 0 : for (i = src.size / 2; i; b++, i--)
160 0 : *b = swab16(*b);
161 0 : break;
162 : }
163 0 : case 32:
164 : {
165 0 : int *b = (int *)src.buffer;
166 0 : for (i = src.size / 4; i; b++, i--)
167 0 : *b = swab32(*b);
168 0 : break;
169 : }
170 0 : case 64:
171 : {
172 0 : long long *b = (long long *)src.buffer;
173 0 : for (i = src.size / 8; i; b++, i--)
174 0 : *b = swab64(*b);
175 0 : break;
176 : }
177 : }
178 0 : }
179 :
180 : // Similar to compress2() but with flags to control zlib features
181 : // Returns true if it worked
182 9 : static int ZPack(const buf_mgr &src, buf_mgr &dst, int flags)
183 : {
184 : z_stream stream;
185 : int err;
186 :
187 9 : memset(&stream, 0, sizeof(stream));
188 9 : stream.next_in = (Bytef *)src.buffer;
189 9 : stream.avail_in = (uInt)src.size;
190 9 : stream.next_out = (Bytef *)dst.buffer;
191 9 : stream.avail_out = (uInt)dst.size;
192 :
193 9 : int level = flags & ZFLAG_LMASK;
194 9 : if (level > 9)
195 0 : level = 9;
196 9 : if (level < 1)
197 0 : level = 1;
198 9 : int wb = MAX_WBITS;
199 : // if gz flag is set, ignore raw request
200 9 : if (flags & ZFLAG_GZ)
201 0 : wb += 16;
202 9 : else if (flags & ZFLAG_RAW)
203 0 : wb = -wb;
204 9 : int memlevel = 8; // Good compromise
205 9 : int strategy = (flags & ZFLAG_SMASK) >> 6;
206 9 : if (strategy > 4)
207 0 : strategy = 0;
208 :
209 9 : err = deflateInit2(&stream, level, Z_DEFLATED, wb, memlevel, strategy);
210 9 : if (err != Z_OK)
211 0 : return err;
212 :
213 9 : err = deflate(&stream, Z_FINISH);
214 9 : if (err != Z_STREAM_END)
215 : {
216 0 : deflateEnd(&stream);
217 0 : return false;
218 : }
219 9 : dst.size = stream.total_out;
220 9 : err = deflateEnd(&stream);
221 9 : return err == Z_OK;
222 : }
223 :
224 : // Similar to uncompress() from zlib, accepts the ZFLAG_RAW
225 : // Return true if it worked
226 9 : static int ZUnPack(const buf_mgr &src, buf_mgr &dst, int flags)
227 : {
228 :
229 : z_stream stream;
230 : int err;
231 :
232 9 : memset(&stream, 0, sizeof(stream));
233 9 : stream.next_in = (Bytef *)src.buffer;
234 9 : stream.avail_in = (uInt)src.size;
235 9 : stream.next_out = (Bytef *)dst.buffer;
236 9 : stream.avail_out = (uInt)dst.size;
237 :
238 : // 32 means autodetec gzip or zlib header, negative 15 is for raw
239 9 : int wb = (ZFLAG_RAW & flags) ? -MAX_WBITS : 32 + MAX_WBITS;
240 9 : err = inflateInit2(&stream, wb);
241 9 : if (err != Z_OK)
242 0 : return false;
243 :
244 9 : err = inflate(&stream, Z_FINISH);
245 9 : if (err != Z_STREAM_END)
246 : {
247 0 : inflateEnd(&stream);
248 0 : return false;
249 : }
250 9 : dst.size = stream.total_out;
251 9 : err = inflateEnd(&stream);
252 9 : return err == Z_OK;
253 : }
254 :
255 : /*
256 : * Deflates a buffer, extrasize is the available size in the buffer past the
257 : * input If the output fits past the data, it uses that area, otherwise it uses
258 : * a temporary buffer and copies the data over the input on return, returning a
259 : * pointer to it. The output size is returned in src.size Returns nullptr when
260 : * compression failed
261 : */
262 9 : static void *DeflateBlock(buf_mgr &src, size_t extrasize, int flags)
263 : {
264 : // The one we might need to allocate
265 9 : void *dbuff = nullptr;
266 9 : buf_mgr dst = {src.buffer + src.size, extrasize};
267 :
268 : // Allocate a temp buffer if there is not sufficient space,
269 : // We need to have a bit more than half the buffer available
270 9 : if (extrasize < (src.size + 64))
271 : {
272 9 : dst.size = src.size + 64;
273 9 : dbuff = VSIMalloc(dst.size);
274 9 : dst.buffer = (char *)dbuff;
275 9 : if (!dst.buffer)
276 0 : return nullptr;
277 : }
278 :
279 9 : if (!ZPack(src, dst, flags))
280 : {
281 0 : CPLFree(dbuff); // Safe to call with NULL
282 0 : return nullptr;
283 : }
284 9 : if (dst.size > src.size)
285 : {
286 0 : CPLError(CE_Failure, CPLE_AppDefined,
287 : "DeflateBlock(): dst.size > src.size");
288 0 : CPLFree(dbuff); // Safe to call with NULL
289 0 : return nullptr;
290 : }
291 :
292 : // source size is used to hold the output size
293 9 : src.size = dst.size;
294 : // If we didn't allocate a buffer, the receiver can use it already
295 9 : if (!dbuff)
296 0 : return dst.buffer;
297 :
298 : // If we allocated a buffer, we need to copy the data to the input buffer.
299 9 : memcpy(src.buffer, dbuff, src.size);
300 9 : CPLFree(dbuff);
301 9 : return src.buffer;
302 : }
303 :
304 : #if defined(ZSTD_SUPPORT)
305 :
306 12 : CPL_NOSANITIZE_UNSIGNED_INT_OVERFLOW static void rankfilter(buf_mgr &src,
307 : size_t factor)
308 : {
309 : // Arange bytes by rank
310 12 : if (factor > 1)
311 : {
312 8 : std::vector<char> tempb(src.size);
313 8 : char *d = tempb.data();
314 43 : for (size_t j = 0; j < factor; j++)
315 9175080 : for (size_t i = j; i < src.size; i += factor)
316 9175040 : *d++ = src.buffer[i];
317 8 : memcpy(src.buffer, tempb.data(), src.size);
318 : }
319 : // byte delta
320 12 : auto p = reinterpret_cast<GByte *>(src.buffer);
321 12 : auto guard = p + src.size;
322 12 : GByte b(0);
323 10223600 : while (p < guard)
324 : {
325 10223600 : GByte temp = *p;
326 10223600 : *p -= b;
327 10223600 : b = temp;
328 10223600 : p++;
329 : }
330 12 : }
331 :
332 10 : CPL_NOSANITIZE_UNSIGNED_INT_OVERFLOW static void derank(buf_mgr &src,
333 : size_t factor)
334 : {
335 : // undo delta
336 10 : auto p = reinterpret_cast<GByte *>(src.buffer);
337 10 : auto guard = p + src.size;
338 10 : GByte b(0);
339 9699340 : while (p < guard)
340 : {
341 9699330 : b += *p;
342 9699330 : *p = b;
343 9699330 : p++;
344 : }
345 10 : if (factor > 1)
346 : { // undo rank separation
347 8 : std::vector<char> tempb(src.size);
348 8 : char *d = tempb.data();
349 8 : size_t chunk = src.size / factor;
350 2097160 : for (size_t i = 0; i < chunk; i++)
351 11272200 : for (size_t j = 0; j < factor; j++)
352 9175040 : *d++ = src.buffer[chunk * j + i];
353 8 : memcpy(src.buffer, tempb.data(), src.size);
354 : }
355 10 : }
356 :
357 : /*
358 : * Compress a buffer using zstd, extrasize is the available size in the buffer
359 : * past the input If ranks > 0, apply the rank filter If the output fits past
360 : * the data, it uses that area, otherwise it uses a temporary buffer and copies
361 : * the data over the input on return, returning a pointer to it. The output size
362 : * is returned in src.size Returns nullptr when compression failed
363 : */
364 12 : static void *ZstdCompBlock(buf_mgr &src, size_t extrasize, int c_level,
365 : ZSTD_CCtx *cctx, size_t ranks)
366 : {
367 12 : if (!cctx)
368 0 : return nullptr;
369 12 : if (ranks && (src.size % ranks) == 0)
370 12 : rankfilter(src, ranks);
371 :
372 : // might need a buffer for the zstd output
373 24 : std::vector<char> dbuff;
374 12 : void *dst = src.buffer + src.size;
375 12 : size_t size = extrasize;
376 : // Allocate a temp buffer if there is not sufficient space.
377 : // Zstd bound is about (size * 1.004 + 64)
378 12 : if (size < ZSTD_compressBound(src.size))
379 : {
380 12 : size = ZSTD_compressBound(src.size);
381 12 : dbuff.resize(size);
382 12 : dst = dbuff.data();
383 : }
384 :
385 : // Use the streaming interface, it's faster and better
386 : // See discussion at https://github.com/facebook/zstd/issues/3729
387 12 : ZSTD_outBuffer output = {dst, size, 0};
388 12 : ZSTD_inBuffer input = {src.buffer, src.size, 0};
389 : // Set level
390 12 : ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, c_level);
391 : // First, pass a continue flag, otherwise it will compress in one go
392 12 : size_t val = ZSTD_compressStream2(cctx, &output, &input, ZSTD_e_continue);
393 : // If it worked, pass the end flag to flush the buffer
394 12 : if (val == 0)
395 12 : val = ZSTD_compressStream2(cctx, &output, &input, ZSTD_e_end);
396 12 : if (ZSTD_isError(val))
397 0 : return nullptr;
398 12 : val = output.pos;
399 :
400 : // If we didn't need the buffer, packed data is already in the user buffer
401 12 : if (dbuff.empty())
402 : {
403 0 : src.size = val;
404 0 : return dst;
405 : }
406 :
407 12 : if (val > (src.size + extrasize))
408 : { // Doesn't fit in user buffer
409 0 : CPLError(CE_Failure, CPLE_AssertionFailed,
410 : "MRF: ZSTD compression buffer too small");
411 0 : return nullptr; // Error
412 : }
413 :
414 12 : memcpy(src.buffer, dbuff.data(), val);
415 12 : src.size = val;
416 12 : return src.buffer;
417 : }
418 : #endif
419 :
420 : //
421 : // The deflate_flags are available in all bands even if the DEFLATE option
422 : // itself is not set. This allows for PNG features to be controlled, as well
423 : // as any other bands that use zlib by itself
424 : //
425 543 : MRFRasterBand::MRFRasterBand(MRFDataset *parent_dataset, const ILImage &image,
426 543 : int band, int ov)
427 : : poMRFDS(parent_dataset),
428 543 : dodeflate(GetOptlist().FetchBoolean("DEFLATE", FALSE)),
429 : // Bring the quality to 0 to 9
430 543 : deflate_flags(image.quality / 10),
431 543 : dozstd(GetOptlist().FetchBoolean("ZSTD", FALSE)), zstd_level(9), m_l(ov),
432 1629 : img(image)
433 : {
434 543 : nBand = band;
435 543 : eDataType = parent_dataset->current.dt;
436 543 : nRasterXSize = img.size.x;
437 543 : nRasterYSize = img.size.y;
438 543 : nBlockXSize = img.pagesize.x;
439 543 : nBlockYSize = img.pagesize.y;
440 543 : nBlocksPerRow = img.pagecount.x;
441 543 : nBlocksPerColumn = img.pagecount.y;
442 543 : img.NoDataValue = MRFRasterBand::GetNoDataValue(&img.hasNoData);
443 :
444 : // Pick up the twists, aka GZ, RAWZ headers
445 543 : if (GetOptlist().FetchBoolean("GZ", FALSE))
446 0 : deflate_flags |= ZFLAG_GZ;
447 543 : else if (GetOptlist().FetchBoolean("RAWZ", FALSE))
448 0 : deflate_flags |= ZFLAG_RAW;
449 : // And Pick up the ZLIB strategy, if any
450 543 : const char *zstrategy = GetOptlist().FetchNameValueDef("Z_STRATEGY", "");
451 543 : int zv = Z_DEFAULT_STRATEGY;
452 543 : if (EQUAL(zstrategy, "Z_HUFFMAN_ONLY"))
453 0 : zv = Z_HUFFMAN_ONLY;
454 543 : else if (EQUAL(zstrategy, "Z_RLE"))
455 0 : zv = Z_RLE;
456 543 : else if (EQUAL(zstrategy, "Z_FILTERED"))
457 0 : zv = Z_FILTERED;
458 543 : else if (EQUAL(zstrategy, "Z_FIXED"))
459 0 : zv = Z_FIXED;
460 543 : deflate_flags |= (zv << 6);
461 543 : if (image.quality < 23 && image.quality > 0)
462 0 : zstd_level = image.quality;
463 :
464 : #if !defined(ZSTD_SUPPORT)
465 : if (dozstd)
466 : { // signal error condition to caller
467 : CPLError(CE_Failure, CPLE_AssertionFailed,
468 : "MRF: ZSTD support is not available");
469 : dozstd = FALSE;
470 : }
471 : #endif
472 : // Chose zstd over deflate if both are enabled and available
473 543 : if (dozstd && dodeflate)
474 0 : dodeflate = FALSE;
475 543 : }
476 :
477 : // Clean up the overviews if they exist
478 1086 : MRFRasterBand::~MRFRasterBand()
479 : {
480 615 : while (!overviews.empty())
481 : {
482 72 : delete overviews.back();
483 72 : overviews.pop_back();
484 : }
485 543 : }
486 :
487 : // Look for a string from the dataset options or from the environment
488 75 : const char *MRFRasterBand::GetOptionValue(const char *opt,
489 : const char *def) const
490 : {
491 75 : const char *optValue = poMRFDS->optlist.FetchNameValue(opt);
492 75 : if (optValue)
493 8 : return optValue;
494 67 : return CPLGetConfigOption(opt, def);
495 : }
496 :
497 : // Utility function, returns a value from a vector corresponding to the band
498 : // index or the first entry
499 157 : static double getBandValue(const std::vector<double> &v, int idx)
500 : {
501 157 : return (static_cast<int>(v.size()) > idx) ? v[idx] : v[0];
502 : }
503 :
504 : // Maybe we should check against the type range?
505 : // It is not keeping track of how many values have been set,
506 : // so the application should set none or all the bands
507 : // This call is only valid during Create
508 16 : CPLErr MRFRasterBand::SetNoDataValue(double val)
509 : {
510 16 : if (poMRFDS->bCrystalized)
511 : {
512 0 : CPLError(CE_Failure, CPLE_AssertionFailed,
513 : "MRF: NoData can be set only during file create");
514 0 : return CE_Failure;
515 : }
516 16 : if (GInt32(poMRFDS->vNoData.size()) < nBand)
517 0 : poMRFDS->vNoData.resize(nBand);
518 16 : poMRFDS->vNoData[nBand - 1] = val;
519 : // We also need to set it for this band
520 16 : img.NoDataValue = val;
521 16 : img.hasNoData = true;
522 16 : return CE_None;
523 : }
524 :
525 5999 : double MRFRasterBand::GetNoDataValue(int *pbSuccess)
526 : {
527 5999 : const std::vector<double> &v = poMRFDS->vNoData;
528 5999 : if (v.empty())
529 5842 : return GDALPamRasterBand::GetNoDataValue(pbSuccess);
530 157 : if (pbSuccess)
531 155 : *pbSuccess = TRUE;
532 157 : return getBandValue(v, nBand - 1);
533 : }
534 :
535 0 : double MRFRasterBand::GetMinimum(int *pbSuccess)
536 : {
537 0 : const std::vector<double> &v = poMRFDS->vMin;
538 0 : if (v.empty())
539 0 : return GDALPamRasterBand::GetMinimum(pbSuccess);
540 0 : if (pbSuccess)
541 0 : *pbSuccess = TRUE;
542 0 : return getBandValue(v, nBand - 1);
543 : }
544 :
545 0 : double MRFRasterBand::GetMaximum(int *pbSuccess)
546 : {
547 0 : const std::vector<double> &v = poMRFDS->vMax;
548 0 : if (v.empty())
549 0 : return GDALPamRasterBand::GetMaximum(pbSuccess);
550 0 : if (pbSuccess)
551 0 : *pbSuccess = TRUE;
552 0 : return getBandValue(v, nBand - 1);
553 : }
554 :
555 : // Fill with typed ndv, count is always in bytes
556 : template <typename T>
557 0 : static CPLErr buff_fill(void *b, size_t count, const T ndv)
558 : {
559 0 : T *buffer = static_cast<T *>(b);
560 0 : count /= sizeof(T);
561 0 : while (count--)
562 0 : *buffer++ = ndv;
563 0 : return CE_None;
564 : }
565 :
566 : /**
567 : *\brief Fills a buffer with no data
568 : *
569 : */
570 56 : CPLErr MRFRasterBand::FillBlock(void *buffer)
571 : {
572 : int success;
573 56 : double ndv = GetNoDataValue(&success);
574 56 : if (!success)
575 56 : ndv = 0.0;
576 56 : size_t bsb = blockSizeBytes();
577 :
578 : // use memset for speed for bytes, or if nodata is zeros
579 56 : if (0.0 == ndv || eDataType == GDT_Byte || eDataType == GDT_Int8)
580 : {
581 56 : memset(buffer, int(ndv), bsb);
582 56 : return CE_None;
583 : }
584 :
585 : #define bf(T) buff_fill<T>(buffer, bsb, T(ndv));
586 0 : switch (eDataType)
587 : {
588 0 : case GDT_UInt16:
589 0 : return bf(GUInt16);
590 0 : case GDT_Int16:
591 0 : return bf(GInt16);
592 0 : case GDT_UInt32:
593 0 : return bf(GUInt32);
594 0 : case GDT_Int32:
595 0 : return bf(GInt32);
596 0 : case GDT_UInt64:
597 0 : return bf(GUInt64);
598 0 : case GDT_Int64:
599 0 : return bf(GInt64);
600 0 : case GDT_Float32:
601 0 : return bf(float);
602 0 : case GDT_Float64:
603 0 : return bf(double);
604 0 : default:
605 0 : break;
606 : }
607 : #undef bf
608 : // Should exit before
609 0 : return CE_Failure;
610 : }
611 :
612 : /*\brief Interleave block fill
613 : *
614 : * Acquire space for all the other bands, fill each one then drop the locks
615 : * The current band output goes directly into the buffer
616 : */
617 :
618 0 : CPLErr MRFRasterBand::FillBlock(int xblk, int yblk, void *buffer)
619 : {
620 0 : std::vector<GDALRasterBlock *> blocks;
621 :
622 0 : for (int i = 0; i < poMRFDS->nBands; i++)
623 : {
624 0 : GDALRasterBand *b = poMRFDS->GetRasterBand(i + 1);
625 0 : if (b->GetOverviewCount() && 0 != m_l)
626 0 : b = b->GetOverview(m_l - 1);
627 :
628 : // Get the other band blocks, keep them around until later
629 0 : if (b == this)
630 : {
631 0 : FillBlock(buffer);
632 : }
633 : else
634 : {
635 0 : GDALRasterBlock *poBlock = b->GetLockedBlockRef(xblk, yblk, 1);
636 0 : if (poBlock == nullptr) // Didn't get this block
637 0 : break;
638 0 : FillBlock(poBlock->GetDataRef());
639 0 : blocks.push_back(poBlock);
640 : }
641 : }
642 :
643 : // Drop the locks for blocks we acquired
644 0 : for (int i = 0; i < int(blocks.size()); i++)
645 0 : blocks[i]->DropLock();
646 :
647 0 : return CE_None;
648 : }
649 :
650 : /*\brief Interleave block read
651 : *
652 : * Acquire space for all the other bands, unpack from the dataset buffer, then
653 : * drop the locks The current band output goes directly into the buffer
654 : */
655 :
656 8 : CPLErr MRFRasterBand::ReadInterleavedBlock(int xblk, int yblk, void *buffer)
657 : {
658 8 : std::vector<GDALRasterBlock *> blocks;
659 :
660 32 : for (int i = 0; i < poMRFDS->nBands; i++)
661 : {
662 24 : GDALRasterBand *b = poMRFDS->GetRasterBand(i + 1);
663 24 : if (b->GetOverviewCount() && 0 != m_l)
664 0 : b = b->GetOverview(m_l - 1);
665 :
666 24 : void *ob = buffer;
667 : // Get the other band blocks, keep them around until later
668 24 : if (b != this)
669 : {
670 16 : GDALRasterBlock *poBlock = b->GetLockedBlockRef(xblk, yblk, 1);
671 16 : if (poBlock == nullptr)
672 0 : break;
673 16 : ob = poBlock->GetDataRef();
674 16 : blocks.push_back(poBlock);
675 : }
676 :
677 : // Just the right mix of templates and macros make deinterleaving tidy
678 24 : void *pbuffer = poMRFDS->GetPBuffer();
679 : #define CpySI(T) \
680 : cpy_stride_in<T>(ob, reinterpret_cast<T *>(pbuffer) + i, \
681 : blockSizeBytes() / sizeof(T), img.pagesize.c)
682 :
683 : // Page is already in poMRFDS->pbuffer, not empty
684 : // There are only four cases, since only the data size matters
685 24 : switch (GDALGetDataTypeSize(eDataType) / 8)
686 : {
687 24 : case 1:
688 24 : CpySI(GByte);
689 24 : break;
690 0 : case 2:
691 0 : CpySI(GInt16);
692 0 : break;
693 0 : case 4:
694 0 : CpySI(GInt32);
695 0 : break;
696 0 : case 8:
697 0 : CpySI(GIntBig);
698 0 : break;
699 : }
700 : }
701 :
702 : #undef CpySI
703 :
704 : // Drop the locks we acquired
705 24 : for (int i = 0; i < int(blocks.size()); i++)
706 16 : blocks[i]->DropLock();
707 :
708 16 : return CE_None;
709 : }
710 :
711 : /**
712 : *\brief Fetch a block from the backing store dataset and keep a copy in the
713 : *cache
714 : *
715 : * @param xblk The X block number, zero based
716 : * @param yblk The Y block number, zero based
717 : * @param buffer buffer
718 : *
719 : */
720 4 : CPLErr MRFRasterBand::FetchBlock(int xblk, int yblk, void *buffer)
721 : {
722 4 : assert(!poMRFDS->source.empty());
723 4 : CPLDebug("MRF_IB", "FetchBlock %d,%d,0,%d, level %d\n", xblk, yblk, nBand,
724 : m_l);
725 :
726 4 : if (poMRFDS->clonedSource) // This is a clone
727 1 : return FetchClonedBlock(xblk, yblk, buffer);
728 :
729 3 : const GInt32 cstride = img.pagesize.c; // 1 if band separate
730 3 : ILSize req(xblk, yblk, 0, (nBand - 1) / cstride, m_l);
731 3 : GUIntBig infooffset = IdxOffset(req, img);
732 :
733 3 : GDALDataset *poSrcDS = nullptr;
734 3 : if (nullptr == (poSrcDS = poMRFDS->GetSrcDS()))
735 : {
736 1 : CPLError(CE_Failure, CPLE_AppDefined, "MRF: Can't open source file %s",
737 1 : poMRFDS->source.c_str());
738 1 : return CE_Failure;
739 : }
740 :
741 : // Scale to base resolution
742 2 : double scl = pow(poMRFDS->scale, m_l);
743 2 : if (0 == m_l)
744 2 : scl = 1; // To allow for precision issues
745 :
746 : // Prepare parameters for RasterIO, they might be different from a full page
747 2 : const GSpacing vsz = GDALGetDataTypeSizeBytes(eDataType);
748 2 : int Xoff = int(xblk * img.pagesize.x * scl + 0.5);
749 2 : int Yoff = int(yblk * img.pagesize.y * scl + 0.5);
750 2 : int readszx = int(img.pagesize.x * scl + 0.5);
751 2 : int readszy = int(img.pagesize.y * scl + 0.5);
752 :
753 : // Compare with the full size and clip to the right and bottom if needed
754 2 : int clip = 0;
755 2 : if (Xoff + readszx > poMRFDS->full.size.x)
756 : {
757 2 : clip |= 1;
758 2 : readszx = poMRFDS->full.size.x - Xoff;
759 : }
760 2 : if (Yoff + readszy > poMRFDS->full.size.y)
761 : {
762 2 : clip |= 1;
763 2 : readszy = poMRFDS->full.size.y - Yoff;
764 : }
765 :
766 : // This is where the whole page fits
767 2 : void *ob = buffer;
768 2 : if (cstride != 1)
769 0 : ob = poMRFDS->GetPBuffer();
770 :
771 : // Fill buffer with NoData if clipping
772 2 : if (clip)
773 2 : FillBlock(ob);
774 :
775 : // Use the dataset RasterIO to read one or all bands if interleaved
776 4 : CPLErr ret = poSrcDS->RasterIO(
777 : GF_Read, Xoff, Yoff, readszx, readszy, ob, pcount(readszx, int(scl)),
778 : pcount(readszy, int(scl)), eDataType, cstride,
779 2 : (1 == cstride) ? &nBand : nullptr, vsz * cstride,
780 2 : vsz * cstride * img.pagesize.x,
781 2 : (cstride != 1) ? vsz : vsz * img.pagesize.x * img.pagesize.y, nullptr);
782 :
783 2 : if (ret != CE_None)
784 0 : return ret;
785 :
786 : // Might have the block in the pbuffer, mark it anyhow
787 2 : poMRFDS->tile = req;
788 : buf_mgr filesrc;
789 2 : filesrc.buffer = (char *)ob;
790 2 : filesrc.size = static_cast<size_t>(img.pageSizeBytes);
791 :
792 2 : if (poMRFDS->bypass_cache)
793 : { // No local caching, just return the data
794 0 : if (1 == cstride)
795 0 : return CE_None;
796 0 : return ReadInterleavedBlock(xblk, yblk, buffer);
797 : }
798 :
799 : // Test to see if it needs to be written, or just marked as checked
800 : int success;
801 2 : double val = GetNoDataValue(&success);
802 2 : if (!success)
803 2 : val = 0.0;
804 :
805 : // TODO: test band by band if data is interleaved
806 2 : if (isAllVal(eDataType, ob, img.pageSizeBytes, val))
807 : {
808 : // Mark it empty and checked, ignore the possible write error
809 0 : poMRFDS->WriteTile((void *)1, infooffset, 0);
810 0 : if (1 == cstride)
811 0 : return CE_None;
812 0 : return ReadInterleavedBlock(xblk, yblk, buffer);
813 : }
814 :
815 : // Write the page in the local cache
816 :
817 : // Have to use a separate buffer for compression output.
818 2 : void *outbuff = VSIMalloc(poMRFDS->pbsize);
819 2 : if (nullptr == outbuff)
820 : {
821 0 : CPLError(CE_Failure, CPLE_AppDefined,
822 : "Can't get buffer for writing page");
823 : // This is not really an error for a cache, the data is fine
824 0 : return CE_Failure;
825 : }
826 :
827 2 : buf_mgr filedst = {static_cast<char *>(outbuff), poMRFDS->pbsize};
828 2 : auto start_time = steady_clock::now();
829 2 : Compress(filedst, filesrc);
830 :
831 : // Where the output is, in case we deflate
832 2 : void *usebuff = outbuff;
833 2 : if (dodeflate)
834 : {
835 0 : usebuff = DeflateBlock(filedst, poMRFDS->pbsize - filedst.size,
836 : deflate_flags);
837 0 : if (!usebuff)
838 : {
839 0 : CPLError(CE_Failure, CPLE_AppDefined, "MRF: Deflate error");
840 0 : return CE_Failure;
841 : }
842 : }
843 :
844 : #if defined(ZSTD_SUPPORT)
845 2 : else if (dozstd)
846 : {
847 0 : size_t ranks = 0; // Assume no need for byte rank sort
848 0 : if (img.comp == IL_NONE || img.comp == IL_ZSTD)
849 0 : ranks =
850 0 : static_cast<size_t>(GDALGetDataTypeSizeBytes(img.dt)) * cstride;
851 0 : usebuff = ZstdCompBlock(filedst, poMRFDS->pbsize - filedst.size,
852 0 : zstd_level, poMRFDS->getzsc(), ranks);
853 0 : if (!usebuff)
854 : {
855 0 : CPLError(CE_Failure, CPLE_AppDefined,
856 : "MRF: ZSTD compression error");
857 0 : return CE_Failure;
858 : }
859 : }
860 : #endif
861 :
862 2 : poMRFDS->write_timer +=
863 2 : duration_cast<nanoseconds>(steady_clock::now() - start_time);
864 :
865 : // Write and update the tile index
866 2 : ret = poMRFDS->WriteTile(usebuff, infooffset, filedst.size);
867 2 : CPLFree(outbuff);
868 :
869 : // If we hit an error or if unpaking is not needed
870 2 : if (ret != CE_None || cstride == 1)
871 2 : return ret;
872 :
873 : // data is already in DS buffer, deinterlace it in pixel blocks
874 0 : return ReadInterleavedBlock(xblk, yblk, buffer);
875 : }
876 :
877 : /**
878 : *\brief Fetch for a cloned MRF
879 : *
880 : * @param xblk The X block number, zero based
881 : * @param yblk The Y block number, zero based
882 : * @param buffer buffer
883 : *
884 : */
885 :
886 1 : CPLErr MRFRasterBand::FetchClonedBlock(int xblk, int yblk, void *buffer)
887 : {
888 1 : CPLDebug("MRF_IB", "FetchClonedBlock %d,%d,0,%d, level %d\n", xblk, yblk,
889 : nBand, m_l);
890 :
891 : // Paranoid check
892 1 : assert(poMRFDS->clonedSource);
893 1 : MRFDataset *poSrc = static_cast<MRFDataset *>(poMRFDS->GetSrcDS());
894 1 : if (nullptr == poSrc)
895 : {
896 0 : CPLError(CE_Failure, CPLE_AppDefined, "MRF: Can't open source file %s",
897 0 : poMRFDS->source.c_str());
898 0 : return CE_Failure;
899 : }
900 :
901 1 : if (poMRFDS->bypass_cache || GF_Read == DataMode())
902 : {
903 : // Can't store, so just fetch from source, which is an MRF with
904 : // identical structure
905 : MRFRasterBand *b =
906 0 : static_cast<MRFRasterBand *>(poSrc->GetRasterBand(nBand));
907 0 : if (b->GetOverviewCount() && m_l)
908 0 : b = static_cast<MRFRasterBand *>(b->GetOverview(m_l - 1));
909 0 : if (b == nullptr)
910 0 : return CE_Failure;
911 0 : return b->IReadBlock(xblk, yblk, buffer);
912 : }
913 :
914 1 : ILSize req(xblk, yblk, 0, (nBand - 1) / img.pagesize.c, m_l);
915 : ILIdx tinfo;
916 :
917 : // Get the cloned source tile info
918 : // The cloned source index is after the current one
919 1 : if (CE_None != poMRFDS->ReadTileIdx(tinfo, req, img, poMRFDS->idxSize))
920 : {
921 0 : CPLError(CE_Failure, CPLE_AppDefined,
922 : "MRF: Unable to read cloned index entry");
923 0 : return CE_Failure;
924 : }
925 :
926 1 : GUIntBig infooffset = IdxOffset(req, img);
927 : CPLErr err;
928 :
929 : // Does the source have this tile?
930 1 : if (tinfo.size == 0)
931 : { // Nope, mark it empty and return fill
932 0 : err = poMRFDS->WriteTile((void *)1, infooffset, 0);
933 0 : if (CE_None != err)
934 0 : return err;
935 0 : return FillBlock(buffer);
936 : }
937 :
938 1 : VSILFILE *srcfd = poSrc->DataFP();
939 1 : if (nullptr == srcfd)
940 : {
941 0 : CPLError(CE_Failure, CPLE_AppDefined,
942 : "MRF: Can't open source data file %s",
943 0 : poMRFDS->source.c_str());
944 0 : return CE_Failure;
945 : }
946 :
947 : // Need to read the tile from the source
948 1 : if (tinfo.size <= 0 || tinfo.size > INT_MAX)
949 : {
950 0 : CPLError(CE_Failure, CPLE_OutOfMemory,
951 : "Invalid tile size " CPL_FRMT_GIB, tinfo.size);
952 0 : return CE_Failure;
953 : }
954 1 : char *buf = static_cast<char *>(VSIMalloc(static_cast<size_t>(tinfo.size)));
955 1 : if (buf == nullptr)
956 : {
957 0 : CPLError(CE_Failure, CPLE_OutOfMemory,
958 : "Cannot allocate " CPL_FRMT_GIB " bytes", tinfo.size);
959 0 : return CE_Failure;
960 : }
961 :
962 1 : VSIFSeekL(srcfd, tinfo.offset, SEEK_SET);
963 2 : if (tinfo.size !=
964 1 : GIntBig(VSIFReadL(buf, 1, static_cast<size_t>(tinfo.size), srcfd)))
965 : {
966 0 : CPLFree(buf);
967 0 : CPLError(CE_Failure, CPLE_AppDefined,
968 : "MRF: Can't read data from source %s",
969 : poSrc->current.datfname.c_str());
970 0 : return CE_Failure;
971 : }
972 :
973 : // Write it then reissue the read
974 1 : err = poMRFDS->WriteTile(buf, infooffset, tinfo.size);
975 1 : CPLFree(buf);
976 1 : if (CE_None != err)
977 0 : return err;
978 : // Reissue read, it will work from the cloned data
979 1 : return IReadBlock(xblk, yblk, buffer);
980 : }
981 :
982 : /**
983 : *\brief read a block in the provided buffer
984 : *
985 : * For separate band model, the DS buffer is not used, the read is direct in
986 : *the buffer For pixel interleaved model, the DS buffer holds the temp copy and
987 : *all the other bands are force read
988 : *
989 : */
990 :
991 1882 : CPLErr MRFRasterBand::IReadBlock(int xblk, int yblk, void *buffer)
992 : {
993 1882 : GInt32 cstride = img.pagesize.c;
994 : ILIdx tinfo;
995 1882 : ILSize req(xblk, yblk, 0, (nBand - 1) / cstride, m_l);
996 3764 : CPLDebug("MRF_IB",
997 : "IReadBlock %d,%d,0,%d, level %d, idxoffset " CPL_FRMT_GIB "\n",
998 1882 : xblk, yblk, nBand - 1, m_l, IdxOffset(req, img));
999 :
1000 : // If this is a caching file and bypass is on, just do the fetch
1001 1882 : if (poMRFDS->bypass_cache && !poMRFDS->source.empty())
1002 0 : return FetchBlock(xblk, yblk, buffer);
1003 :
1004 1882 : tinfo.size = 0; // Just in case it is missing
1005 1882 : if (CE_None != poMRFDS->ReadTileIdx(tinfo, req, img))
1006 : {
1007 0 : if (!poMRFDS->no_errors)
1008 : {
1009 0 : CPLError(CE_Failure, CPLE_AppDefined,
1010 : "MRF: Unable to read index at offset " CPL_FRMT_GIB,
1011 0 : IdxOffset(req, img));
1012 0 : return CE_Failure;
1013 : }
1014 0 : return FillBlock(buffer);
1015 : }
1016 :
1017 1882 : if (0 == tinfo.size)
1018 : { // Could be missing or it could be caching
1019 : // Offset != 0 means no data, Update mode is for local MRFs only
1020 : // if caching index mode is RO don't try to fetch
1021 : // Also, caching MRFs can't be opened in update mode
1022 12 : if (0 != tinfo.offset || GA_Update == poMRFDS->eAccess ||
1023 18 : poMRFDS->source.empty() || IdxMode() == GF_Read)
1024 2 : return FillBlock(buffer);
1025 :
1026 : // caching MRF, need to fetch a block
1027 4 : return FetchBlock(xblk, yblk, buffer);
1028 : }
1029 :
1030 1876 : CPLDebug("MRF_IB", "Tinfo offset " CPL_FRMT_GIB ", size " CPL_FRMT_GIB "\n",
1031 : tinfo.offset, tinfo.size);
1032 : // If we have a tile, read it
1033 :
1034 : // Should use a permanent buffer, like the pbuffer mechanism
1035 : // Get a large buffer, in case we need to unzip
1036 :
1037 : // We add a padding of 3 bytes since in LERC1 decompression, we can
1038 : // dereference a unsigned int at the end of the buffer, that can be
1039 : // partially out of the buffer.
1040 : // Can be reproduced with :
1041 : // gdal_translate ../autotest/gcore/data/byte.tif out.mrf -of MRF -co
1042 : // COMPRESS=LERC -co OPTIONS=V1:YES -ot Float32 valgrind gdalinfo -checksum
1043 : // out.mrf Invalid read of size 4 at BitStuffer::read(unsigned char**,
1044 : // std::vector<unsigned int, std::allocator<unsigned int> >&) const
1045 : // (BitStuffer.cpp:153)
1046 :
1047 : // No stored tile should be larger than twice the raw size.
1048 1876 : if (tinfo.size <= 0 || tinfo.size > poMRFDS->pbsize * 2)
1049 : {
1050 0 : if (!poMRFDS->no_errors)
1051 : {
1052 0 : CPLError(CE_Failure, CPLE_OutOfMemory,
1053 : "Stored tile is too large: " CPL_FRMT_GIB, tinfo.size);
1054 0 : return CE_Failure;
1055 : }
1056 0 : return FillBlock(buffer);
1057 : }
1058 :
1059 1876 : VSILFILE *dfp = DataFP();
1060 :
1061 : // No data file to read from
1062 1876 : if (dfp == nullptr)
1063 0 : return CE_Failure;
1064 :
1065 1876 : void *data = VSIMalloc(static_cast<size_t>(tinfo.size + PADDING_BYTES));
1066 1876 : if (data == nullptr)
1067 : {
1068 0 : CPLError(CE_Failure, CPLE_OutOfMemory,
1069 : "Could not allocate memory for tile size: " CPL_FRMT_GIB,
1070 : tinfo.size);
1071 0 : return CE_Failure;
1072 : }
1073 :
1074 : // This part is not thread safe, but it is what GDAL expects
1075 1876 : VSIFSeekL(dfp, tinfo.offset, SEEK_SET);
1076 1876 : if (1 != VSIFReadL(data, static_cast<size_t>(tinfo.size), 1, dfp))
1077 : {
1078 0 : CPLFree(data);
1079 0 : if (poMRFDS->no_errors)
1080 0 : return FillBlock(buffer);
1081 0 : CPLError(CE_Failure, CPLE_AppDefined, "Unable to read data page, %d@%x",
1082 0 : static_cast<int>(tinfo.size), static_cast<int>(tinfo.offset));
1083 0 : return CE_Failure;
1084 : }
1085 :
1086 : /* initialize padding bytes */
1087 1876 : memset(((char *)data) + static_cast<size_t>(tinfo.size), 0, PADDING_BYTES);
1088 1876 : buf_mgr src = {(char *)data, static_cast<size_t>(tinfo.size)};
1089 : buf_mgr dst;
1090 :
1091 1876 : auto start_time = steady_clock::now();
1092 :
1093 : // We got the data, do we need to decompress it before decoding?
1094 1876 : if (dodeflate)
1095 : {
1096 9 : if (img.pageSizeBytes > INT_MAX - 1440)
1097 : {
1098 0 : CPLFree(data);
1099 0 : CPLError(CE_Failure, CPLE_AppDefined, "Page size is too big at %d",
1100 : img.pageSizeBytes);
1101 0 : return CE_Failure;
1102 : }
1103 9 : dst.size =
1104 9 : img.pageSizeBytes +
1105 : 1440; // in case the packed page is a bit larger than the raw one
1106 9 : dst.buffer = static_cast<char *>(VSIMalloc(dst.size));
1107 9 : if (nullptr == dst.buffer)
1108 : {
1109 0 : CPLFree(data);
1110 0 : CPLError(CE_Failure, CPLE_OutOfMemory, "Cannot allocate %d bytes",
1111 0 : static_cast<int>(dst.size));
1112 0 : return CE_Failure;
1113 : }
1114 :
1115 9 : if (ZUnPack(src, dst, deflate_flags))
1116 : { // Got it unpacked, update the pointers
1117 9 : CPLFree(data);
1118 9 : data = dst.buffer;
1119 9 : tinfo.size = dst.size;
1120 : }
1121 : else
1122 : { // assume the page was not gzipped, warn only
1123 0 : CPLFree(dst.buffer);
1124 0 : if (!poMRFDS->no_errors)
1125 0 : CPLError(CE_Warning, CPLE_AppDefined, "Can't inflate page!");
1126 : }
1127 : }
1128 :
1129 : #if defined(ZSTD_SUPPORT)
1130 : // undo ZSTD
1131 1867 : else if (dozstd)
1132 : {
1133 10 : auto ctx = poMRFDS->getzsd();
1134 10 : if (!ctx)
1135 : {
1136 0 : CPLFree(data);
1137 0 : CPLError(CE_Failure, CPLE_AppDefined, "Can't acquire ZSTD context");
1138 0 : return CE_Failure;
1139 : }
1140 10 : if (img.pageSizeBytes > INT_MAX - 1440)
1141 : {
1142 0 : CPLFree(data);
1143 0 : CPLError(CE_Failure, CPLE_AppDefined, "Page is too large at %d",
1144 : img.pageSizeBytes);
1145 0 : return CE_Failure;
1146 : }
1147 10 : dst.size =
1148 10 : img.pageSizeBytes +
1149 : 1440; // Allow for a slight increase from previous compressions
1150 10 : dst.buffer = static_cast<char *>(VSIMalloc(dst.size));
1151 10 : if (nullptr == dst.buffer)
1152 : {
1153 0 : CPLFree(data);
1154 0 : CPLError(CE_Failure, CPLE_OutOfMemory, "Cannot allocate %d bytes",
1155 0 : static_cast<int>(dst.size));
1156 0 : return CE_Failure;
1157 : }
1158 :
1159 20 : auto raw_size = ZSTD_decompressDCtx(ctx, dst.buffer, dst.size,
1160 10 : src.buffer, src.size);
1161 10 : if (ZSTD_isError(raw_size))
1162 : { // assume page was not packed, warn only
1163 0 : CPLFree(dst.buffer);
1164 0 : if (!poMRFDS->no_errors)
1165 0 : CPLError(CE_Warning, CPLE_AppDefined,
1166 : "Can't unpack ZSTD page!");
1167 : }
1168 : else
1169 : {
1170 10 : CPLFree(data); // The compressed data
1171 10 : data = dst.buffer;
1172 10 : tinfo.size = raw_size;
1173 : // Might need to undo the rank sort
1174 10 : size_t ranks = 0;
1175 10 : if (img.comp == IL_NONE || img.comp == IL_ZSTD)
1176 10 : ranks = static_cast<size_t>(GDALGetDataTypeSizeBytes(img.dt)) *
1177 10 : img.pagesize.c;
1178 10 : if (ranks)
1179 : {
1180 10 : src.buffer = static_cast<char *>(data);
1181 10 : src.size = static_cast<size_t>(tinfo.size);
1182 10 : derank(src, ranks);
1183 : }
1184 : }
1185 : }
1186 : #endif
1187 :
1188 1876 : src.buffer = static_cast<char *>(data);
1189 1876 : src.size = static_cast<size_t>(tinfo.size);
1190 :
1191 : // After unpacking, the size has to be pageSizeBytes
1192 : // If pages are interleaved, use the dataset page buffer instead
1193 1876 : dst.buffer = reinterpret_cast<char *>(
1194 8 : (1 == cstride) ? buffer : poMRFDS->GetPBuffer());
1195 1876 : dst.size = img.pageSizeBytes;
1196 :
1197 1876 : if (poMRFDS->no_errors)
1198 0 : CPLPushErrorHandler(CPLQuietErrorHandler);
1199 1876 : CPLErr ret = Decompress(dst, src);
1200 :
1201 1876 : poMRFDS->read_timer +=
1202 1876 : duration_cast<nanoseconds>(steady_clock::now() - start_time);
1203 :
1204 1876 : dst.size =
1205 1876 : img.pageSizeBytes; // In case the decompress failed, force it back
1206 :
1207 : // Swap whatever we decompressed if we need to
1208 1876 : if (is_Endianness_Dependent(img.dt, img.comp) && (img.nbo != NET_ORDER))
1209 0 : swab_buff(dst, img);
1210 :
1211 1876 : CPLFree(data);
1212 1876 : if (poMRFDS->no_errors)
1213 : {
1214 0 : CPLPopErrorHandler();
1215 0 : if (ret != CE_None) // Set each page buffer to the correct no data
1216 : // value, then proceed
1217 0 : return (1 == cstride) ? FillBlock(buffer)
1218 0 : : FillBlock(xblk, yblk, buffer);
1219 : }
1220 :
1221 : // If pages are separate or we had errors, we're done
1222 1876 : if (1 == cstride || CE_None != ret)
1223 1868 : return ret;
1224 :
1225 : // De-interleave page from dataset buffer and return
1226 8 : return ReadInterleavedBlock(xblk, yblk, buffer);
1227 : }
1228 :
1229 : /**
1230 : *\brief Write a block from the provided buffer
1231 : *
1232 : * Same trick as read, use a temporary tile buffer for pixel interleave
1233 : * For band separate, use a
1234 : * Write the block once it has all the bands, report
1235 : * if a new block is started before the old one was completed
1236 : *
1237 : */
1238 :
1239 5081 : CPLErr MRFRasterBand::IWriteBlock(int xblk, int yblk, void *buffer)
1240 : {
1241 5081 : GInt32 cstride = img.pagesize.c;
1242 5081 : ILSize req(xblk, yblk, 0, (nBand - 1) / cstride, m_l);
1243 5081 : GUIntBig infooffset = IdxOffset(req, img);
1244 :
1245 5081 : CPLDebug("MRF_IB", "IWriteBlock %d,%d,0,%d, level %d, stride %d\n", xblk,
1246 : yblk, nBand, m_l, cstride);
1247 :
1248 : // Finish the Create call
1249 5081 : if (!poMRFDS->bCrystalized && !poMRFDS->Crystalize())
1250 : {
1251 0 : CPLError(CE_Failure, CPLE_AppDefined, "MRF: Error creating files");
1252 0 : return CE_Failure;
1253 : }
1254 :
1255 5081 : if (1 == cstride)
1256 : { // Separate bands, we can write it as is
1257 : // Empty page skip
1258 : int success;
1259 5070 : double val = GetNoDataValue(&success);
1260 5070 : if (!success)
1261 4998 : val = 0.0;
1262 5070 : if (isAllVal(eDataType, buffer, img.pageSizeBytes, val))
1263 1 : return poMRFDS->WriteTile(nullptr, infooffset, 0);
1264 :
1265 : // Use the pbuffer to hold the compressed page before writing it
1266 5069 : poMRFDS->tile = ILSize(); // Mark it corrupt
1267 :
1268 : buf_mgr src;
1269 5069 : src.buffer = (char *)buffer;
1270 5069 : src.size = static_cast<size_t>(img.pageSizeBytes);
1271 5069 : buf_mgr dst = {(char *)poMRFDS->GetPBuffer(),
1272 5069 : poMRFDS->GetPBufferSize()};
1273 :
1274 : // Swab the source before encoding if we need to
1275 5069 : if (is_Endianness_Dependent(img.dt, img.comp) && (img.nbo != NET_ORDER))
1276 0 : swab_buff(src, img);
1277 :
1278 5069 : auto start_time = steady_clock::now();
1279 :
1280 : // Compress functions need to return the compressed size in
1281 : // the bytes in buffer field
1282 5069 : Compress(dst, src);
1283 5069 : void *usebuff = dst.buffer;
1284 5069 : if (dodeflate)
1285 : {
1286 : usebuff =
1287 9 : DeflateBlock(dst, poMRFDS->pbsize - dst.size, deflate_flags);
1288 9 : if (!usebuff)
1289 : {
1290 0 : CPLError(CE_Failure, CPLE_AppDefined, "MRF: Deflate error");
1291 0 : return CE_Failure;
1292 : }
1293 : }
1294 :
1295 : #if defined(ZSTD_SUPPORT)
1296 5060 : else if (dozstd)
1297 : {
1298 11 : size_t ranks = 0; // Assume no need for byte rank sort
1299 11 : if (img.comp == IL_NONE || img.comp == IL_ZSTD)
1300 11 : ranks = static_cast<size_t>(GDALGetDataTypeSizeBytes(img.dt));
1301 11 : usebuff = ZstdCompBlock(dst, poMRFDS->pbsize - dst.size, zstd_level,
1302 11 : poMRFDS->getzsc(), ranks);
1303 11 : if (!usebuff)
1304 : {
1305 0 : CPLError(CE_Failure, CPLE_AppDefined,
1306 : "MRF: Zstd Compression error");
1307 0 : return CE_Failure;
1308 : }
1309 : }
1310 : #endif
1311 5069 : poMRFDS->write_timer +=
1312 5069 : duration_cast<nanoseconds>(steady_clock::now() - start_time);
1313 5069 : return poMRFDS->WriteTile(usebuff, infooffset, dst.size);
1314 : }
1315 :
1316 : // Multiple bands per page, use a temporary to assemble the page
1317 : // Temporary is large because we use it to hold both the uncompressed and
1318 : // the compressed
1319 11 : poMRFDS->tile = req;
1320 11 : poMRFDS->bdirty = 0;
1321 :
1322 : // Keep track of what bands are empty
1323 11 : GUIntBig empties = 0;
1324 :
1325 11 : void *tbuffer = VSIMalloc(img.pageSizeBytes + poMRFDS->pbsize);
1326 :
1327 11 : if (!tbuffer)
1328 : {
1329 0 : CPLError(CE_Failure, CPLE_AppDefined,
1330 : "MRF: Can't allocate write buffer");
1331 0 : return CE_Failure;
1332 : }
1333 :
1334 : // Get the other bands from the block cache
1335 44 : for (int iBand = 0; iBand < poMRFDS->nBands; iBand++)
1336 : {
1337 33 : char *pabyThisImage = nullptr;
1338 33 : GDALRasterBlock *poBlock = nullptr;
1339 :
1340 33 : if (iBand == nBand - 1)
1341 : {
1342 11 : pabyThisImage = reinterpret_cast<char *>(buffer);
1343 11 : poMRFDS->bdirty |= bandbit();
1344 : }
1345 : else
1346 : {
1347 22 : GDALRasterBand *band = poMRFDS->GetRasterBand(iBand + 1);
1348 : // Pick the right overview
1349 22 : if (m_l)
1350 0 : band = band->GetOverview(m_l - 1);
1351 : poBlock = (reinterpret_cast<MRFRasterBand *>(band))
1352 22 : ->TryGetLockedBlockRef(xblk, yblk);
1353 22 : if (nullptr == poBlock)
1354 0 : continue;
1355 : // This is where the image data is for this band
1356 :
1357 22 : pabyThisImage = reinterpret_cast<char *>(poBlock->GetDataRef());
1358 22 : poMRFDS->bdirty |= bandbit(iBand);
1359 : }
1360 :
1361 : // Keep track of empty bands, but encode them anyhow, in case some are
1362 : // not empty
1363 : int success;
1364 33 : double val = GetNoDataValue(&success);
1365 33 : if (!success)
1366 33 : val = 0.0;
1367 33 : if (isAllVal(eDataType, pabyThisImage, blockSizeBytes(), val))
1368 6 : empties |= bandbit(iBand);
1369 :
1370 : // Copy the data into the dataset buffer here
1371 : // Just the right mix of templates and macros make this real tidy
1372 : #define CpySO(T) \
1373 : cpy_stride_out<T>((reinterpret_cast<T *>(tbuffer)) + iBand, pabyThisImage, \
1374 : blockSizeBytes() / sizeof(T), cstride)
1375 :
1376 : // Build the page in tbuffer
1377 33 : switch (GDALGetDataTypeSize(eDataType) / 8)
1378 : {
1379 33 : case 1:
1380 33 : CpySO(GByte);
1381 33 : break;
1382 0 : case 2:
1383 0 : CpySO(GInt16);
1384 0 : break;
1385 0 : case 4:
1386 0 : CpySO(GInt32);
1387 0 : break;
1388 0 : case 8:
1389 0 : CpySO(GIntBig);
1390 0 : break;
1391 0 : default:
1392 : {
1393 0 : CPLError(CE_Failure, CPLE_AppDefined,
1394 : "MRF: Write datatype of %d bytes "
1395 : "not implemented",
1396 0 : GDALGetDataTypeSize(eDataType) / 8);
1397 0 : if (poBlock != nullptr)
1398 : {
1399 0 : poBlock->MarkClean();
1400 0 : poBlock->DropLock();
1401 : }
1402 0 : CPLFree(tbuffer);
1403 0 : return CE_Failure;
1404 : }
1405 : }
1406 :
1407 33 : if (poBlock != nullptr)
1408 : {
1409 22 : poBlock->MarkClean();
1410 22 : poBlock->DropLock();
1411 : }
1412 : }
1413 :
1414 : // Should keep track of the individual band buffers and only mix them if
1415 : // this is not an empty page ( move the Copy with Stride Out from above
1416 : // below this test This way works fine, but it does work extra for empty
1417 : // pages
1418 :
1419 11 : if (GIntBig(empties) == AllBandMask())
1420 : {
1421 0 : CPLFree(tbuffer);
1422 0 : return poMRFDS->WriteTile(nullptr, infooffset, 0);
1423 : }
1424 :
1425 11 : if (poMRFDS->bdirty != AllBandMask())
1426 0 : CPLError(CE_Warning, CPLE_AppDefined,
1427 : "MRF: IWrite, band dirty mask is " CPL_FRMT_GIB
1428 : " instead of " CPL_FRMT_GIB,
1429 0 : poMRFDS->bdirty, AllBandMask());
1430 :
1431 : buf_mgr src;
1432 11 : src.buffer = (char *)tbuffer;
1433 11 : src.size = static_cast<size_t>(img.pageSizeBytes);
1434 :
1435 : // Use the space after pagesizebytes for compressed output, it is of pbsize
1436 11 : char *outbuff = (char *)tbuffer + img.pageSizeBytes;
1437 :
1438 11 : buf_mgr dst = {outbuff, poMRFDS->pbsize};
1439 : CPLErr ret;
1440 :
1441 11 : auto start_time = steady_clock::now();
1442 :
1443 11 : ret = Compress(dst, src);
1444 11 : if (ret != CE_None)
1445 : {
1446 : // Compress failed, write it as an empty tile
1447 0 : CPLFree(tbuffer);
1448 0 : poMRFDS->WriteTile(nullptr, infooffset, 0);
1449 0 : return CE_None; // Should report the error, but it triggers partial
1450 : // band attempts
1451 : }
1452 :
1453 : // Where the output is, in case we deflate
1454 11 : void *usebuff = outbuff;
1455 11 : if (dodeflate)
1456 : {
1457 : // Move the packed part at the start of tbuffer, to make more space
1458 : // available
1459 0 : memcpy(tbuffer, outbuff, dst.size);
1460 0 : dst.buffer = static_cast<char *>(tbuffer);
1461 0 : usebuff = DeflateBlock(dst,
1462 0 : static_cast<size_t>(img.pageSizeBytes) +
1463 0 : poMRFDS->pbsize - dst.size,
1464 : deflate_flags);
1465 0 : if (!usebuff)
1466 0 : CPLError(CE_Failure, CPLE_AppDefined, "MRF: Deflate error");
1467 : }
1468 :
1469 : #if defined(ZSTD_SUPPORT)
1470 11 : else if (dozstd)
1471 : {
1472 1 : memcpy(tbuffer, outbuff, dst.size);
1473 1 : dst.buffer = static_cast<char *>(tbuffer);
1474 1 : size_t ranks = 0; // Assume no need for byte rank sort
1475 1 : if (img.comp == IL_NONE || img.comp == IL_ZSTD)
1476 1 : ranks =
1477 1 : static_cast<size_t>(GDALGetDataTypeSizeBytes(img.dt)) * cstride;
1478 3 : usebuff = ZstdCompBlock(dst,
1479 1 : static_cast<size_t>(img.pageSizeBytes) +
1480 1 : poMRFDS->pbsize - dst.size,
1481 1 : zstd_level, poMRFDS->getzsc(), ranks);
1482 1 : if (!usebuff)
1483 0 : CPLError(CE_Failure, CPLE_AppDefined,
1484 : "MRF: ZStd compression error");
1485 : }
1486 : #endif
1487 :
1488 11 : poMRFDS->write_timer +=
1489 11 : duration_cast<nanoseconds>(steady_clock::now() - start_time);
1490 :
1491 11 : if (!usebuff)
1492 : { // Error was signaled
1493 0 : CPLFree(tbuffer);
1494 0 : poMRFDS->WriteTile(nullptr, infooffset, 0);
1495 0 : poMRFDS->bdirty = 0;
1496 0 : return CE_Failure;
1497 : }
1498 :
1499 11 : ret = poMRFDS->WriteTile(usebuff, infooffset, dst.size);
1500 11 : CPLFree(tbuffer);
1501 :
1502 11 : poMRFDS->bdirty = 0;
1503 11 : return ret;
1504 : }
1505 :
1506 : //
1507 : // Tests if a given block exists without reading it
1508 : // returns false only when it is definitely not existing
1509 : //
1510 160 : bool MRFRasterBand::TestBlock(int xblk, int yblk)
1511 : {
1512 : // When bypassing the cache, assume all blocks are valid
1513 160 : if (poMRFDS->bypass_cache && !poMRFDS->source.empty())
1514 0 : return true;
1515 :
1516 : // Blocks outside of image have no data by default
1517 160 : if (xblk < 0 || yblk < 0 || xblk >= img.pagecount.x ||
1518 146 : yblk >= img.pagecount.y)
1519 25 : return false;
1520 :
1521 : ILIdx tinfo;
1522 135 : GInt32 cstride = img.pagesize.c;
1523 135 : ILSize req(xblk, yblk, 0, (nBand - 1) / cstride, m_l);
1524 :
1525 135 : if (CE_None != poMRFDS->ReadTileIdx(tinfo, req, img))
1526 : // Got an error reading the tile index
1527 0 : return !poMRFDS->no_errors;
1528 :
1529 : // Got an index, if the size is readable, the block does exist
1530 135 : if (0 < tinfo.size && tinfo.size < poMRFDS->pbsize * 2)
1531 135 : return true;
1532 :
1533 : // We are caching, but the tile has not been checked, so it could exist
1534 0 : return (!poMRFDS->source.empty() && 0 == tinfo.offset);
1535 : }
1536 :
1537 183 : int MRFRasterBand::GetOverviewCount()
1538 : {
1539 : // First try internal overviews
1540 183 : int nInternalOverviewCount = static_cast<int>(overviews.size());
1541 183 : if (nInternalOverviewCount > 0)
1542 137 : return nInternalOverviewCount;
1543 46 : return GDALPamRasterBand::GetOverviewCount();
1544 : }
1545 :
1546 158 : GDALRasterBand *MRFRasterBand::GetOverview(int n)
1547 : {
1548 : // First try internal overviews
1549 158 : if (n >= 0 && n < (int)overviews.size())
1550 85 : return overviews[n];
1551 73 : return GDALPamRasterBand::GetOverview(n);
1552 : }
1553 :
1554 : NAMESPACE_MRF_END
|
