Line data Source code
1 : /*
2 : * Copyright (c) 2002-2012, California Institute of Technology.
3 : * All rights reserved. Based on Government Sponsored Research under contracts
4 : * NAS7-1407 and/or NAS7-03001.
5 : *
6 : * Redistribution and use in source and binary forms, with or without
7 : * modification, are permitted provided that the following conditions are met:
8 : * 1. Redistributions of source code must retain the above copyright notice,
9 : * this list of conditions and the following disclaimer.
10 : * 2. Redistributions in binary form must reproduce the above copyright
11 : * notice, this list of conditions and the following disclaimer in the
12 : * documentation and/or other materials provided with the distribution.
13 : * 3. Neither the name of the California Institute of Technology (Caltech),
14 : * its operating division the Jet Propulsion Laboratory (JPL), the National
15 : * Aeronautics and Space Administration (NASA), nor the names of its
16 : * contributors may be used to endorse or promote products derived from this
17 : * software without specific prior written permission.
18 : *
19 : * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20 : * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 : * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 : * ARE DISCLAIMED. IN NO EVENT SHALL THE CALIFORNIA INSTITUTE OF TECHNOLOGY BE
23 : * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 : * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 : * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 : * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 : * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 : * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 : * POSSIBILITY OF SUCH DAMAGE.
30 : *
31 : * Copyright 2014-2021 Esri
32 : *
33 : * Licensed under the Apache License, Version 2.0 (the "License");
34 : * you may not use this file except in compliance with the License.
35 : * You may obtain a copy of the License at
36 : *
37 : * http://www.apache.org/licenses/LICENSE-2.0
38 : *
39 : * Unless required by applicable law or agreed to in writing, software
40 : * distributed under the License is distributed on an "AS IS" BASIS,
41 : * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
42 : * See the License for the specific language governing permissions and
43 : * limitations under the License.
44 : */
45 :
46 : /******************************************************************************
47 : *
48 : * Project: Meta Raster File Format Driver Implementation, RasterBand
49 : * Purpose: Implementation of MRF band
50 : *
51 : * Author: Lucian Plesea, Lucian.Plesea jpl.nasa.gov, lplesea esri.com
52 : *
53 : ****************************************************************************/
54 :
55 : #include "marfa.h"
56 : #include "gdal_priv.h"
57 : #include "ogr_srs_api.h"
58 : #include "ogr_spatialref.h"
59 :
60 : #include <vector>
61 : #include <cassert>
62 : #include <zlib.h>
63 : #if defined(ZSTD_SUPPORT)
64 : #include <zstd.h>
65 : #endif
66 :
67 : using namespace std::chrono;
68 :
69 : NAMESPACE_MRF_START
70 :
71 : // packs a block of a given type, with a stride
72 : // Count is the number of items that need to be copied
73 : // These are separate to allow for optimization
74 :
75 : template <typename T>
76 24 : static void cpy_stride_in(void *dst, void *src, int c, int stride)
77 : {
78 24 : T *s = reinterpret_cast<T *>(src);
79 24 : T *d = reinterpret_cast<T *>(dst);
80 :
81 6291480 : while (c--)
82 : {
83 6291460 : *d++ = *s;
84 6291460 : s += stride;
85 : }
86 24 : }
87 :
88 : template <typename T>
89 33 : static void cpy_stride_out(void *dst, void *src, int c, int stride)
90 : {
91 33 : T *s = reinterpret_cast<T *>(src);
92 33 : T *d = reinterpret_cast<T *>(dst);
93 :
94 8650780 : while (c--)
95 : {
96 8650750 : *d = *s++;
97 8650750 : d += stride;
98 : }
99 33 : }
100 :
101 : // Does every value in the buffer have the same value, using strict comparison
102 : template <typename T>
103 5105 : inline int isAllVal(const T *b, size_t bytecount, double ndv)
104 : {
105 5105 : T val = static_cast<T>(ndv);
106 5105 : size_t count = bytecount / sizeof(T);
107 1843727 : for (; count; --count)
108 : {
109 1843727 : if (*(b++) != val)
110 : {
111 5098 : return FALSE;
112 : }
113 : }
114 7 : return TRUE;
115 : }
116 :
117 : // Dispatcher based on gdal data type
118 5105 : static int isAllVal(GDALDataType gt, void *b, size_t bytecount, double ndv)
119 : {
120 : // Test to see if it has data
121 5105 : int isempty = false;
122 :
123 : // A case branch in a temporary macro, conversion from gdal enum to type
124 : #define TEST_T(GType, T) \
125 : case GType: \
126 : isempty = isAllVal(reinterpret_cast<T *>(b), bytecount, ndv); \
127 : break
128 :
129 5105 : switch (gt)
130 : {
131 4942 : TEST_T(GDT_Byte, GByte);
132 0 : TEST_T(GDT_Int8, GInt8);
133 28 : TEST_T(GDT_UInt16, GUInt16);
134 27 : TEST_T(GDT_Int16, GInt16);
135 26 : TEST_T(GDT_UInt32, GUInt32);
136 26 : TEST_T(GDT_Int32, GInt32);
137 0 : TEST_T(GDT_UInt64, GUInt64);
138 4 : TEST_T(GDT_Int64, GInt64);
139 26 : TEST_T(GDT_Float32, float);
140 26 : TEST_T(GDT_Float64, double);
141 0 : default:
142 0 : break;
143 : }
144 : #undef TEST_T
145 :
146 5105 : return isempty;
147 : }
148 :
149 : // Swap bytes in place, unconditional
150 : // cppcheck-suppress constParameterReference
151 0 : static void swab_buff(buf_mgr &src, const ILImage &img)
152 : {
153 : size_t i;
154 0 : switch (GDALGetDataTypeSize(img.dt))
155 : {
156 0 : case 16:
157 : {
158 0 : short int *b = (short int *)src.buffer;
159 0 : for (i = src.size / 2; i; b++, i--)
160 0 : *b = swab16(*b);
161 0 : break;
162 : }
163 0 : case 32:
164 : {
165 0 : int *b = (int *)src.buffer;
166 0 : for (i = src.size / 4; i; b++, i--)
167 0 : *b = swab32(*b);
168 0 : break;
169 : }
170 0 : case 64:
171 : {
172 0 : long long *b = (long long *)src.buffer;
173 0 : for (i = src.size / 8; i; b++, i--)
174 0 : *b = swab64(*b);
175 0 : break;
176 : }
177 : }
178 0 : }
179 :
180 : // Similar to compress2() but with flags to control zlib features
181 : // Returns true if it worked
182 9 : static int ZPack(const buf_mgr &src, buf_mgr &dst, int flags)
183 : {
184 : z_stream stream;
185 : int err;
186 :
187 9 : memset(&stream, 0, sizeof(stream));
188 9 : stream.next_in = (Bytef *)src.buffer;
189 9 : stream.avail_in = (uInt)src.size;
190 9 : stream.next_out = (Bytef *)dst.buffer;
191 9 : stream.avail_out = (uInt)dst.size;
192 :
193 9 : int level = flags & ZFLAG_LMASK;
194 9 : if (level > 9)
195 0 : level = 9;
196 9 : if (level < 1)
197 0 : level = 1;
198 9 : int wb = MAX_WBITS;
199 : // if gz flag is set, ignore raw request
200 9 : if (flags & ZFLAG_GZ)
201 0 : wb += 16;
202 9 : else if (flags & ZFLAG_RAW)
203 0 : wb = -wb;
204 9 : int memlevel = 8; // Good compromise
205 9 : int strategy = (flags & ZFLAG_SMASK) >> 6;
206 9 : if (strategy > 4)
207 0 : strategy = 0;
208 :
209 9 : err = deflateInit2(&stream, level, Z_DEFLATED, wb, memlevel, strategy);
210 9 : if (err != Z_OK)
211 : {
212 0 : deflateEnd(&stream);
213 0 : return err;
214 : }
215 :
216 9 : err = deflate(&stream, Z_FINISH);
217 9 : if (err != Z_STREAM_END)
218 : {
219 0 : deflateEnd(&stream);
220 0 : return false;
221 : }
222 9 : dst.size = stream.total_out;
223 9 : err = deflateEnd(&stream);
224 9 : return err == Z_OK;
225 : }
226 :
227 : // Similar to uncompress() from zlib, accepts the ZFLAG_RAW
228 : // Return true if it worked
229 9 : static int ZUnPack(const buf_mgr &src, buf_mgr &dst, int flags)
230 : {
231 :
232 : z_stream stream;
233 : int err;
234 :
235 9 : memset(&stream, 0, sizeof(stream));
236 9 : stream.next_in = (Bytef *)src.buffer;
237 9 : stream.avail_in = (uInt)src.size;
238 9 : stream.next_out = (Bytef *)dst.buffer;
239 9 : stream.avail_out = (uInt)dst.size;
240 :
241 : // 32 means autodetec gzip or zlib header, negative 15 is for raw
242 9 : int wb = (ZFLAG_RAW & flags) ? -MAX_WBITS : 32 + MAX_WBITS;
243 9 : err = inflateInit2(&stream, wb);
244 9 : if (err != Z_OK)
245 0 : return false;
246 :
247 9 : err = inflate(&stream, Z_FINISH);
248 9 : if (err != Z_STREAM_END)
249 : {
250 0 : inflateEnd(&stream);
251 0 : return false;
252 : }
253 9 : dst.size = stream.total_out;
254 9 : err = inflateEnd(&stream);
255 9 : return err == Z_OK;
256 : }
257 :
258 : /*
259 : * Deflates a buffer, extrasize is the available size in the buffer past the
260 : * input If the output fits past the data, it uses that area, otherwise it uses
261 : * a temporary buffer and copies the data over the input on return, returning a
262 : * pointer to it. The output size is returned in src.size Returns nullptr when
263 : * compression failed
264 : */
265 9 : static void *DeflateBlock(buf_mgr &src, size_t extrasize, int flags)
266 : {
267 : // The one we might need to allocate
268 9 : void *dbuff = nullptr;
269 9 : buf_mgr dst = {src.buffer + src.size, extrasize};
270 :
271 : // Allocate a temp buffer if there is not sufficient space,
272 : // We need to have a bit more than half the buffer available
273 9 : if (extrasize < (src.size + 64))
274 : {
275 9 : dst.size = src.size + 64;
276 9 : dbuff = VSIMalloc(dst.size);
277 9 : dst.buffer = (char *)dbuff;
278 9 : if (!dst.buffer)
279 0 : return nullptr;
280 : }
281 :
282 9 : if (!ZPack(src, dst, flags))
283 : {
284 0 : CPLFree(dbuff); // Safe to call with NULL
285 0 : return nullptr;
286 : }
287 9 : if (dst.size > src.size)
288 : {
289 0 : CPLError(CE_Failure, CPLE_AppDefined,
290 : "DeflateBlock(): dst.size > src.size");
291 0 : CPLFree(dbuff); // Safe to call with NULL
292 0 : return nullptr;
293 : }
294 :
295 : // source size is used to hold the output size
296 9 : src.size = dst.size;
297 : // If we didn't allocate a buffer, the receiver can use it already
298 9 : if (!dbuff)
299 0 : return dst.buffer;
300 :
301 : // If we allocated a buffer, we need to copy the data to the input buffer.
302 9 : memcpy(src.buffer, dbuff, src.size);
303 9 : CPLFree(dbuff);
304 9 : return src.buffer;
305 : }
306 :
307 : #if defined(ZSTD_SUPPORT)
308 :
309 12 : CPL_NOSANITIZE_UNSIGNED_INT_OVERFLOW static void rankfilter(buf_mgr &src,
310 : size_t factor)
311 : {
312 : // Arange bytes by rank
313 12 : if (factor > 1)
314 : {
315 8 : std::vector<char> tempb(src.size);
316 8 : char *d = tempb.data();
317 43 : for (size_t j = 0; j < factor; j++)
318 9175080 : for (size_t i = j; i < src.size; i += factor)
319 9175040 : *d++ = src.buffer[i];
320 8 : memcpy(src.buffer, tempb.data(), src.size);
321 : }
322 : // byte delta
323 12 : auto p = reinterpret_cast<GByte *>(src.buffer);
324 12 : auto guard = p + src.size;
325 12 : GByte b(0);
326 10223600 : while (p < guard)
327 : {
328 10223600 : GByte temp = *p;
329 10223600 : *p -= b;
330 10223600 : b = temp;
331 10223600 : p++;
332 : }
333 12 : }
334 :
335 10 : CPL_NOSANITIZE_UNSIGNED_INT_OVERFLOW static void derank(buf_mgr &src,
336 : size_t factor)
337 : {
338 : // undo delta
339 10 : auto p = reinterpret_cast<GByte *>(src.buffer);
340 10 : auto guard = p + src.size;
341 10 : GByte b(0);
342 9699340 : while (p < guard)
343 : {
344 9699330 : b += *p;
345 9699330 : *p = b;
346 9699330 : p++;
347 : }
348 10 : if (factor > 1)
349 : { // undo rank separation
350 8 : std::vector<char> tempb(src.size);
351 8 : char *d = tempb.data();
352 8 : size_t chunk = src.size / factor;
353 2097160 : for (size_t i = 0; i < chunk; i++)
354 11272200 : for (size_t j = 0; j < factor; j++)
355 9175040 : *d++ = src.buffer[chunk * j + i];
356 8 : memcpy(src.buffer, tempb.data(), src.size);
357 : }
358 10 : }
359 :
360 : /*
361 : * Compress a buffer using zstd, extrasize is the available size in the buffer
362 : * past the input If ranks > 0, apply the rank filter If the output fits past
363 : * the data, it uses that area, otherwise it uses a temporary buffer and copies
364 : * the data over the input on return, returning a pointer to it. The output size
365 : * is returned in src.size Returns nullptr when compression failed
366 : */
367 12 : static void *ZstdCompBlock(buf_mgr &src, size_t extrasize, int c_level,
368 : ZSTD_CCtx *cctx, size_t ranks)
369 : {
370 12 : if (!cctx)
371 0 : return nullptr;
372 12 : if (ranks && (src.size % ranks) == 0)
373 12 : rankfilter(src, ranks);
374 :
375 : // might need a buffer for the zstd output
376 24 : std::vector<char> dbuff;
377 12 : void *dst = src.buffer + src.size;
378 12 : size_t size = extrasize;
379 : // Allocate a temp buffer if there is not sufficient space.
380 : // Zstd bound is about (size * 1.004 + 64)
381 12 : if (size < ZSTD_compressBound(src.size))
382 : {
383 12 : size = ZSTD_compressBound(src.size);
384 12 : dbuff.resize(size);
385 12 : dst = dbuff.data();
386 : }
387 :
388 : // Use the streaming interface, it's faster and better
389 : // See discussion at https://github.com/facebook/zstd/issues/3729
390 12 : ZSTD_outBuffer output = {dst, size, 0};
391 12 : ZSTD_inBuffer input = {src.buffer, src.size, 0};
392 : // Set level
393 12 : ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, c_level);
394 : // First, pass a continue flag, otherwise it will compress in one go
395 12 : size_t val = ZSTD_compressStream2(cctx, &output, &input, ZSTD_e_continue);
396 : // If it worked, pass the end flag to flush the buffer
397 12 : if (val == 0)
398 12 : val = ZSTD_compressStream2(cctx, &output, &input, ZSTD_e_end);
399 12 : if (ZSTD_isError(val))
400 0 : return nullptr;
401 12 : val = output.pos;
402 :
403 : // If we didn't need the buffer, packed data is already in the user buffer
404 12 : if (dbuff.empty())
405 : {
406 0 : src.size = val;
407 0 : return dst;
408 : }
409 :
410 12 : if (val > (src.size + extrasize))
411 : { // Doesn't fit in user buffer
412 0 : CPLError(CE_Failure, CPLE_AssertionFailed,
413 : "MRF: ZSTD compression buffer too small");
414 0 : return nullptr; // Error
415 : }
416 :
417 12 : memcpy(src.buffer, dbuff.data(), val);
418 12 : src.size = val;
419 12 : return src.buffer;
420 : }
421 : #endif
422 :
423 : //
424 : // The deflate_flags are available in all bands even if the DEFLATE option
425 : // itself is not set. This allows for PNG features to be controlled, as well
426 : // as any other bands that use zlib by itself
427 : //
428 543 : MRFRasterBand::MRFRasterBand(MRFDataset *parent_dataset, const ILImage &image,
429 543 : int band, int ov)
430 : : poMRFDS(parent_dataset),
431 543 : dodeflate(GetOptlist().FetchBoolean("DEFLATE", FALSE)),
432 : // Bring the quality to 0 to 9
433 543 : deflate_flags(image.quality / 10),
434 543 : dozstd(GetOptlist().FetchBoolean("ZSTD", FALSE)), zstd_level(9), m_l(ov),
435 1629 : img(image)
436 : {
437 543 : nBand = band;
438 543 : eDataType = parent_dataset->current.dt;
439 543 : nRasterXSize = img.size.x;
440 543 : nRasterYSize = img.size.y;
441 543 : nBlockXSize = img.pagesize.x;
442 543 : nBlockYSize = img.pagesize.y;
443 543 : nBlocksPerRow = img.pagecount.x;
444 543 : nBlocksPerColumn = img.pagecount.y;
445 543 : img.NoDataValue = MRFRasterBand::GetNoDataValue(&img.hasNoData);
446 :
447 : // Pick up the twists, aka GZ, RAWZ headers
448 543 : if (GetOptlist().FetchBoolean("GZ", FALSE))
449 0 : deflate_flags |= ZFLAG_GZ;
450 543 : else if (GetOptlist().FetchBoolean("RAWZ", FALSE))
451 0 : deflate_flags |= ZFLAG_RAW;
452 : // And Pick up the ZLIB strategy, if any
453 543 : const char *zstrategy = GetOptlist().FetchNameValueDef("Z_STRATEGY", "");
454 543 : int zv = Z_DEFAULT_STRATEGY;
455 543 : if (EQUAL(zstrategy, "Z_HUFFMAN_ONLY"))
456 0 : zv = Z_HUFFMAN_ONLY;
457 543 : else if (EQUAL(zstrategy, "Z_RLE"))
458 0 : zv = Z_RLE;
459 543 : else if (EQUAL(zstrategy, "Z_FILTERED"))
460 0 : zv = Z_FILTERED;
461 543 : else if (EQUAL(zstrategy, "Z_FIXED"))
462 0 : zv = Z_FIXED;
463 543 : deflate_flags |= (zv << 6);
464 543 : if (image.quality < 23 && image.quality > 0)
465 0 : zstd_level = image.quality;
466 :
467 : #if !defined(ZSTD_SUPPORT)
468 : if (dozstd)
469 : { // signal error condition to caller
470 : CPLError(CE_Failure, CPLE_AssertionFailed,
471 : "MRF: ZSTD support is not available");
472 : dozstd = FALSE;
473 : }
474 : #endif
475 : // Chose zstd over deflate if both are enabled and available
476 543 : if (dozstd && dodeflate)
477 0 : dodeflate = FALSE;
478 543 : }
479 :
480 : // Clean up the overviews if they exist
481 1086 : MRFRasterBand::~MRFRasterBand()
482 : {
483 615 : while (!overviews.empty())
484 : {
485 72 : delete overviews.back();
486 72 : overviews.pop_back();
487 : }
488 543 : }
489 :
490 : // Look for a string from the dataset options or from the environment
491 75 : const char *MRFRasterBand::GetOptionValue(const char *opt,
492 : const char *def) const
493 : {
494 75 : const char *optValue = poMRFDS->optlist.FetchNameValue(opt);
495 75 : if (optValue)
496 8 : return optValue;
497 67 : return CPLGetConfigOption(opt, def);
498 : }
499 :
500 : // Utility function, returns a value from a vector corresponding to the band
501 : // index or the first entry
502 157 : static double getBandValue(const std::vector<double> &v, int idx)
503 : {
504 157 : return (static_cast<int>(v.size()) > idx) ? v[idx] : v[0];
505 : }
506 :
507 : // Maybe we should check against the type range?
508 : // It is not keeping track of how many values have been set,
509 : // so the application should set none or all the bands
510 : // This call is only valid during Create
511 16 : CPLErr MRFRasterBand::SetNoDataValue(double val)
512 : {
513 16 : if (poMRFDS->bCrystalized)
514 : {
515 0 : CPLError(CE_Failure, CPLE_AssertionFailed,
516 : "MRF: NoData can be set only during file create");
517 0 : return CE_Failure;
518 : }
519 16 : if (GInt32(poMRFDS->vNoData.size()) < nBand)
520 0 : poMRFDS->vNoData.resize(nBand);
521 16 : poMRFDS->vNoData[nBand - 1] = val;
522 : // We also need to set it for this band
523 16 : img.NoDataValue = val;
524 16 : img.hasNoData = true;
525 16 : return CE_None;
526 : }
527 :
528 5999 : double MRFRasterBand::GetNoDataValue(int *pbSuccess)
529 : {
530 5999 : const std::vector<double> &v = poMRFDS->vNoData;
531 5999 : if (v.empty())
532 5842 : return GDALPamRasterBand::GetNoDataValue(pbSuccess);
533 157 : if (pbSuccess)
534 155 : *pbSuccess = TRUE;
535 157 : return getBandValue(v, nBand - 1);
536 : }
537 :
538 0 : double MRFRasterBand::GetMinimum(int *pbSuccess)
539 : {
540 0 : const std::vector<double> &v = poMRFDS->vMin;
541 0 : if (v.empty())
542 0 : return GDALPamRasterBand::GetMinimum(pbSuccess);
543 0 : if (pbSuccess)
544 0 : *pbSuccess = TRUE;
545 0 : return getBandValue(v, nBand - 1);
546 : }
547 :
548 0 : double MRFRasterBand::GetMaximum(int *pbSuccess)
549 : {
550 0 : const std::vector<double> &v = poMRFDS->vMax;
551 0 : if (v.empty())
552 0 : return GDALPamRasterBand::GetMaximum(pbSuccess);
553 0 : if (pbSuccess)
554 0 : *pbSuccess = TRUE;
555 0 : return getBandValue(v, nBand - 1);
556 : }
557 :
558 : // Fill with typed ndv, count is always in bytes
559 : template <typename T>
560 0 : static CPLErr buff_fill(void *b, size_t count, const T ndv)
561 : {
562 0 : T *buffer = static_cast<T *>(b);
563 0 : count /= sizeof(T);
564 0 : while (count--)
565 0 : *buffer++ = ndv;
566 0 : return CE_None;
567 : }
568 :
569 : /**
570 : *\brief Fills a buffer with no data
571 : *
572 : */
573 56 : CPLErr MRFRasterBand::FillBlock(void *buffer)
574 : {
575 : int success;
576 56 : double ndv = GetNoDataValue(&success);
577 56 : if (!success)
578 56 : ndv = 0.0;
579 56 : size_t bsb = blockSizeBytes();
580 :
581 : // use memset for speed for bytes, or if nodata is zeros
582 56 : if (0.0 == ndv || eDataType == GDT_Byte || eDataType == GDT_Int8)
583 : {
584 56 : memset(buffer, int(ndv), bsb);
585 56 : return CE_None;
586 : }
587 :
588 : #define bf(T) buff_fill<T>(buffer, bsb, T(ndv));
589 0 : switch (eDataType)
590 : {
591 0 : case GDT_UInt16:
592 0 : return bf(GUInt16);
593 0 : case GDT_Int16:
594 0 : return bf(GInt16);
595 0 : case GDT_UInt32:
596 0 : return bf(GUInt32);
597 0 : case GDT_Int32:
598 0 : return bf(GInt32);
599 0 : case GDT_UInt64:
600 0 : return bf(GUInt64);
601 0 : case GDT_Int64:
602 0 : return bf(GInt64);
603 0 : case GDT_Float32:
604 0 : return bf(float);
605 0 : case GDT_Float64:
606 0 : return bf(double);
607 0 : default:
608 0 : break;
609 : }
610 : #undef bf
611 : // Should exit before
612 0 : return CE_Failure;
613 : }
614 :
615 : /*\brief Interleave block fill
616 : *
617 : * Acquire space for all the other bands, fill each one then drop the locks
618 : * The current band output goes directly into the buffer
619 : */
620 :
621 0 : CPLErr MRFRasterBand::FillBlock(int xblk, int yblk, void *buffer)
622 : {
623 0 : std::vector<GDALRasterBlock *> blocks;
624 :
625 0 : for (int i = 0; i < poMRFDS->nBands; i++)
626 : {
627 0 : GDALRasterBand *b = poMRFDS->GetRasterBand(i + 1);
628 0 : if (b->GetOverviewCount() && 0 != m_l)
629 0 : b = b->GetOverview(m_l - 1);
630 :
631 : // Get the other band blocks, keep them around until later
632 0 : if (b == this)
633 : {
634 0 : FillBlock(buffer);
635 : }
636 : else
637 : {
638 0 : GDALRasterBlock *poBlock = b->GetLockedBlockRef(xblk, yblk, 1);
639 0 : if (poBlock == nullptr) // Didn't get this block
640 0 : break;
641 0 : FillBlock(poBlock->GetDataRef());
642 0 : blocks.push_back(poBlock);
643 : }
644 : }
645 :
646 : // Drop the locks for blocks we acquired
647 0 : for (int i = 0; i < int(blocks.size()); i++)
648 0 : blocks[i]->DropLock();
649 :
650 0 : return CE_None;
651 : }
652 :
653 : /*\brief Interleave block read
654 : *
655 : * Acquire space for all the other bands, unpack from the dataset buffer, then
656 : * drop the locks The current band output goes directly into the buffer
657 : */
658 :
659 8 : CPLErr MRFRasterBand::ReadInterleavedBlock(int xblk, int yblk, void *buffer)
660 : {
661 8 : std::vector<GDALRasterBlock *> blocks;
662 :
663 32 : for (int i = 0; i < poMRFDS->nBands; i++)
664 : {
665 24 : GDALRasterBand *b = poMRFDS->GetRasterBand(i + 1);
666 24 : if (b->GetOverviewCount() && 0 != m_l)
667 0 : b = b->GetOverview(m_l - 1);
668 :
669 24 : void *ob = buffer;
670 : // Get the other band blocks, keep them around until later
671 24 : if (b != this)
672 : {
673 16 : GDALRasterBlock *poBlock = b->GetLockedBlockRef(xblk, yblk, 1);
674 16 : if (poBlock == nullptr)
675 0 : break;
676 16 : ob = poBlock->GetDataRef();
677 16 : blocks.push_back(poBlock);
678 : }
679 :
680 : // Just the right mix of templates and macros make deinterleaving tidy
681 24 : void *pbuffer = poMRFDS->GetPBuffer();
682 : #define CpySI(T) \
683 : cpy_stride_in<T>(ob, reinterpret_cast<T *>(pbuffer) + i, \
684 : blockSizeBytes() / sizeof(T), img.pagesize.c)
685 :
686 : // Page is already in poMRFDS->pbuffer, not empty
687 : // There are only four cases, since only the data size matters
688 24 : switch (GDALGetDataTypeSize(eDataType) / 8)
689 : {
690 24 : case 1:
691 24 : CpySI(GByte);
692 24 : break;
693 0 : case 2:
694 0 : CpySI(GInt16);
695 0 : break;
696 0 : case 4:
697 0 : CpySI(GInt32);
698 0 : break;
699 0 : case 8:
700 0 : CpySI(GIntBig);
701 0 : break;
702 : }
703 : }
704 :
705 : #undef CpySI
706 :
707 : // Drop the locks we acquired
708 24 : for (int i = 0; i < int(blocks.size()); i++)
709 16 : blocks[i]->DropLock();
710 :
711 16 : return CE_None;
712 : }
713 :
714 : /**
715 : *\brief Fetch a block from the backing store dataset and keep a copy in the
716 : *cache
717 : *
718 : * @param xblk The X block number, zero based
719 : * @param yblk The Y block number, zero based
720 : * @param buffer buffer
721 : *
722 : */
723 4 : CPLErr MRFRasterBand::FetchBlock(int xblk, int yblk, void *buffer)
724 : {
725 4 : assert(!poMRFDS->source.empty());
726 4 : CPLDebug("MRF_IB", "FetchBlock %d,%d,0,%d, level %d\n", xblk, yblk, nBand,
727 : m_l);
728 :
729 4 : if (poMRFDS->clonedSource) // This is a clone
730 1 : return FetchClonedBlock(xblk, yblk, buffer);
731 :
732 3 : const GInt32 cstride = img.pagesize.c; // 1 if band separate
733 3 : ILSize req(xblk, yblk, 0, (nBand - 1) / cstride, m_l);
734 3 : GUIntBig infooffset = IdxOffset(req, img);
735 :
736 3 : GDALDataset *poSrcDS = nullptr;
737 3 : if (nullptr == (poSrcDS = poMRFDS->GetSrcDS()))
738 : {
739 1 : CPLError(CE_Failure, CPLE_AppDefined, "MRF: Can't open source file %s",
740 1 : poMRFDS->source.c_str());
741 1 : return CE_Failure;
742 : }
743 :
744 : // Scale to base resolution
745 2 : double scl = pow(poMRFDS->scale, m_l);
746 2 : if (0 == m_l)
747 2 : scl = 1; // To allow for precision issues
748 :
749 : // Prepare parameters for RasterIO, they might be different from a full page
750 2 : const GSpacing vsz = GDALGetDataTypeSizeBytes(eDataType);
751 2 : int Xoff = int(xblk * img.pagesize.x * scl + 0.5);
752 2 : int Yoff = int(yblk * img.pagesize.y * scl + 0.5);
753 2 : int readszx = int(img.pagesize.x * scl + 0.5);
754 2 : int readszy = int(img.pagesize.y * scl + 0.5);
755 :
756 : // Compare with the full size and clip to the right and bottom if needed
757 2 : int clip = 0;
758 2 : if (Xoff + readszx > poMRFDS->full.size.x)
759 : {
760 2 : clip |= 1;
761 2 : readszx = poMRFDS->full.size.x - Xoff;
762 : }
763 2 : if (Yoff + readszy > poMRFDS->full.size.y)
764 : {
765 2 : clip |= 1;
766 2 : readszy = poMRFDS->full.size.y - Yoff;
767 : }
768 :
769 : // This is where the whole page fits
770 2 : void *ob = buffer;
771 2 : if (cstride != 1)
772 0 : ob = poMRFDS->GetPBuffer();
773 :
774 : // Fill buffer with NoData if clipping
775 2 : if (clip)
776 2 : FillBlock(ob);
777 :
778 : // Use the dataset RasterIO to read one or all bands if interleaved
779 4 : CPLErr ret = poSrcDS->RasterIO(
780 : GF_Read, Xoff, Yoff, readszx, readszy, ob, pcount(readszx, int(scl)),
781 : pcount(readszy, int(scl)), eDataType, cstride,
782 2 : (1 == cstride) ? &nBand : nullptr, vsz * cstride,
783 2 : vsz * cstride * img.pagesize.x,
784 2 : (cstride != 1) ? vsz : vsz * img.pagesize.x * img.pagesize.y, nullptr);
785 :
786 2 : if (ret != CE_None)
787 0 : return ret;
788 :
789 : // Might have the block in the pbuffer, mark it anyhow
790 2 : poMRFDS->tile = req;
791 : buf_mgr filesrc;
792 2 : filesrc.buffer = (char *)ob;
793 2 : filesrc.size = static_cast<size_t>(img.pageSizeBytes);
794 :
795 2 : if (poMRFDS->bypass_cache)
796 : { // No local caching, just return the data
797 0 : if (1 == cstride)
798 0 : return CE_None;
799 0 : return ReadInterleavedBlock(xblk, yblk, buffer);
800 : }
801 :
802 : // Test to see if it needs to be written, or just marked as checked
803 : int success;
804 2 : double val = GetNoDataValue(&success);
805 2 : if (!success)
806 2 : val = 0.0;
807 :
808 : // TODO: test band by band if data is interleaved
809 2 : if (isAllVal(eDataType, ob, img.pageSizeBytes, val))
810 : {
811 : // Mark it empty and checked, ignore the possible write error
812 0 : poMRFDS->WriteTile((void *)1, infooffset, 0);
813 0 : if (1 == cstride)
814 0 : return CE_None;
815 0 : return ReadInterleavedBlock(xblk, yblk, buffer);
816 : }
817 :
818 : // Write the page in the local cache
819 :
820 : // Have to use a separate buffer for compression output.
821 2 : void *outbuff = VSIMalloc(poMRFDS->pbsize);
822 2 : if (nullptr == outbuff)
823 : {
824 0 : CPLError(CE_Failure, CPLE_AppDefined,
825 : "Can't get buffer for writing page");
826 : // This is not really an error for a cache, the data is fine
827 0 : return CE_Failure;
828 : }
829 :
830 2 : buf_mgr filedst = {static_cast<char *>(outbuff), poMRFDS->pbsize};
831 2 : auto start_time = steady_clock::now();
832 2 : Compress(filedst, filesrc);
833 :
834 : // Where the output is, in case we deflate
835 2 : void *usebuff = outbuff;
836 2 : if (dodeflate)
837 : {
838 0 : usebuff = DeflateBlock(filedst, poMRFDS->pbsize - filedst.size,
839 : deflate_flags);
840 0 : if (!usebuff)
841 : {
842 0 : CPLError(CE_Failure, CPLE_AppDefined, "MRF: Deflate error");
843 0 : return CE_Failure;
844 : }
845 : }
846 :
847 : #if defined(ZSTD_SUPPORT)
848 2 : else if (dozstd)
849 : {
850 0 : size_t ranks = 0; // Assume no need for byte rank sort
851 0 : if (img.comp == IL_NONE || img.comp == IL_ZSTD)
852 0 : ranks =
853 0 : static_cast<size_t>(GDALGetDataTypeSizeBytes(img.dt)) * cstride;
854 0 : usebuff = ZstdCompBlock(filedst, poMRFDS->pbsize - filedst.size,
855 0 : zstd_level, poMRFDS->getzsc(), ranks);
856 0 : if (!usebuff)
857 : {
858 0 : CPLError(CE_Failure, CPLE_AppDefined,
859 : "MRF: ZSTD compression error");
860 0 : return CE_Failure;
861 : }
862 : }
863 : #endif
864 :
865 2 : poMRFDS->write_timer +=
866 2 : duration_cast<nanoseconds>(steady_clock::now() - start_time);
867 :
868 : // Write and update the tile index
869 2 : ret = poMRFDS->WriteTile(usebuff, infooffset, filedst.size);
870 2 : CPLFree(outbuff);
871 :
872 : // If we hit an error or if unpaking is not needed
873 2 : if (ret != CE_None || cstride == 1)
874 2 : return ret;
875 :
876 : // data is already in DS buffer, deinterlace it in pixel blocks
877 0 : return ReadInterleavedBlock(xblk, yblk, buffer);
878 : }
879 :
880 : /**
881 : *\brief Fetch for a cloned MRF
882 : *
883 : * @param xblk The X block number, zero based
884 : * @param yblk The Y block number, zero based
885 : * @param buffer buffer
886 : *
887 : */
888 :
889 1 : CPLErr MRFRasterBand::FetchClonedBlock(int xblk, int yblk, void *buffer)
890 : {
891 1 : CPLDebug("MRF_IB", "FetchClonedBlock %d,%d,0,%d, level %d\n", xblk, yblk,
892 : nBand, m_l);
893 :
894 : // Paranoid check
895 1 : assert(poMRFDS->clonedSource);
896 1 : MRFDataset *poSrc = static_cast<MRFDataset *>(poMRFDS->GetSrcDS());
897 1 : if (nullptr == poSrc)
898 : {
899 0 : CPLError(CE_Failure, CPLE_AppDefined, "MRF: Can't open source file %s",
900 0 : poMRFDS->source.c_str());
901 0 : return CE_Failure;
902 : }
903 :
904 1 : if (poMRFDS->bypass_cache || GF_Read == DataMode())
905 : {
906 : // Can't store, so just fetch from source, which is an MRF with
907 : // identical structure
908 : MRFRasterBand *b =
909 0 : static_cast<MRFRasterBand *>(poSrc->GetRasterBand(nBand));
910 0 : if (b->GetOverviewCount() && m_l)
911 0 : b = static_cast<MRFRasterBand *>(b->GetOverview(m_l - 1));
912 0 : if (b == nullptr)
913 0 : return CE_Failure;
914 0 : return b->IReadBlock(xblk, yblk, buffer);
915 : }
916 :
917 1 : ILSize req(xblk, yblk, 0, (nBand - 1) / img.pagesize.c, m_l);
918 : ILIdx tinfo;
919 :
920 : // Get the cloned source tile info
921 : // The cloned source index is after the current one
922 1 : if (CE_None != poMRFDS->ReadTileIdx(tinfo, req, img, poMRFDS->idxSize))
923 : {
924 0 : CPLError(CE_Failure, CPLE_AppDefined,
925 : "MRF: Unable to read cloned index entry");
926 0 : return CE_Failure;
927 : }
928 :
929 1 : GUIntBig infooffset = IdxOffset(req, img);
930 : CPLErr err;
931 :
932 : // Does the source have this tile?
933 1 : if (tinfo.size == 0)
934 : { // Nope, mark it empty and return fill
935 0 : err = poMRFDS->WriteTile((void *)1, infooffset, 0);
936 0 : if (CE_None != err)
937 0 : return err;
938 0 : return FillBlock(buffer);
939 : }
940 :
941 1 : VSILFILE *srcfd = poSrc->DataFP();
942 1 : if (nullptr == srcfd)
943 : {
944 0 : CPLError(CE_Failure, CPLE_AppDefined,
945 : "MRF: Can't open source data file %s",
946 0 : poMRFDS->source.c_str());
947 0 : return CE_Failure;
948 : }
949 :
950 : // Need to read the tile from the source
951 1 : if (tinfo.size <= 0 || tinfo.size > INT_MAX)
952 : {
953 0 : CPLError(CE_Failure, CPLE_OutOfMemory,
954 : "Invalid tile size " CPL_FRMT_GIB, tinfo.size);
955 0 : return CE_Failure;
956 : }
957 1 : char *buf = static_cast<char *>(VSIMalloc(static_cast<size_t>(tinfo.size)));
958 1 : if (buf == nullptr)
959 : {
960 0 : CPLError(CE_Failure, CPLE_OutOfMemory,
961 : "Cannot allocate " CPL_FRMT_GIB " bytes", tinfo.size);
962 0 : return CE_Failure;
963 : }
964 :
965 1 : VSIFSeekL(srcfd, tinfo.offset, SEEK_SET);
966 2 : if (tinfo.size !=
967 1 : GIntBig(VSIFReadL(buf, 1, static_cast<size_t>(tinfo.size), srcfd)))
968 : {
969 0 : CPLFree(buf);
970 0 : CPLError(CE_Failure, CPLE_AppDefined,
971 : "MRF: Can't read data from source %s",
972 : poSrc->current.datfname.c_str());
973 0 : return CE_Failure;
974 : }
975 :
976 : // Write it then reissue the read
977 1 : err = poMRFDS->WriteTile(buf, infooffset, tinfo.size);
978 1 : CPLFree(buf);
979 1 : if (CE_None != err)
980 0 : return err;
981 : // Reissue read, it will work from the cloned data
982 1 : return IReadBlock(xblk, yblk, buffer);
983 : }
984 :
985 : /**
986 : *\brief read a block in the provided buffer
987 : *
988 : * For separate band model, the DS buffer is not used, the read is direct in
989 : *the buffer For pixel interleaved model, the DS buffer holds the temp copy and
990 : *all the other bands are force read
991 : *
992 : */
993 :
994 1882 : CPLErr MRFRasterBand::IReadBlock(int xblk, int yblk, void *buffer)
995 : {
996 1882 : GInt32 cstride = img.pagesize.c;
997 : ILIdx tinfo;
998 1882 : ILSize req(xblk, yblk, 0, (nBand - 1) / cstride, m_l);
999 3764 : CPLDebug("MRF_IB",
1000 : "IReadBlock %d,%d,0,%d, level %d, idxoffset " CPL_FRMT_GIB "\n",
1001 1882 : xblk, yblk, nBand - 1, m_l, IdxOffset(req, img));
1002 :
1003 : // If this is a caching file and bypass is on, just do the fetch
1004 1882 : if (poMRFDS->bypass_cache && !poMRFDS->source.empty())
1005 0 : return FetchBlock(xblk, yblk, buffer);
1006 :
1007 1882 : tinfo.size = 0; // Just in case it is missing
1008 1882 : if (CE_None != poMRFDS->ReadTileIdx(tinfo, req, img))
1009 : {
1010 0 : if (!poMRFDS->no_errors)
1011 : {
1012 0 : CPLError(CE_Failure, CPLE_AppDefined,
1013 : "MRF: Unable to read index at offset " CPL_FRMT_GIB,
1014 0 : IdxOffset(req, img));
1015 0 : return CE_Failure;
1016 : }
1017 0 : return FillBlock(buffer);
1018 : }
1019 :
1020 1882 : if (0 == tinfo.size)
1021 : { // Could be missing or it could be caching
1022 : // Offset != 0 means no data, Update mode is for local MRFs only
1023 : // if caching index mode is RO don't try to fetch
1024 : // Also, caching MRFs can't be opened in update mode
1025 12 : if (0 != tinfo.offset || GA_Update == poMRFDS->eAccess ||
1026 18 : poMRFDS->source.empty() || IdxMode() == GF_Read)
1027 2 : return FillBlock(buffer);
1028 :
1029 : // caching MRF, need to fetch a block
1030 4 : return FetchBlock(xblk, yblk, buffer);
1031 : }
1032 :
1033 1876 : CPLDebug("MRF_IB", "Tinfo offset " CPL_FRMT_GIB ", size " CPL_FRMT_GIB "\n",
1034 : tinfo.offset, tinfo.size);
1035 : // If we have a tile, read it
1036 :
1037 : // Should use a permanent buffer, like the pbuffer mechanism
1038 : // Get a large buffer, in case we need to unzip
1039 :
1040 : // We add a padding of 3 bytes since in LERC1 decompression, we can
1041 : // dereference a unsigned int at the end of the buffer, that can be
1042 : // partially out of the buffer.
1043 : // Can be reproduced with :
1044 : // gdal_translate ../autotest/gcore/data/byte.tif out.mrf -of MRF -co
1045 : // COMPRESS=LERC -co OPTIONS=V1:YES -ot Float32 valgrind gdalinfo -checksum
1046 : // out.mrf Invalid read of size 4 at BitStuffer::read(unsigned char**,
1047 : // std::vector<unsigned int, std::allocator<unsigned int> >&) const
1048 : // (BitStuffer.cpp:153)
1049 :
1050 : // No stored tile should be larger than twice the raw size.
1051 1876 : if (tinfo.size <= 0 || tinfo.size > poMRFDS->pbsize * 2)
1052 : {
1053 0 : if (!poMRFDS->no_errors)
1054 : {
1055 0 : CPLError(CE_Failure, CPLE_OutOfMemory,
1056 : "Stored tile is too large: " CPL_FRMT_GIB, tinfo.size);
1057 0 : return CE_Failure;
1058 : }
1059 0 : return FillBlock(buffer);
1060 : }
1061 :
1062 1876 : VSILFILE *dfp = DataFP();
1063 :
1064 : // No data file to read from
1065 1876 : if (dfp == nullptr)
1066 0 : return CE_Failure;
1067 :
1068 1876 : void *data = VSIMalloc(static_cast<size_t>(tinfo.size + PADDING_BYTES));
1069 1876 : if (data == nullptr)
1070 : {
1071 0 : CPLError(CE_Failure, CPLE_OutOfMemory,
1072 : "Could not allocate memory for tile size: " CPL_FRMT_GIB,
1073 : tinfo.size);
1074 0 : return CE_Failure;
1075 : }
1076 :
1077 : // This part is not thread safe, but it is what GDAL expects
1078 1876 : VSIFSeekL(dfp, tinfo.offset, SEEK_SET);
1079 1876 : if (1 != VSIFReadL(data, static_cast<size_t>(tinfo.size), 1, dfp))
1080 : {
1081 0 : CPLFree(data);
1082 0 : if (poMRFDS->no_errors)
1083 0 : return FillBlock(buffer);
1084 0 : CPLError(CE_Failure, CPLE_AppDefined, "Unable to read data page, %d@%x",
1085 0 : static_cast<int>(tinfo.size), static_cast<int>(tinfo.offset));
1086 0 : return CE_Failure;
1087 : }
1088 :
1089 : /* initialize padding bytes */
1090 1876 : memset(((char *)data) + static_cast<size_t>(tinfo.size), 0, PADDING_BYTES);
1091 1876 : buf_mgr src = {(char *)data, static_cast<size_t>(tinfo.size)};
1092 : buf_mgr dst;
1093 :
1094 1876 : auto start_time = steady_clock::now();
1095 :
1096 : // We got the data, do we need to decompress it before decoding?
1097 1876 : if (dodeflate)
1098 : {
1099 9 : if (img.pageSizeBytes > INT_MAX - 1440)
1100 : {
1101 0 : CPLFree(data);
1102 0 : CPLError(CE_Failure, CPLE_AppDefined, "Page size is too big at %d",
1103 : img.pageSizeBytes);
1104 0 : return CE_Failure;
1105 : }
1106 9 : dst.size =
1107 9 : img.pageSizeBytes +
1108 : 1440; // in case the packed page is a bit larger than the raw one
1109 9 : dst.buffer = static_cast<char *>(VSIMalloc(dst.size));
1110 9 : if (nullptr == dst.buffer)
1111 : {
1112 0 : CPLFree(data);
1113 0 : CPLError(CE_Failure, CPLE_OutOfMemory, "Cannot allocate %d bytes",
1114 0 : static_cast<int>(dst.size));
1115 0 : return CE_Failure;
1116 : }
1117 :
1118 9 : if (ZUnPack(src, dst, deflate_flags))
1119 : { // Got it unpacked, update the pointers
1120 9 : CPLFree(data);
1121 9 : data = dst.buffer;
1122 9 : tinfo.size = dst.size;
1123 : }
1124 : else
1125 : { // assume the page was not gzipped, warn only
1126 0 : CPLFree(dst.buffer);
1127 0 : if (!poMRFDS->no_errors)
1128 0 : CPLError(CE_Warning, CPLE_AppDefined, "Can't inflate page!");
1129 : }
1130 : }
1131 :
1132 : #if defined(ZSTD_SUPPORT)
1133 : // undo ZSTD
1134 1867 : else if (dozstd)
1135 : {
1136 10 : auto ctx = poMRFDS->getzsd();
1137 10 : if (!ctx)
1138 : {
1139 0 : CPLFree(data);
1140 0 : CPLError(CE_Failure, CPLE_AppDefined, "Can't acquire ZSTD context");
1141 0 : return CE_Failure;
1142 : }
1143 10 : if (img.pageSizeBytes > INT_MAX - 1440)
1144 : {
1145 0 : CPLFree(data);
1146 0 : CPLError(CE_Failure, CPLE_AppDefined, "Page is too large at %d",
1147 : img.pageSizeBytes);
1148 0 : return CE_Failure;
1149 : }
1150 10 : dst.size =
1151 10 : img.pageSizeBytes +
1152 : 1440; // Allow for a slight increase from previous compressions
1153 10 : dst.buffer = static_cast<char *>(VSIMalloc(dst.size));
1154 10 : if (nullptr == dst.buffer)
1155 : {
1156 0 : CPLFree(data);
1157 0 : CPLError(CE_Failure, CPLE_OutOfMemory, "Cannot allocate %d bytes",
1158 0 : static_cast<int>(dst.size));
1159 0 : return CE_Failure;
1160 : }
1161 :
1162 20 : auto raw_size = ZSTD_decompressDCtx(ctx, dst.buffer, dst.size,
1163 10 : src.buffer, src.size);
1164 10 : if (ZSTD_isError(raw_size))
1165 : { // assume page was not packed, warn only
1166 0 : CPLFree(dst.buffer);
1167 0 : if (!poMRFDS->no_errors)
1168 0 : CPLError(CE_Warning, CPLE_AppDefined,
1169 : "Can't unpack ZSTD page!");
1170 : }
1171 : else
1172 : {
1173 10 : CPLFree(data); // The compressed data
1174 10 : data = dst.buffer;
1175 10 : tinfo.size = raw_size;
1176 : // Might need to undo the rank sort
1177 10 : size_t ranks = 0;
1178 10 : if (img.comp == IL_NONE || img.comp == IL_ZSTD)
1179 10 : ranks = static_cast<size_t>(GDALGetDataTypeSizeBytes(img.dt)) *
1180 10 : img.pagesize.c;
1181 10 : if (ranks)
1182 : {
1183 10 : src.buffer = static_cast<char *>(data);
1184 10 : src.size = static_cast<size_t>(tinfo.size);
1185 10 : derank(src, ranks);
1186 : }
1187 : }
1188 : }
1189 : #endif
1190 :
1191 1876 : src.buffer = static_cast<char *>(data);
1192 1876 : src.size = static_cast<size_t>(tinfo.size);
1193 :
1194 : // After unpacking, the size has to be pageSizeBytes
1195 : // If pages are interleaved, use the dataset page buffer instead
1196 1876 : dst.buffer = reinterpret_cast<char *>(
1197 8 : (1 == cstride) ? buffer : poMRFDS->GetPBuffer());
1198 1876 : dst.size = img.pageSizeBytes;
1199 :
1200 1876 : if (poMRFDS->no_errors)
1201 0 : CPLPushErrorHandler(CPLQuietErrorHandler);
1202 1876 : CPLErr ret = Decompress(dst, src);
1203 :
1204 1876 : poMRFDS->read_timer +=
1205 1876 : duration_cast<nanoseconds>(steady_clock::now() - start_time);
1206 :
1207 1876 : dst.size =
1208 1876 : img.pageSizeBytes; // In case the decompress failed, force it back
1209 :
1210 : // Swap whatever we decompressed if we need to
1211 1876 : if (is_Endianness_Dependent(img.dt, img.comp) && (img.nbo != NET_ORDER))
1212 0 : swab_buff(dst, img);
1213 :
1214 1876 : CPLFree(data);
1215 1876 : if (poMRFDS->no_errors)
1216 : {
1217 0 : CPLPopErrorHandler();
1218 0 : if (ret != CE_None) // Set each page buffer to the correct no data
1219 : // value, then proceed
1220 0 : return (1 == cstride) ? FillBlock(buffer)
1221 0 : : FillBlock(xblk, yblk, buffer);
1222 : }
1223 :
1224 : // If pages are separate or we had errors, we're done
1225 1876 : if (1 == cstride || CE_None != ret)
1226 1868 : return ret;
1227 :
1228 : // De-interleave page from dataset buffer and return
1229 8 : return ReadInterleavedBlock(xblk, yblk, buffer);
1230 : }
1231 :
1232 : /**
1233 : *\brief Write a block from the provided buffer
1234 : *
1235 : * Same trick as read, use a temporary tile buffer for pixel interleave
1236 : * For band separate, use a
1237 : * Write the block once it has all the bands, report
1238 : * if a new block is started before the old one was completed
1239 : *
1240 : */
1241 :
1242 5081 : CPLErr MRFRasterBand::IWriteBlock(int xblk, int yblk, void *buffer)
1243 : {
1244 5081 : GInt32 cstride = img.pagesize.c;
1245 5081 : ILSize req(xblk, yblk, 0, (nBand - 1) / cstride, m_l);
1246 5081 : GUIntBig infooffset = IdxOffset(req, img);
1247 :
1248 5081 : CPLDebug("MRF_IB", "IWriteBlock %d,%d,0,%d, level %d, stride %d\n", xblk,
1249 : yblk, nBand, m_l, cstride);
1250 :
1251 : // Finish the Create call
1252 5081 : if (!poMRFDS->bCrystalized && !poMRFDS->Crystalize())
1253 : {
1254 0 : CPLError(CE_Failure, CPLE_AppDefined, "MRF: Error creating files");
1255 0 : return CE_Failure;
1256 : }
1257 :
1258 5081 : if (1 == cstride)
1259 : { // Separate bands, we can write it as is
1260 : // Empty page skip
1261 : int success;
1262 5070 : double val = GetNoDataValue(&success);
1263 5070 : if (!success)
1264 4998 : val = 0.0;
1265 5070 : if (isAllVal(eDataType, buffer, img.pageSizeBytes, val))
1266 1 : return poMRFDS->WriteTile(nullptr, infooffset, 0);
1267 :
1268 : // Use the pbuffer to hold the compressed page before writing it
1269 5069 : poMRFDS->tile = ILSize(); // Mark it corrupt
1270 :
1271 : buf_mgr src;
1272 5069 : src.buffer = (char *)buffer;
1273 5069 : src.size = static_cast<size_t>(img.pageSizeBytes);
1274 5069 : buf_mgr dst = {(char *)poMRFDS->GetPBuffer(),
1275 5069 : poMRFDS->GetPBufferSize()};
1276 :
1277 : // Swab the source before encoding if we need to
1278 5069 : if (is_Endianness_Dependent(img.dt, img.comp) && (img.nbo != NET_ORDER))
1279 0 : swab_buff(src, img);
1280 :
1281 5069 : auto start_time = steady_clock::now();
1282 :
1283 : // Compress functions need to return the compressed size in
1284 : // the bytes in buffer field
1285 5069 : Compress(dst, src);
1286 5069 : void *usebuff = dst.buffer;
1287 5069 : if (dodeflate)
1288 : {
1289 : usebuff =
1290 9 : DeflateBlock(dst, poMRFDS->pbsize - dst.size, deflate_flags);
1291 9 : if (!usebuff)
1292 : {
1293 0 : CPLError(CE_Failure, CPLE_AppDefined, "MRF: Deflate error");
1294 0 : return CE_Failure;
1295 : }
1296 : }
1297 :
1298 : #if defined(ZSTD_SUPPORT)
1299 5060 : else if (dozstd)
1300 : {
1301 11 : size_t ranks = 0; // Assume no need for byte rank sort
1302 11 : if (img.comp == IL_NONE || img.comp == IL_ZSTD)
1303 11 : ranks = static_cast<size_t>(GDALGetDataTypeSizeBytes(img.dt));
1304 11 : usebuff = ZstdCompBlock(dst, poMRFDS->pbsize - dst.size, zstd_level,
1305 11 : poMRFDS->getzsc(), ranks);
1306 11 : if (!usebuff)
1307 : {
1308 0 : CPLError(CE_Failure, CPLE_AppDefined,
1309 : "MRF: Zstd Compression error");
1310 0 : return CE_Failure;
1311 : }
1312 : }
1313 : #endif
1314 5069 : poMRFDS->write_timer +=
1315 5069 : duration_cast<nanoseconds>(steady_clock::now() - start_time);
1316 5069 : return poMRFDS->WriteTile(usebuff, infooffset, dst.size);
1317 : }
1318 :
1319 : // Multiple bands per page, use a temporary to assemble the page
1320 : // Temporary is large because we use it to hold both the uncompressed and
1321 : // the compressed
1322 11 : poMRFDS->tile = req;
1323 11 : poMRFDS->bdirty = 0;
1324 :
1325 : // Keep track of what bands are empty
1326 11 : GUIntBig empties = 0;
1327 :
1328 11 : void *tbuffer = VSIMalloc(img.pageSizeBytes + poMRFDS->pbsize);
1329 :
1330 11 : if (!tbuffer)
1331 : {
1332 0 : CPLError(CE_Failure, CPLE_AppDefined,
1333 : "MRF: Can't allocate write buffer");
1334 0 : return CE_Failure;
1335 : }
1336 :
1337 : // Get the other bands from the block cache
1338 44 : for (int iBand = 0; iBand < poMRFDS->nBands; iBand++)
1339 : {
1340 33 : char *pabyThisImage = nullptr;
1341 33 : GDALRasterBlock *poBlock = nullptr;
1342 :
1343 33 : if (iBand == nBand - 1)
1344 : {
1345 11 : pabyThisImage = reinterpret_cast<char *>(buffer);
1346 11 : poMRFDS->bdirty |= bandbit();
1347 : }
1348 : else
1349 : {
1350 22 : GDALRasterBand *band = poMRFDS->GetRasterBand(iBand + 1);
1351 : // Pick the right overview
1352 22 : if (m_l)
1353 0 : band = band->GetOverview(m_l - 1);
1354 : poBlock = (reinterpret_cast<MRFRasterBand *>(band))
1355 22 : ->TryGetLockedBlockRef(xblk, yblk);
1356 22 : if (nullptr == poBlock)
1357 0 : continue;
1358 : // This is where the image data is for this band
1359 :
1360 22 : pabyThisImage = reinterpret_cast<char *>(poBlock->GetDataRef());
1361 22 : poMRFDS->bdirty |= bandbit(iBand);
1362 : }
1363 :
1364 : // Keep track of empty bands, but encode them anyhow, in case some are
1365 : // not empty
1366 : int success;
1367 33 : double val = GetNoDataValue(&success);
1368 33 : if (!success)
1369 33 : val = 0.0;
1370 33 : if (isAllVal(eDataType, pabyThisImage, blockSizeBytes(), val))
1371 6 : empties |= bandbit(iBand);
1372 :
1373 : // Copy the data into the dataset buffer here
1374 : // Just the right mix of templates and macros make this real tidy
1375 : #define CpySO(T) \
1376 : cpy_stride_out<T>((reinterpret_cast<T *>(tbuffer)) + iBand, pabyThisImage, \
1377 : blockSizeBytes() / sizeof(T), cstride)
1378 :
1379 : // Build the page in tbuffer
1380 33 : switch (GDALGetDataTypeSize(eDataType) / 8)
1381 : {
1382 33 : case 1:
1383 33 : CpySO(GByte);
1384 33 : break;
1385 0 : case 2:
1386 0 : CpySO(GInt16);
1387 0 : break;
1388 0 : case 4:
1389 0 : CpySO(GInt32);
1390 0 : break;
1391 0 : case 8:
1392 0 : CpySO(GIntBig);
1393 0 : break;
1394 0 : default:
1395 : {
1396 0 : CPLError(CE_Failure, CPLE_AppDefined,
1397 : "MRF: Write datatype of %d bytes "
1398 : "not implemented",
1399 0 : GDALGetDataTypeSize(eDataType) / 8);
1400 0 : if (poBlock != nullptr)
1401 : {
1402 0 : poBlock->MarkClean();
1403 0 : poBlock->DropLock();
1404 : }
1405 0 : CPLFree(tbuffer);
1406 0 : return CE_Failure;
1407 : }
1408 : }
1409 :
1410 33 : if (poBlock != nullptr)
1411 : {
1412 22 : poBlock->MarkClean();
1413 22 : poBlock->DropLock();
1414 : }
1415 : }
1416 :
1417 : // Should keep track of the individual band buffers and only mix them if
1418 : // this is not an empty page ( move the Copy with Stride Out from above
1419 : // below this test This way works fine, but it does work extra for empty
1420 : // pages
1421 :
1422 11 : if (GIntBig(empties) == AllBandMask())
1423 : {
1424 0 : CPLFree(tbuffer);
1425 0 : return poMRFDS->WriteTile(nullptr, infooffset, 0);
1426 : }
1427 :
1428 11 : if (poMRFDS->bdirty != AllBandMask())
1429 0 : CPLError(CE_Warning, CPLE_AppDefined,
1430 : "MRF: IWrite, band dirty mask is " CPL_FRMT_GIB
1431 : " instead of " CPL_FRMT_GIB,
1432 0 : poMRFDS->bdirty, AllBandMask());
1433 :
1434 : buf_mgr src;
1435 11 : src.buffer = (char *)tbuffer;
1436 11 : src.size = static_cast<size_t>(img.pageSizeBytes);
1437 :
1438 : // Use the space after pagesizebytes for compressed output, it is of pbsize
1439 11 : char *outbuff = (char *)tbuffer + img.pageSizeBytes;
1440 :
1441 11 : buf_mgr dst = {outbuff, poMRFDS->pbsize};
1442 : CPLErr ret;
1443 :
1444 11 : auto start_time = steady_clock::now();
1445 :
1446 11 : ret = Compress(dst, src);
1447 11 : if (ret != CE_None)
1448 : {
1449 : // Compress failed, write it as an empty tile
1450 0 : CPLFree(tbuffer);
1451 0 : poMRFDS->WriteTile(nullptr, infooffset, 0);
1452 0 : return CE_None; // Should report the error, but it triggers partial
1453 : // band attempts
1454 : }
1455 :
1456 : // Where the output is, in case we deflate
1457 11 : void *usebuff = outbuff;
1458 11 : if (dodeflate)
1459 : {
1460 : // Move the packed part at the start of tbuffer, to make more space
1461 : // available
1462 0 : memcpy(tbuffer, outbuff, dst.size);
1463 0 : dst.buffer = static_cast<char *>(tbuffer);
1464 0 : usebuff = DeflateBlock(dst,
1465 0 : static_cast<size_t>(img.pageSizeBytes) +
1466 0 : poMRFDS->pbsize - dst.size,
1467 : deflate_flags);
1468 0 : if (!usebuff)
1469 0 : CPLError(CE_Failure, CPLE_AppDefined, "MRF: Deflate error");
1470 : }
1471 :
1472 : #if defined(ZSTD_SUPPORT)
1473 11 : else if (dozstd)
1474 : {
1475 1 : memcpy(tbuffer, outbuff, dst.size);
1476 1 : dst.buffer = static_cast<char *>(tbuffer);
1477 1 : size_t ranks = 0; // Assume no need for byte rank sort
1478 1 : if (img.comp == IL_NONE || img.comp == IL_ZSTD)
1479 1 : ranks =
1480 1 : static_cast<size_t>(GDALGetDataTypeSizeBytes(img.dt)) * cstride;
1481 3 : usebuff = ZstdCompBlock(dst,
1482 1 : static_cast<size_t>(img.pageSizeBytes) +
1483 1 : poMRFDS->pbsize - dst.size,
1484 1 : zstd_level, poMRFDS->getzsc(), ranks);
1485 1 : if (!usebuff)
1486 0 : CPLError(CE_Failure, CPLE_AppDefined,
1487 : "MRF: ZStd compression error");
1488 : }
1489 : #endif
1490 :
1491 11 : poMRFDS->write_timer +=
1492 11 : duration_cast<nanoseconds>(steady_clock::now() - start_time);
1493 :
1494 11 : if (!usebuff)
1495 : { // Error was signaled
1496 0 : CPLFree(tbuffer);
1497 0 : poMRFDS->WriteTile(nullptr, infooffset, 0);
1498 0 : poMRFDS->bdirty = 0;
1499 0 : return CE_Failure;
1500 : }
1501 :
1502 11 : ret = poMRFDS->WriteTile(usebuff, infooffset, dst.size);
1503 11 : CPLFree(tbuffer);
1504 :
1505 11 : poMRFDS->bdirty = 0;
1506 11 : return ret;
1507 : }
1508 :
1509 : //
1510 : // Tests if a given block exists without reading it
1511 : // returns false only when it is definitely not existing
1512 : //
1513 160 : bool MRFRasterBand::TestBlock(int xblk, int yblk)
1514 : {
1515 : // When bypassing the cache, assume all blocks are valid
1516 160 : if (poMRFDS->bypass_cache && !poMRFDS->source.empty())
1517 0 : return true;
1518 :
1519 : // Blocks outside of image have no data by default
1520 160 : if (xblk < 0 || yblk < 0 || xblk >= img.pagecount.x ||
1521 146 : yblk >= img.pagecount.y)
1522 25 : return false;
1523 :
1524 : ILIdx tinfo;
1525 135 : GInt32 cstride = img.pagesize.c;
1526 135 : ILSize req(xblk, yblk, 0, (nBand - 1) / cstride, m_l);
1527 :
1528 135 : if (CE_None != poMRFDS->ReadTileIdx(tinfo, req, img))
1529 : // Got an error reading the tile index
1530 0 : return !poMRFDS->no_errors;
1531 :
1532 : // Got an index, if the size is readable, the block does exist
1533 135 : if (0 < tinfo.size && tinfo.size < poMRFDS->pbsize * 2)
1534 135 : return true;
1535 :
1536 : // We are caching, but the tile has not been checked, so it could exist
1537 0 : return (!poMRFDS->source.empty() && 0 == tinfo.offset);
1538 : }
1539 :
1540 183 : int MRFRasterBand::GetOverviewCount()
1541 : {
1542 : // First try internal overviews
1543 183 : int nInternalOverviewCount = static_cast<int>(overviews.size());
1544 183 : if (nInternalOverviewCount > 0)
1545 137 : return nInternalOverviewCount;
1546 46 : return GDALPamRasterBand::GetOverviewCount();
1547 : }
1548 :
1549 158 : GDALRasterBand *MRFRasterBand::GetOverview(int n)
1550 : {
1551 : // First try internal overviews
1552 158 : if (n >= 0 && n < (int)overviews.size())
1553 85 : return overviews[n];
1554 73 : return GDALPamRasterBand::GetOverview(n);
1555 : }
1556 :
1557 : NAMESPACE_MRF_END
|
