Line data Source code
1 : /*
2 : * Copyright (c) 2002-2012, California Institute of Technology.
3 : * All rights reserved. Based on Government Sponsored Research under contracts
4 : * NAS7-1407 and/or NAS7-03001.
5 : *
6 : * Redistribution and use in source and binary forms, with or without
7 : * modification, are permitted provided that the following conditions are met:
8 : * 1. Redistributions of source code must retain the above copyright notice,
9 : * this list of conditions and the following disclaimer.
10 : * 2. Redistributions in binary form must reproduce the above copyright
11 : * notice, this list of conditions and the following disclaimer in the
12 : * documentation and/or other materials provided with the distribution.
13 : * 3. Neither the name of the California Institute of Technology (Caltech),
14 : * its operating division the Jet Propulsion Laboratory (JPL), the National
15 : * Aeronautics and Space Administration (NASA), nor the names of its
16 : * contributors may be used to endorse or promote products derived from this
17 : * software without specific prior written permission.
18 : *
19 : * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20 : * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 : * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 : * ARE DISCLAIMED. IN NO EVENT SHALL THE CALIFORNIA INSTITUTE OF TECHNOLOGY BE
23 : * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 : * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 : * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 : * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 : * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 : * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 : * POSSIBILITY OF SUCH DAMAGE.
30 : *
31 : * Copyright 2014-2021 Esri
32 : *
33 : * Licensed under the Apache License, Version 2.0 (the "License");
34 : * you may not use this file except in compliance with the License.
35 : * You may obtain a copy of the License at
36 : *
37 : * http://www.apache.org/licenses/LICENSE-2.0
38 : *
39 : * Unless required by applicable law or agreed to in writing, software
40 : * distributed under the License is distributed on an "AS IS" BASIS,
41 : * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
42 : * See the License for the specific language governing permissions and
43 : * limitations under the License.
44 : */
45 :
46 : /******************************************************************************
47 : *
48 : * Project: Meta Raster File Format Driver Implementation, RasterBand
49 : * Purpose: Implementation of MRF band
50 : *
51 : * Author: Lucian Plesea, Lucian.Plesea jpl.nasa.gov, lplesea esri.com
52 : *
53 : ****************************************************************************/
54 :
55 : #include "marfa.h"
56 : #include "gdal_priv.h"
57 : #include "ogr_srs_api.h"
58 : #include "ogr_spatialref.h"
59 :
60 : #include <vector>
61 : #include <algorithm>
62 : #include <cassert>
63 : #include <zlib.h>
64 : #if defined(ZSTD_SUPPORT)
65 : #include <zstd.h>
66 : #endif
67 :
68 : using namespace std::chrono;
69 :
70 : NAMESPACE_MRF_START
71 :
72 : // packs a block of a given type, with a stride
73 : // Count is the number of items that need to be copied
74 : // These are separate to allow for optimization
75 :
76 : template <typename T>
77 24 : static void cpy_stride_in(void *dst, void *src, int c, int stride)
78 : {
79 24 : T *s = reinterpret_cast<T *>(src);
80 24 : T *d = reinterpret_cast<T *>(dst);
81 :
82 6291480 : while (c--)
83 : {
84 6291460 : *d++ = *s;
85 6291460 : s += stride;
86 : }
87 24 : }
88 :
89 : template <typename T>
90 24 : static void cpy_stride_out(void *dst, void *src, int c, int stride)
91 : {
92 24 : T *s = reinterpret_cast<T *>(src);
93 24 : T *d = reinterpret_cast<T *>(dst);
94 :
95 6291480 : while (c--)
96 : {
97 6291460 : *d = *s++;
98 6291460 : d += stride;
99 : }
100 24 : }
101 :
102 : // Does every value in the buffer have the same value, using strict comparison
103 : template <typename T>
104 5110 : inline int isAllVal(const T *b, size_t bytecount, double ndv)
105 : {
106 5110 : T val = static_cast<T>(ndv);
107 5110 : size_t count = bytecount / sizeof(T);
108 2892307 : for (; count; --count)
109 : {
110 2892297 : if (*(b++) != val)
111 : {
112 5099 : return FALSE;
113 : }
114 : }
115 11 : return TRUE;
116 : }
117 :
118 : // Dispatcher based on gdal data type
119 5110 : static int isAllVal(GDALDataType gt, void *b, size_t bytecount, double ndv)
120 : {
121 : // Test to see if it has data
122 5110 : int isempty = false;
123 :
124 : // A case branch in a temporary macro, conversion from gdal enum to type
125 : #define TEST_T(GType, T) \
126 : case GType: \
127 : isempty = isAllVal(reinterpret_cast<T *>(b), bytecount, ndv); \
128 : break
129 :
130 5110 : switch (gt)
131 : {
132 4947 : TEST_T(GDT_Byte, GByte);
133 0 : TEST_T(GDT_Int8, GInt8);
134 28 : TEST_T(GDT_UInt16, GUInt16);
135 27 : TEST_T(GDT_Int16, GInt16);
136 26 : TEST_T(GDT_UInt32, GUInt32);
137 26 : TEST_T(GDT_Int32, GInt32);
138 0 : TEST_T(GDT_UInt64, GUInt64);
139 4 : TEST_T(GDT_Int64, GInt64);
140 26 : TEST_T(GDT_Float32, float);
141 26 : TEST_T(GDT_Float64, double);
142 0 : default:
143 0 : break;
144 : }
145 : #undef TEST_T
146 :
147 5110 : return isempty;
148 : }
149 :
150 : // Swap bytes in place, unconditional
151 : // cppcheck-suppress constParameterReference
152 0 : static void swab_buff(buf_mgr &src, const ILImage &img)
153 : {
154 : size_t i;
155 0 : switch (GDALGetDataTypeSize(img.dt))
156 : {
157 0 : case 16:
158 : {
159 0 : short int *b = reinterpret_cast<short int *>(src.buffer);
160 0 : for (i = src.size / 2; i; b++, i--)
161 0 : *b = swab16(*b);
162 0 : break;
163 : }
164 0 : case 32:
165 : {
166 0 : int *b = reinterpret_cast<int *>(src.buffer);
167 0 : for (i = src.size / 4; i; b++, i--)
168 0 : *b = swab32(*b);
169 0 : break;
170 : }
171 0 : case 64:
172 : {
173 0 : long long *b = reinterpret_cast<long long *>(src.buffer);
174 0 : for (i = src.size / 8; i; b++, i--)
175 0 : *b = swab64(*b);
176 0 : break;
177 : }
178 : }
179 0 : }
180 :
181 : // Similar to compress2() but with flags to control zlib features
182 : // Returns true if it worked
183 9 : static int ZPack(const buf_mgr &src, buf_mgr &dst, int flags)
184 : {
185 : z_stream stream;
186 : int err;
187 :
188 9 : memset(&stream, 0, sizeof(stream));
189 9 : stream.next_in = reinterpret_cast<Bytef *>(src.buffer);
190 9 : stream.avail_in = (uInt)src.size;
191 9 : stream.next_out = reinterpret_cast<Bytef *>(dst.buffer);
192 9 : stream.avail_out = (uInt)dst.size;
193 :
194 9 : int level = std::clamp(flags & ZFLAG_LMASK, 1, 9);
195 9 : int wb = MAX_WBITS;
196 : // if gz flag is set, ignore raw request
197 9 : if (flags & ZFLAG_GZ)
198 0 : wb += 16;
199 9 : else if (flags & ZFLAG_RAW)
200 0 : wb = -wb;
201 9 : int memlevel = 8; // Good compromise
202 9 : int strategy = (flags & ZFLAG_SMASK) >> 6;
203 9 : if (strategy > 4)
204 0 : strategy = 0;
205 :
206 9 : err = deflateInit2(&stream, level, Z_DEFLATED, wb, memlevel, strategy);
207 9 : if (err != Z_OK)
208 : {
209 0 : deflateEnd(&stream);
210 0 : return err;
211 : }
212 :
213 9 : err = deflate(&stream, Z_FINISH);
214 9 : if (err != Z_STREAM_END)
215 : {
216 0 : deflateEnd(&stream);
217 0 : return false;
218 : }
219 9 : dst.size = stream.total_out;
220 9 : err = deflateEnd(&stream);
221 9 : return err == Z_OK;
222 : }
223 :
224 : // Similar to uncompress() from zlib, accepts the ZFLAG_RAW
225 : // Return true if it worked
226 9 : static int ZUnPack(const buf_mgr &src, buf_mgr &dst, int flags)
227 : {
228 :
229 : z_stream stream;
230 : int err;
231 :
232 9 : memset(&stream, 0, sizeof(stream));
233 9 : stream.next_in = reinterpret_cast<Bytef *>(src.buffer);
234 9 : stream.avail_in = (uInt)src.size;
235 9 : stream.next_out = reinterpret_cast<Bytef *>(dst.buffer);
236 9 : stream.avail_out = (uInt)dst.size;
237 :
238 : // 32 means autodetec gzip or zlib header, negative 15 is for raw
239 9 : int wb = (ZFLAG_RAW & flags) ? -MAX_WBITS : 32 + MAX_WBITS;
240 9 : err = inflateInit2(&stream, wb);
241 9 : if (err != Z_OK)
242 0 : return false;
243 :
244 9 : err = inflate(&stream, Z_FINISH);
245 9 : if (err != Z_STREAM_END)
246 : {
247 0 : inflateEnd(&stream);
248 0 : return false;
249 : }
250 9 : dst.size = stream.total_out;
251 9 : err = inflateEnd(&stream);
252 9 : return err == Z_OK;
253 : }
254 :
255 : /*
256 : * Deflates a buffer, extrasize is the available size in the buffer past the
257 : * input If the output fits past the data, it uses that area, otherwise it uses
258 : * a temporary buffer and copies the data over the input on return, returning a
259 : * pointer to it. The output size is returned in src.size Returns nullptr when
260 : * compression failed
261 : */
262 9 : static void *DeflateBlock(buf_mgr &src, size_t extrasize, int flags)
263 : {
264 : // The one we might need to allocate
265 9 : void *dbuff = nullptr;
266 9 : buf_mgr dst = {src.buffer + src.size, extrasize};
267 :
268 : // Allocate a temp buffer if there is not sufficient space,
269 : // We need to have a bit more than half the buffer available
270 9 : if (extrasize < (src.size + 64))
271 : {
272 9 : dst.size = src.size + 64;
273 9 : dbuff = VSIMalloc(dst.size);
274 9 : dst.buffer = (char *)dbuff;
275 9 : if (!dst.buffer)
276 0 : return nullptr;
277 : }
278 :
279 9 : if (!ZPack(src, dst, flags))
280 : {
281 0 : CPLFree(dbuff); // Safe to call with NULL
282 0 : return nullptr;
283 : }
284 :
285 9 : if (src.size + extrasize < dst.size)
286 : {
287 0 : CPLError(CE_Failure, CPLE_AppDefined,
288 : "DeflateBlock(): too small buffer");
289 0 : CPLFree(dbuff);
290 0 : return nullptr;
291 : }
292 :
293 : // source size is used to hold the output size
294 9 : src.size = dst.size;
295 : // If we didn't allocate a buffer, the receiver can use it already
296 9 : if (!dbuff)
297 0 : return dst.buffer;
298 :
299 : // If we allocated a buffer, we need to copy the data to the input buffer.
300 9 : memcpy(src.buffer, dbuff, src.size);
301 9 : CPLFree(dbuff);
302 9 : return src.buffer;
303 : }
304 :
305 : #if defined(ZSTD_SUPPORT)
306 :
307 12 : CPL_NOSANITIZE_UNSIGNED_INT_OVERFLOW static void rankfilter(buf_mgr &src,
308 : size_t factor)
309 : {
310 : // Arange bytes by rank
311 12 : if (factor > 1)
312 : {
313 8 : std::vector<char> tempb(src.size);
314 8 : char *d = tempb.data();
315 43 : for (size_t j = 0; j < factor; j++)
316 9175080 : for (size_t i = j; i < src.size; i += factor)
317 9175040 : *d++ = src.buffer[i];
318 8 : memcpy(src.buffer, tempb.data(), src.size);
319 : }
320 : // byte delta
321 12 : auto p = reinterpret_cast<GByte *>(src.buffer);
322 12 : auto guard = p + src.size;
323 12 : GByte b(0);
324 10223600 : while (p < guard)
325 : {
326 10223600 : GByte temp = *p;
327 10223600 : *p -= b;
328 10223600 : b = temp;
329 10223600 : p++;
330 : }
331 12 : }
332 :
333 10 : CPL_NOSANITIZE_UNSIGNED_INT_OVERFLOW static void derank(buf_mgr &src,
334 : size_t factor)
335 : {
336 : // undo delta
337 10 : auto p = reinterpret_cast<GByte *>(src.buffer);
338 10 : auto guard = p + src.size;
339 10 : GByte b(0);
340 9699340 : while (p < guard)
341 : {
342 9699330 : b += *p;
343 9699330 : *p = b;
344 9699330 : p++;
345 : }
346 10 : if (factor > 1)
347 : { // undo rank separation
348 8 : std::vector<char> tempb(src.size);
349 8 : char *d = tempb.data();
350 8 : size_t chunk = src.size / factor;
351 2097160 : for (size_t i = 0; i < chunk; i++)
352 11272200 : for (size_t j = 0; j < factor; j++)
353 9175040 : *d++ = src.buffer[chunk * j + i];
354 8 : memcpy(src.buffer, tempb.data(), src.size);
355 : }
356 10 : }
357 :
358 : /*
359 : * Compress a buffer using zstd, extrasize is the available size in the buffer
360 : * past the input If ranks > 0, apply the rank filter If the output fits past
361 : * the data, it uses that area, otherwise it uses a temporary buffer and copies
362 : * the data over the input on return, returning a pointer to it. The output size
363 : * is returned in src.size Returns nullptr when compression failed
364 : */
365 12 : static void *ZstdCompBlock(buf_mgr &src, size_t extrasize, int c_level,
366 : ZSTD_CCtx *cctx, size_t ranks)
367 : {
368 12 : if (!cctx)
369 0 : return nullptr;
370 12 : if (ranks && (src.size % ranks) == 0)
371 12 : rankfilter(src, ranks);
372 :
373 : // might need a buffer for the zstd output
374 24 : std::vector<char> dbuff;
375 12 : void *dst = src.buffer + src.size;
376 12 : size_t size = extrasize;
377 : // Allocate a temp buffer if there is not sufficient space.
378 : // Zstd bound is about (size * 1.004 + 64)
379 12 : if (size < ZSTD_compressBound(src.size))
380 : {
381 12 : size = ZSTD_compressBound(src.size);
382 12 : dbuff.resize(size);
383 12 : dst = dbuff.data();
384 : }
385 :
386 : // Use the streaming interface, it's faster and better
387 : // See discussion at https://github.com/facebook/zstd/issues/3729
388 12 : ZSTD_outBuffer output = {dst, size, 0};
389 12 : ZSTD_inBuffer input = {src.buffer, src.size, 0};
390 : // Set level
391 12 : ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, c_level);
392 : // First, pass a continue flag, otherwise it will compress in one go
393 12 : size_t val = ZSTD_compressStream2(cctx, &output, &input, ZSTD_e_continue);
394 : // If it worked, pass the end flag to flush the buffer
395 12 : if (val == 0)
396 12 : val = ZSTD_compressStream2(cctx, &output, &input, ZSTD_e_end);
397 12 : if (ZSTD_isError(val))
398 0 : return nullptr;
399 12 : val = output.pos;
400 :
401 : // If we didn't need the buffer, packed data is already in the user buffer
402 12 : if (dbuff.empty())
403 : {
404 0 : src.size = val;
405 0 : return dst;
406 : }
407 :
408 12 : if (val > (src.size + extrasize))
409 : { // Doesn't fit in user buffer
410 0 : CPLError(CE_Failure, CPLE_AssertionFailed,
411 : "MRF: ZSTD compression buffer too small");
412 0 : return nullptr; // Error
413 : }
414 :
415 12 : memcpy(src.buffer, dbuff.data(), val);
416 12 : src.size = val;
417 12 : return src.buffer;
418 : }
419 : #endif
420 :
421 : //
422 : // The deflate_flags are available in all bands even if the DEFLATE option
423 : // itself is not set. This allows for PNG features to be controlled, as well
424 : // as any other bands that use zlib by itself
425 : //
426 547 : MRFRasterBand::MRFRasterBand(MRFDataset *parent_dataset, const ILImage &image,
427 547 : int band, int ov)
428 : : poMRFDS(parent_dataset),
429 547 : dodeflate(GetOptlist().FetchBoolean("DEFLATE", FALSE)),
430 : // Bring the quality to 0 to 9
431 547 : deflate_flags(image.quality / 10),
432 547 : dozstd(GetOptlist().FetchBoolean("ZSTD", FALSE)), zstd_level(9), m_l(ov),
433 1641 : img(image)
434 : {
435 547 : nBand = band;
436 547 : eDataType = parent_dataset->current.dt;
437 547 : nRasterXSize = img.size.x;
438 547 : nRasterYSize = img.size.y;
439 547 : nBlockXSize = img.pagesize.x;
440 547 : nBlockYSize = img.pagesize.y;
441 547 : nBlocksPerRow = img.pagecount.x;
442 547 : nBlocksPerColumn = img.pagecount.y;
443 547 : img.NoDataValue = MRFRasterBand::GetNoDataValue(&img.hasNoData);
444 :
445 : // Pick up the twists, aka GZ, RAWZ headers
446 547 : if (GetOptlist().FetchBoolean("GZ", FALSE))
447 0 : deflate_flags |= ZFLAG_GZ;
448 547 : else if (GetOptlist().FetchBoolean("RAWZ", FALSE))
449 0 : deflate_flags |= ZFLAG_RAW;
450 : // And Pick up the ZLIB strategy, if any
451 547 : const char *zstrategy = GetOptlist().FetchNameValueDef("Z_STRATEGY", "");
452 547 : int zv = Z_DEFAULT_STRATEGY;
453 547 : if (EQUAL(zstrategy, "Z_HUFFMAN_ONLY"))
454 0 : zv = Z_HUFFMAN_ONLY;
455 547 : else if (EQUAL(zstrategy, "Z_RLE"))
456 0 : zv = Z_RLE;
457 547 : else if (EQUAL(zstrategy, "Z_FILTERED"))
458 0 : zv = Z_FILTERED;
459 547 : else if (EQUAL(zstrategy, "Z_FIXED"))
460 0 : zv = Z_FIXED;
461 547 : deflate_flags |= (zv << 6);
462 547 : if (image.quality < 23 && image.quality > 0)
463 0 : zstd_level = image.quality;
464 :
465 : #if !defined(ZSTD_SUPPORT)
466 : if (dozstd)
467 : { // signal error condition to caller
468 : CPLError(CE_Failure, CPLE_AssertionFailed,
469 : "MRF: ZSTD support is not available");
470 : dozstd = FALSE;
471 : }
472 : #endif
473 : // Chose zstd over deflate if both are enabled and available
474 547 : if (dozstd && dodeflate)
475 0 : dodeflate = FALSE;
476 547 : }
477 :
478 : // Clean up the overviews if they exist
479 1094 : MRFRasterBand::~MRFRasterBand()
480 : {
481 619 : while (!overviews.empty())
482 : {
483 72 : delete overviews.back();
484 72 : overviews.pop_back();
485 : }
486 547 : }
487 :
488 : // Look for a string from the dataset options or from the environment
489 75 : const char *MRFRasterBand::GetOptionValue(const char *opt,
490 : const char *def) const
491 : {
492 75 : const char *optValue = poMRFDS->optlist.FetchNameValue(opt);
493 75 : if (optValue)
494 8 : return optValue;
495 67 : return CPLGetConfigOption(opt, def);
496 : }
497 :
498 : // Utility function, returns a value from a vector corresponding to the band
499 : // index or the first entry
500 157 : static double getBandValue(const std::vector<double> &v, int idx)
501 : {
502 157 : return (static_cast<int>(v.size()) > idx) ? v[idx] : v[0];
503 : }
504 :
505 : // Maybe we should check against the type range?
506 : // It is not keeping track of how many values have been set,
507 : // so the application should set none or all the bands
508 : // This call is only valid during Create
509 16 : CPLErr MRFRasterBand::SetNoDataValue(double val)
510 : {
511 16 : if (poMRFDS->bCrystalized)
512 : {
513 0 : CPLError(CE_Failure, CPLE_AssertionFailed,
514 : "MRF: NoData can be set only during file create");
515 0 : return CE_Failure;
516 : }
517 16 : if (GInt32(poMRFDS->vNoData.size()) < nBand)
518 0 : poMRFDS->vNoData.resize(nBand);
519 16 : poMRFDS->vNoData[nBand - 1] = val;
520 : // We also need to set it for this band
521 16 : img.NoDataValue = val;
522 16 : img.hasNoData = true;
523 16 : return CE_None;
524 : }
525 :
526 6008 : double MRFRasterBand::GetNoDataValue(int *pbSuccess)
527 : {
528 6008 : const std::vector<double> &v = poMRFDS->vNoData;
529 6008 : if (v.empty())
530 5851 : return GDALPamRasterBand::GetNoDataValue(pbSuccess);
531 157 : if (pbSuccess)
532 155 : *pbSuccess = TRUE;
533 157 : return getBandValue(v, nBand - 1);
534 : }
535 :
536 0 : double MRFRasterBand::GetMinimum(int *pbSuccess)
537 : {
538 0 : const std::vector<double> &v = poMRFDS->vMin;
539 0 : if (v.empty())
540 0 : return GDALPamRasterBand::GetMinimum(pbSuccess);
541 0 : if (pbSuccess)
542 0 : *pbSuccess = TRUE;
543 0 : return getBandValue(v, nBand - 1);
544 : }
545 :
546 0 : double MRFRasterBand::GetMaximum(int *pbSuccess)
547 : {
548 0 : const std::vector<double> &v = poMRFDS->vMax;
549 0 : if (v.empty())
550 0 : return GDALPamRasterBand::GetMaximum(pbSuccess);
551 0 : if (pbSuccess)
552 0 : *pbSuccess = TRUE;
553 0 : return getBandValue(v, nBand - 1);
554 : }
555 :
556 : // Fill with typed ndv, count is always in bytes
557 : template <typename T>
558 0 : static CPLErr buff_fill(void *b, size_t count, const T ndv)
559 : {
560 0 : T *buffer = static_cast<T *>(b);
561 0 : count /= sizeof(T);
562 0 : while (count--)
563 0 : *buffer++ = ndv;
564 0 : return CE_None;
565 : }
566 :
567 : /**
568 : *\brief Fills a buffer with no data
569 : *
570 : */
571 56 : CPLErr MRFRasterBand::FillBlock(void *buffer)
572 : {
573 : int success;
574 56 : double ndv = GetNoDataValue(&success);
575 56 : if (!success)
576 56 : ndv = 0.0;
577 56 : size_t bsb = blockSizeBytes();
578 :
579 : // use memset for speed for bytes, or if nodata is zeros
580 56 : if (0.0 == ndv || eDataType == GDT_Byte || eDataType == GDT_Int8)
581 : {
582 56 : memset(buffer, int(ndv), bsb);
583 56 : return CE_None;
584 : }
585 :
586 : #define bf(T) buff_fill<T>(buffer, bsb, T(ndv));
587 0 : switch (eDataType)
588 : {
589 0 : case GDT_UInt16:
590 0 : return bf(GUInt16);
591 0 : case GDT_Int16:
592 0 : return bf(GInt16);
593 0 : case GDT_UInt32:
594 0 : return bf(GUInt32);
595 0 : case GDT_Int32:
596 0 : return bf(GInt32);
597 0 : case GDT_UInt64:
598 0 : return bf(GUInt64);
599 0 : case GDT_Int64:
600 0 : return bf(GInt64);
601 0 : case GDT_Float32:
602 0 : return bf(float);
603 0 : case GDT_Float64:
604 0 : return bf(double);
605 0 : default:
606 0 : break;
607 : }
608 : #undef bf
609 : // Should exit before
610 0 : return CE_Failure;
611 : }
612 :
613 : /*\brief Interleave block fill
614 : *
615 : * Acquire space for all the other bands, fill each one then drop the locks
616 : * The current band output goes directly into the buffer
617 : */
618 :
619 0 : CPLErr MRFRasterBand::FillBlock(int xblk, int yblk, void *buffer)
620 : {
621 0 : std::vector<GDALRasterBlock *> blocks;
622 :
623 0 : for (int i = 0; i < poMRFDS->nBands; i++)
624 : {
625 0 : GDALRasterBand *b = poMRFDS->GetRasterBand(i + 1);
626 0 : if (b->GetOverviewCount() && 0 != m_l)
627 0 : b = b->GetOverview(m_l - 1);
628 :
629 : // Get the other band blocks, keep them around until later
630 0 : if (b == this)
631 : {
632 0 : FillBlock(buffer);
633 : }
634 : else
635 : {
636 0 : GDALRasterBlock *poBlock = b->GetLockedBlockRef(xblk, yblk, 1);
637 0 : if (poBlock == nullptr) // Didn't get this block
638 0 : break;
639 0 : FillBlock(poBlock->GetDataRef());
640 0 : blocks.push_back(poBlock);
641 : }
642 : }
643 :
644 : // Drop the locks for blocks we acquired
645 0 : for (int i = 0; i < int(blocks.size()); i++)
646 0 : blocks[i]->DropLock();
647 :
648 0 : return CE_None;
649 : }
650 :
651 : /*\brief Interleave block read
652 : *
653 : * Acquire space for all the other bands, unpack from the dataset buffer, then
654 : * drop the locks The current band output goes directly into the buffer
655 : */
656 :
657 8 : CPLErr MRFRasterBand::ReadInterleavedBlock(int xblk, int yblk, void *buffer)
658 : {
659 8 : std::vector<GDALRasterBlock *> blocks;
660 :
661 32 : for (int i = 0; i < poMRFDS->nBands; i++)
662 : {
663 24 : GDALRasterBand *b = poMRFDS->GetRasterBand(i + 1);
664 24 : if (b->GetOverviewCount() && 0 != m_l)
665 0 : b = b->GetOverview(m_l - 1);
666 :
667 24 : void *ob = buffer;
668 : // Get the other band blocks, keep them around until later
669 24 : if (b != this)
670 : {
671 16 : GDALRasterBlock *poBlock = b->GetLockedBlockRef(xblk, yblk, 1);
672 16 : if (poBlock == nullptr)
673 0 : break;
674 16 : ob = poBlock->GetDataRef();
675 16 : blocks.push_back(poBlock);
676 : }
677 :
678 : // Just the right mix of templates and macros make deinterleaving tidy
679 24 : void *pbuffer = poMRFDS->GetPBuffer();
680 : #define CpySI(T) \
681 : cpy_stride_in<T>(ob, reinterpret_cast<T *>(pbuffer) + i, \
682 : blockSizeBytes() / sizeof(T), img.pagesize.c)
683 :
684 : // Page is already in poMRFDS->pbuffer, not empty
685 : // There are only four cases, since only the data size matters
686 24 : switch (GDALGetDataTypeSize(eDataType) / 8)
687 : {
688 24 : case 1:
689 24 : CpySI(GByte);
690 24 : break;
691 0 : case 2:
692 0 : CpySI(GInt16);
693 0 : break;
694 0 : case 4:
695 0 : CpySI(GInt32);
696 0 : break;
697 0 : case 8:
698 0 : CpySI(GIntBig);
699 0 : break;
700 : }
701 : }
702 :
703 : #undef CpySI
704 :
705 : // Drop the locks we acquired
706 24 : for (int i = 0; i < int(blocks.size()); i++)
707 16 : blocks[i]->DropLock();
708 :
709 16 : return CE_None;
710 : }
711 :
712 : /**
713 : *\brief Fetch a block from the backing store dataset and keep a copy in the
714 : *cache
715 : *
716 : * @param xblk The X block number, zero based
717 : * @param yblk The Y block number, zero based
718 : * @param buffer buffer
719 : *
720 : */
721 4 : CPLErr MRFRasterBand::FetchBlock(int xblk, int yblk, void *buffer)
722 : {
723 4 : assert(!poMRFDS->source.empty());
724 4 : CPLDebug("MRF_IB", "FetchBlock %d,%d,0,%d, level %d\n", xblk, yblk, nBand,
725 : m_l);
726 :
727 4 : if (poMRFDS->clonedSource) // This is a clone
728 1 : return FetchClonedBlock(xblk, yblk, buffer);
729 :
730 3 : const GInt32 cstride = img.pagesize.c; // 1 if band separate
731 3 : ILSize req(xblk, yblk, 0, (nBand - 1) / cstride, m_l);
732 3 : GUIntBig infooffset = IdxOffset(req, img);
733 :
734 3 : GDALDataset *poSrcDS = nullptr;
735 3 : if (nullptr == (poSrcDS = poMRFDS->GetSrcDS()))
736 : {
737 1 : CPLError(CE_Failure, CPLE_AppDefined, "MRF: Can't open source file %s",
738 1 : poMRFDS->source.c_str());
739 1 : return CE_Failure;
740 : }
741 :
742 : // Scale to base resolution
743 2 : double scl = pow(poMRFDS->scale, m_l);
744 2 : if (0 == m_l)
745 2 : scl = 1; // To allow for precision issues
746 :
747 : // Prepare parameters for RasterIO, they might be different from a full page
748 2 : const GSpacing vsz = GDALGetDataTypeSizeBytes(eDataType);
749 2 : int Xoff = int(xblk * img.pagesize.x * scl + 0.5);
750 2 : int Yoff = int(yblk * img.pagesize.y * scl + 0.5);
751 2 : int readszx = int(img.pagesize.x * scl + 0.5);
752 2 : int readszy = int(img.pagesize.y * scl + 0.5);
753 :
754 : // Compare with the full size and clip to the right and bottom if needed
755 2 : int clip = 0;
756 2 : if (Xoff + readszx > poMRFDS->full.size.x)
757 : {
758 2 : clip |= 1;
759 2 : readszx = poMRFDS->full.size.x - Xoff;
760 : }
761 2 : if (Yoff + readszy > poMRFDS->full.size.y)
762 : {
763 2 : clip |= 1;
764 2 : readszy = poMRFDS->full.size.y - Yoff;
765 : }
766 :
767 : // This is where the whole page fits
768 2 : void *ob = buffer;
769 2 : if (cstride != 1)
770 0 : ob = poMRFDS->GetPBuffer();
771 :
772 : // Fill buffer with NoData if clipping
773 2 : if (clip)
774 2 : FillBlock(ob);
775 :
776 : // Use the dataset RasterIO to read one or all bands if interleaved
777 4 : CPLErr ret = poSrcDS->RasterIO(
778 : GF_Read, Xoff, Yoff, readszx, readszy, ob, pcount(readszx, int(scl)),
779 : pcount(readszy, int(scl)), eDataType, cstride,
780 2 : (1 == cstride) ? &nBand : nullptr, vsz * cstride,
781 2 : vsz * cstride * img.pagesize.x,
782 2 : (cstride != 1) ? vsz : vsz * img.pagesize.x * img.pagesize.y, nullptr);
783 :
784 2 : if (ret != CE_None)
785 0 : return ret;
786 :
787 : // Might have the block in the pbuffer, mark it anyhow
788 2 : poMRFDS->tile = req;
789 : buf_mgr filesrc;
790 2 : filesrc.buffer = (char *)ob;
791 2 : filesrc.size = static_cast<size_t>(img.pageSizeBytes);
792 :
793 2 : if (poMRFDS->bypass_cache)
794 : { // No local caching, just return the data
795 0 : if (1 == cstride)
796 0 : return CE_None;
797 0 : return ReadInterleavedBlock(xblk, yblk, buffer);
798 : }
799 :
800 : // Test to see if it needs to be written, or just marked as checked
801 : int success;
802 2 : double val = GetNoDataValue(&success);
803 2 : if (!success)
804 2 : val = 0.0;
805 :
806 : // TODO: test band by band if data is interleaved
807 2 : if (isAllVal(eDataType, ob, img.pageSizeBytes, val))
808 : {
809 : // Mark it empty and checked, ignore the possible write error
810 0 : poMRFDS->WriteTile((void *)1, infooffset, 0);
811 0 : if (1 == cstride)
812 0 : return CE_None;
813 0 : return ReadInterleavedBlock(xblk, yblk, buffer);
814 : }
815 :
816 : // Write the page in the local cache
817 :
818 : // Have to use a separate buffer for compression output.
819 2 : void *outbuff = VSIMalloc(poMRFDS->pbsize);
820 2 : if (nullptr == outbuff)
821 : {
822 0 : CPLError(CE_Failure, CPLE_AppDefined,
823 : "Can't get buffer for writing page");
824 : // This is not really an error for a cache, the data is fine
825 0 : return CE_Failure;
826 : }
827 :
828 2 : buf_mgr filedst = {static_cast<char *>(outbuff), poMRFDS->pbsize};
829 2 : auto start_time = steady_clock::now();
830 2 : if (Compress(filedst, filesrc) != CE_None)
831 : {
832 0 : return CE_Failure;
833 : }
834 :
835 : // Where the output is, in case we deflate
836 2 : void *usebuff = outbuff;
837 2 : if (dodeflate)
838 : {
839 0 : CPLAssert(poMRFDS->pbsize <= filedst.size);
840 0 : usebuff = DeflateBlock(filedst, poMRFDS->pbsize - filedst.size,
841 : deflate_flags);
842 0 : if (!usebuff)
843 : {
844 0 : CPLError(CE_Failure, CPLE_AppDefined, "MRF: Deflate error");
845 0 : return CE_Failure;
846 : }
847 : }
848 :
849 : #if defined(ZSTD_SUPPORT)
850 2 : else if (dozstd)
851 : {
852 0 : size_t ranks = 0; // Assume no need for byte rank sort
853 0 : if (img.comp == IL_NONE || img.comp == IL_ZSTD)
854 0 : ranks =
855 0 : static_cast<size_t>(GDALGetDataTypeSizeBytes(img.dt)) * cstride;
856 0 : usebuff = ZstdCompBlock(filedst, poMRFDS->pbsize - filedst.size,
857 0 : zstd_level, poMRFDS->getzsc(), ranks);
858 0 : if (!usebuff)
859 : {
860 0 : CPLError(CE_Failure, CPLE_AppDefined,
861 : "MRF: ZSTD compression error");
862 0 : return CE_Failure;
863 : }
864 : }
865 : #endif
866 :
867 2 : poMRFDS->write_timer +=
868 2 : duration_cast<nanoseconds>(steady_clock::now() - start_time);
869 :
870 : // Write and update the tile index
871 2 : ret = poMRFDS->WriteTile(usebuff, infooffset, filedst.size);
872 2 : CPLFree(outbuff);
873 :
874 : // If we hit an error or if unpaking is not needed
875 2 : if (ret != CE_None || cstride == 1)
876 2 : return ret;
877 :
878 : // data is already in DS buffer, deinterlace it in pixel blocks
879 0 : return ReadInterleavedBlock(xblk, yblk, buffer);
880 : }
881 :
882 : /**
883 : *\brief Fetch for a cloned MRF
884 : *
885 : * @param xblk The X block number, zero based
886 : * @param yblk The Y block number, zero based
887 : * @param buffer buffer
888 : *
889 : */
890 :
891 1 : CPLErr MRFRasterBand::FetchClonedBlock(int xblk, int yblk, void *buffer)
892 : {
893 1 : CPLDebug("MRF_IB", "FetchClonedBlock %d,%d,0,%d, level %d\n", xblk, yblk,
894 : nBand, m_l);
895 :
896 : // Paranoid check
897 1 : assert(poMRFDS->clonedSource);
898 1 : MRFDataset *poSrc = static_cast<MRFDataset *>(poMRFDS->GetSrcDS());
899 1 : if (nullptr == poSrc)
900 : {
901 0 : CPLError(CE_Failure, CPLE_AppDefined, "MRF: Can't open source file %s",
902 0 : poMRFDS->source.c_str());
903 0 : return CE_Failure;
904 : }
905 :
906 1 : if (poMRFDS->bypass_cache || GF_Read == DataMode())
907 : {
908 : // Can't store, so just fetch from source, which is an MRF with
909 : // identical structure
910 : MRFRasterBand *b =
911 0 : static_cast<MRFRasterBand *>(poSrc->GetRasterBand(nBand));
912 0 : if (b->GetOverviewCount() && m_l)
913 0 : b = static_cast<MRFRasterBand *>(b->GetOverview(m_l - 1));
914 0 : if (b == nullptr)
915 0 : return CE_Failure;
916 0 : return b->IReadBlock(xblk, yblk, buffer);
917 : }
918 :
919 1 : ILSize req(xblk, yblk, 0, (nBand - 1) / img.pagesize.c, m_l);
920 : ILIdx tinfo;
921 :
922 : // Get the cloned source tile info
923 : // The cloned source index is after the current one
924 1 : if (CE_None != poMRFDS->ReadTileIdx(tinfo, req, img, poMRFDS->idxSize))
925 : {
926 0 : CPLError(CE_Failure, CPLE_AppDefined,
927 : "MRF: Unable to read cloned index entry");
928 0 : return CE_Failure;
929 : }
930 :
931 1 : GUIntBig infooffset = IdxOffset(req, img);
932 : CPLErr err;
933 :
934 : // Does the source have this tile?
935 1 : if (tinfo.size == 0)
936 : { // Nope, mark it empty and return fill
937 0 : err = poMRFDS->WriteTile((void *)1, infooffset, 0);
938 0 : if (CE_None != err)
939 0 : return err;
940 0 : return FillBlock(buffer);
941 : }
942 :
943 1 : VSILFILE *srcfd = poSrc->DataFP();
944 1 : if (nullptr == srcfd)
945 : {
946 0 : CPLError(CE_Failure, CPLE_AppDefined,
947 : "MRF: Can't open source data file %s",
948 0 : poMRFDS->source.c_str());
949 0 : return CE_Failure;
950 : }
951 :
952 : // Need to read the tile from the source
953 1 : if (tinfo.size <= 0 || tinfo.size > INT_MAX)
954 : {
955 0 : CPLError(CE_Failure, CPLE_OutOfMemory,
956 : "Invalid tile size " CPL_FRMT_GIB, tinfo.size);
957 0 : return CE_Failure;
958 : }
959 1 : char *buf = static_cast<char *>(VSIMalloc(static_cast<size_t>(tinfo.size)));
960 1 : if (buf == nullptr)
961 : {
962 0 : CPLError(CE_Failure, CPLE_OutOfMemory,
963 : "Cannot allocate " CPL_FRMT_GIB " bytes", tinfo.size);
964 0 : return CE_Failure;
965 : }
966 :
967 1 : VSIFSeekL(srcfd, tinfo.offset, SEEK_SET);
968 2 : if (tinfo.size !=
969 1 : GIntBig(VSIFReadL(buf, 1, static_cast<size_t>(tinfo.size), srcfd)))
970 : {
971 0 : CPLFree(buf);
972 0 : CPLError(CE_Failure, CPLE_AppDefined,
973 : "MRF: Can't read data from source %s",
974 : poSrc->current.datfname.c_str());
975 0 : return CE_Failure;
976 : }
977 :
978 : // Write it then reissue the read
979 1 : err = poMRFDS->WriteTile(buf, infooffset, tinfo.size);
980 1 : CPLFree(buf);
981 1 : if (CE_None != err)
982 0 : return err;
983 : // Reissue read, it will work from the cloned data
984 1 : return IReadBlock(xblk, yblk, buffer);
985 : }
986 :
987 : /**
988 : *\brief read a block in the provided buffer
989 : *
990 : * For separate band model, the DS buffer is not used, the read is direct in
991 : *the buffer For pixel interleaved model, the DS buffer holds the temp copy and
992 : *all the other bands are force read
993 : *
994 : */
995 :
996 1882 : CPLErr MRFRasterBand::IReadBlock(int xblk, int yblk, void *buffer)
997 : {
998 1882 : GInt32 cstride = img.pagesize.c;
999 : ILIdx tinfo;
1000 1882 : ILSize req(xblk, yblk, 0, (nBand - 1) / cstride, m_l);
1001 3764 : CPLDebug("MRF_IB",
1002 : "IReadBlock %d,%d,0,%d, level %d, idxoffset " CPL_FRMT_GIB "\n",
1003 1882 : xblk, yblk, nBand - 1, m_l, IdxOffset(req, img));
1004 :
1005 : // If this is a caching file and bypass is on, just do the fetch
1006 1882 : if (poMRFDS->bypass_cache && !poMRFDS->source.empty())
1007 0 : return FetchBlock(xblk, yblk, buffer);
1008 :
1009 1882 : tinfo.size = 0; // Just in case it is missing
1010 1882 : if (CE_None != poMRFDS->ReadTileIdx(tinfo, req, img))
1011 : {
1012 0 : if (!poMRFDS->no_errors)
1013 : {
1014 0 : CPLError(CE_Failure, CPLE_AppDefined,
1015 : "MRF: Unable to read index at offset " CPL_FRMT_GIB,
1016 0 : IdxOffset(req, img));
1017 0 : return CE_Failure;
1018 : }
1019 0 : return FillBlock(buffer);
1020 : }
1021 :
1022 1882 : if (0 == tinfo.size)
1023 : { // Could be missing or it could be caching
1024 : // Offset != 0 means no data, Update mode is for local MRFs only
1025 : // if caching index mode is RO don't try to fetch
1026 : // Also, caching MRFs can't be opened in update mode
1027 12 : if (0 != tinfo.offset || GA_Update == poMRFDS->eAccess ||
1028 18 : poMRFDS->source.empty() || IdxMode() == GF_Read)
1029 2 : return FillBlock(buffer);
1030 :
1031 : // caching MRF, need to fetch a block
1032 4 : return FetchBlock(xblk, yblk, buffer);
1033 : }
1034 :
1035 1876 : CPLDebug("MRF_IB", "Tinfo offset " CPL_FRMT_GIB ", size " CPL_FRMT_GIB "\n",
1036 : tinfo.offset, tinfo.size);
1037 : // If we have a tile, read it
1038 :
1039 : // Should use a permanent buffer, like the pbuffer mechanism
1040 : // Get a large buffer, in case we need to unzip
1041 :
1042 : // We add a padding of 3 bytes since in LERC1 decompression, we can
1043 : // dereference a unsigned int at the end of the buffer, that can be
1044 : // partially out of the buffer.
1045 : // Can be reproduced with :
1046 : // gdal_translate ../autotest/gcore/data/byte.tif out.mrf -of MRF -co
1047 : // COMPRESS=LERC -co OPTIONS=V1:YES -ot Float32 valgrind gdalinfo -checksum
1048 : // out.mrf Invalid read of size 4 at BitStuffer::read(unsigned char**,
1049 : // std::vector<unsigned int, std::allocator<unsigned int> >&) const
1050 : // (BitStuffer.cpp:153)
1051 :
1052 : // No stored tile should be larger than twice the raw size.
1053 1876 : if (tinfo.size <= 0 || tinfo.size > poMRFDS->pbsize * 2)
1054 : {
1055 0 : if (!poMRFDS->no_errors)
1056 : {
1057 0 : CPLError(CE_Failure, CPLE_OutOfMemory,
1058 : "Stored tile is too large: " CPL_FRMT_GIB, tinfo.size);
1059 0 : return CE_Failure;
1060 : }
1061 0 : return FillBlock(buffer);
1062 : }
1063 :
1064 1876 : VSILFILE *dfp = DataFP();
1065 :
1066 : // No data file to read from
1067 1876 : if (dfp == nullptr)
1068 0 : return CE_Failure;
1069 :
1070 1876 : void *data = VSIMalloc(static_cast<size_t>(tinfo.size + PADDING_BYTES));
1071 1876 : if (data == nullptr)
1072 : {
1073 0 : CPLError(CE_Failure, CPLE_OutOfMemory,
1074 : "Could not allocate memory for tile size: " CPL_FRMT_GIB,
1075 : tinfo.size);
1076 0 : return CE_Failure;
1077 : }
1078 :
1079 : // This part is not thread safe, but it is what GDAL expects
1080 1876 : VSIFSeekL(dfp, tinfo.offset, SEEK_SET);
1081 1876 : if (1 != VSIFReadL(data, static_cast<size_t>(tinfo.size), 1, dfp))
1082 : {
1083 0 : CPLFree(data);
1084 0 : if (poMRFDS->no_errors)
1085 0 : return FillBlock(buffer);
1086 0 : CPLError(CE_Failure, CPLE_AppDefined, "Unable to read data page, %d@%x",
1087 0 : static_cast<int>(tinfo.size), static_cast<int>(tinfo.offset));
1088 0 : return CE_Failure;
1089 : }
1090 :
1091 : /* initialize padding bytes */
1092 1876 : memset(((char *)data) + static_cast<size_t>(tinfo.size), 0, PADDING_BYTES);
1093 1876 : buf_mgr src = {(char *)data, static_cast<size_t>(tinfo.size)};
1094 : buf_mgr dst;
1095 :
1096 1876 : auto start_time = steady_clock::now();
1097 :
1098 : // We got the data, do we need to decompress it before decoding?
1099 1876 : if (dodeflate)
1100 : {
1101 9 : if (img.pageSizeBytes > INT_MAX - 1440)
1102 : {
1103 0 : CPLFree(data);
1104 0 : CPLError(CE_Failure, CPLE_AppDefined, "Page size is too big at %d",
1105 : img.pageSizeBytes);
1106 0 : return CE_Failure;
1107 : }
1108 9 : dst.size =
1109 9 : img.pageSizeBytes +
1110 : 1440; // in case the packed page is a bit larger than the raw one
1111 9 : dst.buffer = static_cast<char *>(VSIMalloc(dst.size));
1112 9 : if (nullptr == dst.buffer)
1113 : {
1114 0 : CPLFree(data);
1115 0 : CPLError(CE_Failure, CPLE_OutOfMemory, "Cannot allocate %d bytes",
1116 0 : static_cast<int>(dst.size));
1117 0 : return CE_Failure;
1118 : }
1119 :
1120 9 : if (ZUnPack(src, dst, deflate_flags))
1121 : { // Got it unpacked, update the pointers
1122 9 : CPLFree(data);
1123 9 : data = dst.buffer;
1124 9 : tinfo.size = dst.size;
1125 : }
1126 : else
1127 : { // assume the page was not gzipped, warn only
1128 0 : CPLFree(dst.buffer);
1129 0 : if (!poMRFDS->no_errors)
1130 0 : CPLError(CE_Warning, CPLE_AppDefined, "Can't inflate page!");
1131 : }
1132 : }
1133 :
1134 : #if defined(ZSTD_SUPPORT)
1135 : // undo ZSTD
1136 1867 : else if (dozstd)
1137 : {
1138 10 : auto ctx = poMRFDS->getzsd();
1139 10 : if (!ctx)
1140 : {
1141 0 : CPLFree(data);
1142 0 : CPLError(CE_Failure, CPLE_AppDefined, "Can't acquire ZSTD context");
1143 0 : return CE_Failure;
1144 : }
1145 10 : if (img.pageSizeBytes > INT_MAX - 1440)
1146 : {
1147 0 : CPLFree(data);
1148 0 : CPLError(CE_Failure, CPLE_AppDefined, "Page is too large at %d",
1149 : img.pageSizeBytes);
1150 0 : return CE_Failure;
1151 : }
1152 10 : dst.size =
1153 10 : img.pageSizeBytes +
1154 : 1440; // Allow for a slight increase from previous compressions
1155 10 : dst.buffer = static_cast<char *>(VSIMalloc(dst.size));
1156 10 : if (nullptr == dst.buffer)
1157 : {
1158 0 : CPLFree(data);
1159 0 : CPLError(CE_Failure, CPLE_OutOfMemory, "Cannot allocate %d bytes",
1160 0 : static_cast<int>(dst.size));
1161 0 : return CE_Failure;
1162 : }
1163 :
1164 20 : auto raw_size = ZSTD_decompressDCtx(ctx, dst.buffer, dst.size,
1165 10 : src.buffer, src.size);
1166 10 : if (ZSTD_isError(raw_size))
1167 : { // assume page was not packed, warn only
1168 0 : CPLFree(dst.buffer);
1169 0 : if (!poMRFDS->no_errors)
1170 0 : CPLError(CE_Warning, CPLE_AppDefined,
1171 : "Can't unpack ZSTD page!");
1172 : }
1173 : else
1174 : {
1175 10 : CPLFree(data); // The compressed data
1176 10 : data = dst.buffer;
1177 10 : tinfo.size = raw_size;
1178 : // Might need to undo the rank sort
1179 10 : size_t ranks = 0;
1180 10 : if (img.comp == IL_NONE || img.comp == IL_ZSTD)
1181 10 : ranks = static_cast<size_t>(GDALGetDataTypeSizeBytes(img.dt)) *
1182 10 : img.pagesize.c;
1183 10 : if (ranks)
1184 : {
1185 10 : src.buffer = static_cast<char *>(data);
1186 10 : src.size = static_cast<size_t>(tinfo.size);
1187 10 : derank(src, ranks);
1188 : }
1189 : }
1190 : }
1191 : #endif
1192 :
1193 1876 : src.buffer = static_cast<char *>(data);
1194 1876 : src.size = static_cast<size_t>(tinfo.size);
1195 :
1196 : // After unpacking, the size has to be pageSizeBytes
1197 : // If pages are interleaved, use the dataset page buffer instead
1198 1876 : dst.buffer = reinterpret_cast<char *>(
1199 8 : (1 == cstride) ? buffer : poMRFDS->GetPBuffer());
1200 1876 : dst.size = img.pageSizeBytes;
1201 :
1202 1876 : if (poMRFDS->no_errors)
1203 0 : CPLPushErrorHandler(CPLQuietErrorHandler);
1204 1876 : CPLErr ret = Decompress(dst, src);
1205 :
1206 1876 : poMRFDS->read_timer +=
1207 1876 : duration_cast<nanoseconds>(steady_clock::now() - start_time);
1208 :
1209 1876 : dst.size =
1210 1876 : img.pageSizeBytes; // In case the decompress failed, force it back
1211 :
1212 : // Swap whatever we decompressed if we need to
1213 1876 : if (is_Endianness_Dependent(img.dt, img.comp) && (img.nbo != NET_ORDER))
1214 0 : swab_buff(dst, img);
1215 :
1216 1876 : CPLFree(data);
1217 1876 : if (poMRFDS->no_errors)
1218 : {
1219 0 : CPLPopErrorHandler();
1220 0 : if (ret != CE_None) // Set each page buffer to the correct no data
1221 : // value, then proceed
1222 0 : return (1 == cstride) ? FillBlock(buffer)
1223 0 : : FillBlock(xblk, yblk, buffer);
1224 : }
1225 :
1226 : // If pages are separate or we had errors, we're done
1227 1876 : if (1 == cstride || CE_None != ret)
1228 1868 : return ret;
1229 :
1230 : // De-interleave page from dataset buffer and return
1231 8 : return ReadInterleavedBlock(xblk, yblk, buffer);
1232 : }
1233 :
1234 : /**
1235 : *\brief Write a block from the provided buffer
1236 : *
1237 : * Same trick as read, use a temporary tile buffer for pixel interleave
1238 : * For band separate, use a
1239 : * Write the block once it has all the bands, report
1240 : * if a new block is started before the old one was completed
1241 : *
1242 : */
1243 :
1244 5092 : CPLErr MRFRasterBand::IWriteBlock(int xblk, int yblk, void *buffer)
1245 : {
1246 5092 : GInt32 cstride = img.pagesize.c;
1247 5092 : ILSize req(xblk, yblk, 0, (nBand - 1) / cstride, m_l);
1248 5092 : GUIntBig infooffset = IdxOffset(req, img);
1249 :
1250 5092 : CPLDebug("MRF_IB", "IWriteBlock %d,%d,0,%d, level %d, stride %d\n", xblk,
1251 : yblk, nBand, m_l, cstride);
1252 :
1253 : // Finish the Create call
1254 5092 : if (!poMRFDS->bCrystalized && !poMRFDS->Crystalize())
1255 : {
1256 0 : CPLError(CE_Failure, CPLE_AppDefined, "MRF: Error creating files");
1257 0 : return CE_Failure;
1258 : }
1259 :
1260 5092 : if (1 == cstride)
1261 : { // Separate bands, we can write it as is
1262 : // Empty page skip
1263 : int success;
1264 5084 : double val = GetNoDataValue(&success);
1265 5084 : if (!success)
1266 5012 : val = 0.0;
1267 5084 : if (isAllVal(eDataType, buffer, img.pageSizeBytes, val))
1268 11 : return poMRFDS->WriteTile(nullptr, infooffset, 0);
1269 :
1270 : // Use the pbuffer to hold the compressed page before writing it
1271 5073 : poMRFDS->tile = ILSize(); // Mark it corrupt
1272 :
1273 : buf_mgr src;
1274 5073 : src.buffer = (char *)buffer;
1275 5073 : src.size = static_cast<size_t>(img.pageSizeBytes);
1276 5073 : buf_mgr dst = {(char *)poMRFDS->GetPBuffer(),
1277 5073 : poMRFDS->GetPBufferSize()};
1278 :
1279 : // Swab the source before encoding if we need to
1280 5073 : if (is_Endianness_Dependent(img.dt, img.comp) && (img.nbo != NET_ORDER))
1281 0 : swab_buff(src, img);
1282 :
1283 5073 : auto start_time = steady_clock::now();
1284 :
1285 : // Compress functions need to return the compressed size in
1286 : // the bytes in buffer field
1287 5073 : if (Compress(dst, src) != CE_None)
1288 0 : return CE_Failure;
1289 :
1290 5073 : void *usebuff = dst.buffer;
1291 5073 : if (dodeflate)
1292 : {
1293 9 : CPLAssert(dst.size <= poMRFDS->pbsize);
1294 : usebuff =
1295 9 : DeflateBlock(dst, poMRFDS->pbsize - dst.size, deflate_flags);
1296 9 : if (!usebuff)
1297 : {
1298 0 : CPLError(CE_Failure, CPLE_AppDefined, "MRF: Deflate error");
1299 0 : return CE_Failure;
1300 : }
1301 : }
1302 :
1303 : #if defined(ZSTD_SUPPORT)
1304 5064 : else if (dozstd)
1305 : {
1306 11 : size_t ranks = 0; // Assume no need for byte rank sort
1307 11 : if (img.comp == IL_NONE || img.comp == IL_ZSTD)
1308 11 : ranks = static_cast<size_t>(GDALGetDataTypeSizeBytes(img.dt));
1309 11 : usebuff = ZstdCompBlock(dst, poMRFDS->pbsize - dst.size, zstd_level,
1310 11 : poMRFDS->getzsc(), ranks);
1311 11 : if (!usebuff)
1312 : {
1313 0 : CPLError(CE_Failure, CPLE_AppDefined,
1314 : "MRF: Zstd Compression error");
1315 0 : return CE_Failure;
1316 : }
1317 : }
1318 : #endif
1319 5073 : poMRFDS->write_timer +=
1320 5073 : duration_cast<nanoseconds>(steady_clock::now() - start_time);
1321 5073 : return poMRFDS->WriteTile(usebuff, infooffset, dst.size);
1322 : }
1323 :
1324 : // Multiple bands per page, use a temporary to assemble the page
1325 : // Temporary is large because we use it to hold both the uncompressed and
1326 : // the compressed
1327 8 : poMRFDS->tile = req;
1328 8 : poMRFDS->bdirty = 0;
1329 :
1330 : // Keep track of what bands are empty
1331 8 : GUIntBig empties = 0;
1332 :
1333 8 : void *tbuffer = VSIMalloc(img.pageSizeBytes + poMRFDS->pbsize);
1334 :
1335 8 : if (!tbuffer)
1336 : {
1337 0 : CPLError(CE_Failure, CPLE_AppDefined,
1338 : "MRF: Can't allocate write buffer");
1339 0 : return CE_Failure;
1340 : }
1341 :
1342 : // Get the other bands from the block cache
1343 32 : for (int iBand = 0; iBand < poMRFDS->nBands; iBand++)
1344 : {
1345 24 : char *pabyThisImage = nullptr;
1346 24 : GDALRasterBlock *poBlock = nullptr;
1347 :
1348 24 : if (iBand == nBand - 1)
1349 : {
1350 8 : pabyThisImage = reinterpret_cast<char *>(buffer);
1351 8 : poMRFDS->bdirty |= bandbit();
1352 : }
1353 : else
1354 : {
1355 16 : GDALRasterBand *band = poMRFDS->GetRasterBand(iBand + 1);
1356 : // Pick the right overview
1357 16 : if (m_l)
1358 0 : band = band->GetOverview(m_l - 1);
1359 : poBlock = (reinterpret_cast<MRFRasterBand *>(band))
1360 16 : ->TryGetLockedBlockRef(xblk, yblk);
1361 16 : if (nullptr == poBlock)
1362 0 : continue;
1363 : // This is where the image data is for this band
1364 :
1365 16 : pabyThisImage = reinterpret_cast<char *>(poBlock->GetDataRef());
1366 16 : poMRFDS->bdirty |= bandbit(iBand);
1367 : }
1368 :
1369 : // Keep track of empty bands, but encode them anyhow, in case some are
1370 : // not empty
1371 : int success;
1372 24 : double val = GetNoDataValue(&success);
1373 24 : if (!success)
1374 24 : val = 0.0;
1375 24 : if (isAllVal(eDataType, pabyThisImage, blockSizeBytes(), val))
1376 0 : empties |= bandbit(iBand);
1377 :
1378 : // Copy the data into the dataset buffer here
1379 : // Just the right mix of templates and macros make this real tidy
1380 : #define CpySO(T) \
1381 : cpy_stride_out<T>((reinterpret_cast<T *>(tbuffer)) + iBand, pabyThisImage, \
1382 : blockSizeBytes() / sizeof(T), cstride)
1383 :
1384 : // Build the page in tbuffer
1385 24 : switch (GDALGetDataTypeSize(eDataType) / 8)
1386 : {
1387 24 : case 1:
1388 24 : CpySO(GByte);
1389 24 : break;
1390 0 : case 2:
1391 0 : CpySO(GInt16);
1392 0 : break;
1393 0 : case 4:
1394 0 : CpySO(GInt32);
1395 0 : break;
1396 0 : case 8:
1397 0 : CpySO(GIntBig);
1398 0 : break;
1399 0 : default:
1400 : {
1401 0 : CPLError(CE_Failure, CPLE_AppDefined,
1402 : "MRF: Write datatype of %d bytes "
1403 : "not implemented",
1404 0 : GDALGetDataTypeSize(eDataType) / 8);
1405 0 : if (poBlock != nullptr)
1406 : {
1407 0 : poBlock->MarkClean();
1408 0 : poBlock->DropLock();
1409 : }
1410 0 : CPLFree(tbuffer);
1411 0 : return CE_Failure;
1412 : }
1413 : }
1414 :
1415 24 : if (poBlock != nullptr)
1416 : {
1417 16 : poBlock->MarkClean();
1418 16 : poBlock->DropLock();
1419 : }
1420 : }
1421 :
1422 : // Should keep track of the individual band buffers and only mix them if
1423 : // this is not an empty page ( move the Copy with Stride Out from above
1424 : // below this test This way works fine, but it does work extra for empty
1425 : // pages
1426 :
1427 8 : if (GIntBig(empties) == AllBandMask())
1428 : {
1429 0 : CPLFree(tbuffer);
1430 0 : return poMRFDS->WriteTile(nullptr, infooffset, 0);
1431 : }
1432 :
1433 8 : if (poMRFDS->bdirty != AllBandMask())
1434 0 : CPLError(CE_Warning, CPLE_AppDefined,
1435 : "MRF: IWrite, band dirty mask is " CPL_FRMT_GIB
1436 : " instead of " CPL_FRMT_GIB,
1437 0 : poMRFDS->bdirty, AllBandMask());
1438 :
1439 : buf_mgr src;
1440 8 : src.buffer = (char *)tbuffer;
1441 8 : src.size = static_cast<size_t>(img.pageSizeBytes);
1442 :
1443 : // Use the space after pagesizebytes for compressed output, it is of pbsize
1444 8 : char *outbuff = (char *)tbuffer + img.pageSizeBytes;
1445 :
1446 8 : buf_mgr dst = {outbuff, poMRFDS->pbsize};
1447 : CPLErr ret;
1448 :
1449 8 : auto start_time = steady_clock::now();
1450 :
1451 8 : ret = Compress(dst, src);
1452 8 : if (ret != CE_None)
1453 : {
1454 : // Compress failed, write it as an empty tile
1455 0 : CPLFree(tbuffer);
1456 0 : poMRFDS->WriteTile(nullptr, infooffset, 0);
1457 0 : return CE_None; // Should report the error, but it triggers partial
1458 : // band attempts
1459 : }
1460 :
1461 : // Where the output is, in case we deflate
1462 8 : void *usebuff = outbuff;
1463 8 : if (dodeflate)
1464 : {
1465 : // Move the packed part at the start of tbuffer, to make more space
1466 : // available
1467 0 : memcpy(tbuffer, outbuff, dst.size);
1468 0 : dst.buffer = static_cast<char *>(tbuffer);
1469 0 : usebuff = DeflateBlock(dst,
1470 0 : static_cast<size_t>(img.pageSizeBytes) +
1471 0 : poMRFDS->pbsize - dst.size,
1472 : deflate_flags);
1473 0 : if (!usebuff)
1474 0 : CPLError(CE_Failure, CPLE_AppDefined, "MRF: Deflate error");
1475 : }
1476 :
1477 : #if defined(ZSTD_SUPPORT)
1478 8 : else if (dozstd)
1479 : {
1480 1 : memcpy(tbuffer, outbuff, dst.size);
1481 1 : dst.buffer = static_cast<char *>(tbuffer);
1482 1 : size_t ranks = 0; // Assume no need for byte rank sort
1483 1 : if (img.comp == IL_NONE || img.comp == IL_ZSTD)
1484 1 : ranks =
1485 1 : static_cast<size_t>(GDALGetDataTypeSizeBytes(img.dt)) * cstride;
1486 3 : usebuff = ZstdCompBlock(dst,
1487 1 : static_cast<size_t>(img.pageSizeBytes) +
1488 1 : poMRFDS->pbsize - dst.size,
1489 1 : zstd_level, poMRFDS->getzsc(), ranks);
1490 1 : if (!usebuff)
1491 0 : CPLError(CE_Failure, CPLE_AppDefined,
1492 : "MRF: ZStd compression error");
1493 : }
1494 : #endif
1495 :
1496 8 : poMRFDS->write_timer +=
1497 8 : duration_cast<nanoseconds>(steady_clock::now() - start_time);
1498 :
1499 8 : if (!usebuff)
1500 : { // Error was signaled
1501 0 : CPLFree(tbuffer);
1502 0 : poMRFDS->WriteTile(nullptr, infooffset, 0);
1503 0 : poMRFDS->bdirty = 0;
1504 0 : return CE_Failure;
1505 : }
1506 :
1507 8 : ret = poMRFDS->WriteTile(usebuff, infooffset, dst.size);
1508 8 : CPLFree(tbuffer);
1509 :
1510 8 : poMRFDS->bdirty = 0;
1511 8 : return ret;
1512 : }
1513 :
1514 : //
1515 : // Tests if a given block exists without reading it
1516 : // returns false only when it is definitely not existing
1517 : //
1518 160 : bool MRFRasterBand::TestBlock(int xblk, int yblk)
1519 : {
1520 : // When bypassing the cache, assume all blocks are valid
1521 160 : if (poMRFDS->bypass_cache && !poMRFDS->source.empty())
1522 0 : return true;
1523 :
1524 : // Blocks outside of image have no data by default
1525 160 : if (xblk < 0 || yblk < 0 || xblk >= img.pagecount.x ||
1526 146 : yblk >= img.pagecount.y)
1527 25 : return false;
1528 :
1529 : ILIdx tinfo;
1530 135 : GInt32 cstride = img.pagesize.c;
1531 135 : ILSize req(xblk, yblk, 0, (nBand - 1) / cstride, m_l);
1532 :
1533 135 : if (CE_None != poMRFDS->ReadTileIdx(tinfo, req, img))
1534 : // Got an error reading the tile index
1535 0 : return !poMRFDS->no_errors;
1536 :
1537 : // Got an index, if the size is readable, the block does exist
1538 135 : if (0 < tinfo.size && tinfo.size < poMRFDS->pbsize * 2)
1539 135 : return true;
1540 :
1541 : // We are caching, but the tile has not been checked, so it could exist
1542 0 : return (!poMRFDS->source.empty() && 0 == tinfo.offset);
1543 : }
1544 :
1545 183 : int MRFRasterBand::GetOverviewCount()
1546 : {
1547 : // First try internal overviews
1548 183 : int nInternalOverviewCount = static_cast<int>(overviews.size());
1549 183 : if (nInternalOverviewCount > 0)
1550 137 : return nInternalOverviewCount;
1551 46 : return GDALPamRasterBand::GetOverviewCount();
1552 : }
1553 :
1554 158 : GDALRasterBand *MRFRasterBand::GetOverview(int n)
1555 : {
1556 : // First try internal overviews
1557 158 : if (n >= 0 && n < (int)overviews.size())
1558 85 : return overviews[n];
1559 73 : return GDALPamRasterBand::GetOverview(n);
1560 : }
1561 :
1562 : NAMESPACE_MRF_END
|
