Line data Source code
1 : /******************************************************************************
2 : *
3 : * Project: GDAL Core
4 : * Purpose: Contains default implementation of GDALRasterBand::IRasterIO()
5 : * and supporting functions of broader utility.
6 : * Author: Frank Warmerdam, warmerdam@pobox.com
7 : *
8 : ******************************************************************************
9 : * Copyright (c) 1998, Frank Warmerdam
10 : * Copyright (c) 2007-2014, Even Rouault <even dot rouault at spatialys.com>
11 : *
12 : * SPDX-License-Identifier: MIT
13 : ****************************************************************************/
14 :
15 : #include "cpl_port.h"
16 : #include "gdal.h"
17 : #include "gdal_priv.h"
18 :
19 : #include <cassert>
20 : #include <climits>
21 : #include <cmath>
22 : #include <cstddef>
23 : #include <cstdio>
24 : #include <cstdlib>
25 : #include <cstring>
26 :
27 : #include <algorithm>
28 : #include <limits>
29 : #include <stdexcept>
30 : #include <type_traits>
31 :
32 : #include "cpl_conv.h"
33 : #include "cpl_cpu_features.h"
34 : #include "cpl_error.h"
35 : #include "cpl_float.h"
36 : #include "cpl_progress.h"
37 : #include "cpl_string.h"
38 : #include "cpl_vsi.h"
39 : #include "gdal_priv_templates.hpp"
40 : #include "gdal_vrt.h"
41 : #include "gdalwarper.h"
42 : #include "memdataset.h"
43 : #include "vrtdataset.h"
44 :
45 : #if defined(__x86_64) || defined(_M_X64)
46 : #include <emmintrin.h>
47 : #define HAVE_SSE2
48 : #elif defined(USE_NEON_OPTIMIZATIONS)
49 : #include "include_sse2neon.h"
50 : #define HAVE_SSE2
51 : #endif
52 :
53 : #ifdef HAVE_SSSE3_AT_COMPILE_TIME
54 : #include "rasterio_ssse3.h"
55 : #ifdef __SSSE3__
56 : #include <tmmintrin.h>
57 : #endif
58 : #endif
59 :
60 : #ifdef __SSE4_1__
61 : #include <smmintrin.h>
62 : #endif
63 :
64 : #ifdef __GNUC__
65 : #define CPL_NOINLINE __attribute__((noinline))
66 : #else
67 : #define CPL_NOINLINE
68 : #endif
69 :
70 : static void GDALFastCopyByte(const GByte *CPL_RESTRICT pSrcData,
71 : int nSrcPixelStride, GByte *CPL_RESTRICT pDstData,
72 : int nDstPixelStride, GPtrDiff_t nWordCount);
73 :
74 : /************************************************************************/
75 : /* DownsamplingIntegerXFactor() */
76 : /************************************************************************/
77 :
78 : template <bool bSameDataType, int DATA_TYPE_SIZE>
79 695677 : static bool DownsamplingIntegerXFactor(
80 : GDALRasterBand *poBand, int iSrcX, int nSrcXInc, GPtrDiff_t iSrcOffsetCst,
81 : GByte *CPL_RESTRICT pabyDstData, int nPixelSpace, int nBufXSize,
82 : GDALDataType eDataType, GDALDataType eBufType, int &nStartBlockX,
83 : int nBlockXSize, GDALRasterBlock *&poBlock, int nLBlockY)
84 : {
85 695677 : const int nBandDataSize =
86 : bSameDataType ? DATA_TYPE_SIZE : GDALGetDataTypeSizeBytes(eDataType);
87 695677 : int nOuterLoopIters = nBufXSize - 1;
88 695677 : const int nIncSrcOffset = nSrcXInc * nBandDataSize;
89 : const GByte *CPL_RESTRICT pabySrcData;
90 695677 : int nEndBlockX = nBlockXSize + nStartBlockX;
91 :
92 695677 : if (iSrcX < nEndBlockX)
93 : {
94 294999 : CPLAssert(poBlock);
95 294999 : goto no_reload_block;
96 : }
97 400678 : goto reload_block;
98 :
99 : // Don't do the last iteration in the loop, as iSrcX might go beyond
100 : // nRasterXSize - 1
101 1264772 : while (--nOuterLoopIters >= 1)
102 : {
103 201834 : iSrcX += nSrcXInc;
104 201834 : pabySrcData += nIncSrcOffset;
105 201834 : pabyDstData += nPixelSpace;
106 :
107 : /* --------------------------------------------------------------------
108 : */
109 : /* Ensure we have the appropriate block loaded. */
110 : /* --------------------------------------------------------------------
111 : */
112 201834 : if (iSrcX >= nEndBlockX)
113 : {
114 201834 : reload_block:
115 : {
116 615102 : const int nLBlockX = iSrcX / nBlockXSize;
117 615102 : nStartBlockX = nLBlockX * nBlockXSize;
118 615102 : nEndBlockX = nStartBlockX + nBlockXSize;
119 :
120 615102 : if (poBlock != nullptr)
121 341314 : poBlock->DropLock();
122 :
123 615102 : poBlock = poBand->GetLockedBlockRef(nLBlockX, nLBlockY, FALSE);
124 615102 : if (poBlock == nullptr)
125 : {
126 1 : return false;
127 : }
128 : }
129 :
130 615101 : no_reload_block:
131 : const GByte *pabySrcBlock =
132 1264772 : static_cast<const GByte *>(poBlock->GetDataRef());
133 1264772 : GPtrDiff_t iSrcOffset =
134 1264772 : (iSrcX - nStartBlockX + iSrcOffsetCst) * nBandDataSize;
135 1264772 : pabySrcData = pabySrcBlock + iSrcOffset;
136 : }
137 :
138 : /* --------------------------------------------------------------------
139 : */
140 : /* Copy the maximum run of pixels. */
141 : /* --------------------------------------------------------------------
142 : */
143 :
144 1264772 : const int nIters = std::min(
145 1264772 : (nEndBlockX - iSrcX + (nSrcXInc - 1)) / nSrcXInc, nOuterLoopIters);
146 : if (bSameDataType)
147 : {
148 1264367 : memcpy(pabyDstData, pabySrcData, nBandDataSize);
149 1264367 : if (nIters > 1)
150 : {
151 : if (DATA_TYPE_SIZE == 1)
152 : {
153 326246 : pabySrcData += nIncSrcOffset;
154 326246 : pabyDstData += nPixelSpace;
155 326246 : GDALFastCopyByte(pabySrcData, nIncSrcOffset, pabyDstData,
156 326246 : nPixelSpace, nIters - 1);
157 326246 : pabySrcData +=
158 326246 : static_cast<GPtrDiff_t>(nIncSrcOffset) * (nIters - 2);
159 326246 : pabyDstData +=
160 326246 : static_cast<GPtrDiff_t>(nPixelSpace) * (nIters - 2);
161 : }
162 : else
163 : {
164 4395158 : for (int i = 0; i < nIters - 1; i++)
165 : {
166 4197064 : pabySrcData += nIncSrcOffset;
167 4197064 : pabyDstData += nPixelSpace;
168 4197064 : memcpy(pabyDstData, pabySrcData, nBandDataSize);
169 : }
170 : }
171 524340 : iSrcX += nSrcXInc * (nIters - 1);
172 524340 : nOuterLoopIters -= nIters - 1;
173 : }
174 : }
175 : else
176 : {
177 : // Type to type conversion ...
178 405 : GDALCopyWords64(pabySrcData, eDataType, nIncSrcOffset, pabyDstData,
179 405 : eBufType, nPixelSpace, std::max(1, nIters));
180 405 : if (nIters > 1)
181 : {
182 198 : pabySrcData +=
183 198 : static_cast<GPtrDiff_t>(nIncSrcOffset) * (nIters - 1);
184 198 : pabyDstData +=
185 198 : static_cast<GPtrDiff_t>(nPixelSpace) * (nIters - 1);
186 198 : iSrcX += nSrcXInc * (nIters - 1);
187 198 : nOuterLoopIters -= nIters - 1;
188 : }
189 : }
190 : }
191 :
192 : // Deal with last iteration to avoid iSrcX to go beyond nRasterXSize - 1
193 1062938 : if (nOuterLoopIters == 0)
194 : {
195 367262 : const int nRasterXSize = poBand->GetXSize();
196 367262 : iSrcX =
197 734524 : static_cast<int>(std::min(static_cast<GInt64>(iSrcX) + nSrcXInc,
198 367262 : static_cast<GInt64>(nRasterXSize - 1)));
199 367262 : pabyDstData += nPixelSpace;
200 367262 : if (iSrcX < nEndBlockX)
201 : {
202 354672 : goto no_reload_block;
203 : }
204 12590 : goto reload_block;
205 : }
206 695676 : return true;
207 : }
208 :
209 : template <class A, class B>
210 2726190 : CPL_NOSANITIZE_UNSIGNED_INT_OVERFLOW inline auto CPLUnsanitizedMul(A a, B b)
211 : {
212 2726190 : return a * b;
213 : }
214 :
215 : /************************************************************************/
216 : /* IRasterIO() */
217 : /* */
218 : /* Default internal implementation of RasterIO() ... utilizes */
219 : /* the Block access methods to satisfy the request. This would */
220 : /* normally only be overridden by formats with overviews. */
221 : /************************************************************************/
222 :
223 6127870 : CPLErr GDALRasterBand::IRasterIO(GDALRWFlag eRWFlag, int nXOff, int nYOff,
224 : int nXSize, int nYSize, void *pData,
225 : int nBufXSize, int nBufYSize,
226 : GDALDataType eBufType, GSpacing nPixelSpace,
227 : GSpacing nLineSpace,
228 : GDALRasterIOExtraArg *psExtraArg)
229 :
230 : {
231 6127870 : if (eRWFlag == GF_Write && eFlushBlockErr != CE_None)
232 : {
233 0 : CPLError(eFlushBlockErr, CPLE_AppDefined,
234 : "An error occurred while writing a dirty block "
235 : "from GDALRasterBand::IRasterIO");
236 0 : CPLErr eErr = eFlushBlockErr;
237 0 : eFlushBlockErr = CE_None;
238 0 : return eErr;
239 : }
240 6127870 : if (nBlockXSize <= 0 || nBlockYSize <= 0)
241 : {
242 86 : CPLError(CE_Failure, CPLE_AppDefined, "Invalid block size");
243 0 : return CE_Failure;
244 : }
245 :
246 6127790 : const int nBandDataSize = GDALGetDataTypeSizeBytes(eDataType);
247 6127770 : const int nBufDataSize = GDALGetDataTypeSizeBytes(eBufType);
248 6127750 : GByte dummyBlock[2] = {0, 0};
249 6127750 : GByte *pabySrcBlock =
250 : dummyBlock; /* to avoid Coverity warning about nullptr dereference */
251 6127750 : GDALRasterBlock *poBlock = nullptr;
252 6127750 : const bool bUseIntegerRequestCoords =
253 6474420 : (!psExtraArg->bFloatingPointWindowValidity ||
254 346676 : (nXOff == psExtraArg->dfXOff && nYOff == psExtraArg->dfYOff &&
255 323297 : nXSize == psExtraArg->dfXSize && nYSize == psExtraArg->dfYSize));
256 :
257 : /* ==================================================================== */
258 : /* A common case is the data requested with the destination */
259 : /* is packed, and the block width is the raster width. */
260 : /* ==================================================================== */
261 6050070 : if (nPixelSpace == nBufDataSize && nLineSpace == nPixelSpace * nXSize &&
262 3203680 : nBlockXSize == GetXSize() && nBufXSize == nXSize &&
263 12177900 : nBufYSize == nYSize && bUseIntegerRequestCoords)
264 : {
265 3091520 : CPLErr eErr = CE_None;
266 3091520 : int nLBlockY = -1;
267 :
268 9283930 : for (int iBufYOff = 0; iBufYOff < nBufYSize; iBufYOff++)
269 : {
270 6193370 : const int iSrcY = iBufYOff + nYOff;
271 :
272 6193370 : if (iSrcY < nLBlockY * nBlockYSize ||
273 6193110 : iSrcY - nBlockYSize >= nLBlockY * nBlockYSize)
274 : {
275 3353920 : nLBlockY = iSrcY / nBlockYSize;
276 3353920 : bool bJustInitialize =
277 295403 : eRWFlag == GF_Write && nXOff == 0 &&
278 3706390 : nXSize == nBlockXSize && nYOff <= nLBlockY * nBlockYSize &&
279 57065 : nYOff + nYSize - nBlockYSize >= nLBlockY * nBlockYSize;
280 :
281 : // Is this a partial tile at right and/or bottom edges of
282 : // the raster, and that is going to be completely written?
283 : // If so, do not load it from storage, but zero it so that
284 : // the content outsize of the validity area is initialized.
285 3353920 : bool bMemZeroBuffer = false;
286 295403 : if (eRWFlag == GF_Write && !bJustInitialize && nXOff == 0 &&
287 23861 : nXSize == nBlockXSize && nYOff <= nLBlockY * nBlockYSize &&
288 3649420 : nYOff + nYSize == GetYSize() &&
289 89 : nLBlockY * nBlockYSize > GetYSize() - nBlockYSize)
290 : {
291 89 : bJustInitialize = true;
292 89 : bMemZeroBuffer = true;
293 : }
294 :
295 3353920 : if (poBlock)
296 262234 : poBlock->DropLock();
297 :
298 3353920 : const GUInt32 nErrorCounter = CPLGetErrorCounter();
299 3353760 : poBlock = GetLockedBlockRef(0, nLBlockY, bJustInitialize);
300 3354100 : if (poBlock == nullptr)
301 : {
302 1078 : if (strstr(CPLGetLastErrorMsg(), "IReadBlock failed") ==
303 : nullptr)
304 : {
305 0 : CPLError(CE_Failure, CPLE_AppDefined,
306 : "GetBlockRef failed at X block offset %d, "
307 : "Y block offset %d%s",
308 : 0, nLBlockY,
309 0 : (nErrorCounter != CPLGetErrorCounter())
310 0 : ? CPLSPrintf(": %s", CPLGetLastErrorMsg())
311 : : "");
312 : }
313 1078 : eErr = CE_Failure;
314 1078 : break;
315 : }
316 :
317 3353020 : if (eRWFlag == GF_Write)
318 295403 : poBlock->MarkDirty();
319 :
320 3353020 : pabySrcBlock = static_cast<GByte *>(poBlock->GetDataRef());
321 3353010 : if (bMemZeroBuffer)
322 : {
323 89 : memset(pabySrcBlock, 0,
324 89 : static_cast<GPtrDiff_t>(nBandDataSize) *
325 89 : nBlockXSize * nBlockYSize);
326 : }
327 : }
328 :
329 6192450 : const auto nSrcByteOffset =
330 6192450 : (static_cast<GPtrDiff_t>(iSrcY - nLBlockY * nBlockYSize) *
331 6192450 : nBlockXSize +
332 6192450 : nXOff) *
333 6192450 : nBandDataSize;
334 :
335 6192450 : if (eDataType == eBufType)
336 : {
337 2555050 : if (eRWFlag == GF_Read)
338 2084740 : memcpy(static_cast<GByte *>(pData) +
339 2084740 : static_cast<GPtrDiff_t>(iBufYOff) * nLineSpace,
340 2084740 : pabySrcBlock + nSrcByteOffset,
341 : static_cast<size_t>(nLineSpace));
342 : else
343 470311 : memcpy(pabySrcBlock + nSrcByteOffset,
344 470311 : static_cast<GByte *>(pData) +
345 470311 : static_cast<GPtrDiff_t>(iBufYOff) * nLineSpace,
346 : static_cast<size_t>(nLineSpace));
347 : }
348 : else
349 : {
350 : // Type to type conversion.
351 3637400 : if (eRWFlag == GF_Read)
352 3615140 : GDALCopyWords64(
353 3615140 : pabySrcBlock + nSrcByteOffset, eDataType, nBandDataSize,
354 : static_cast<GByte *>(pData) +
355 3615140 : static_cast<GPtrDiff_t>(iBufYOff) * nLineSpace,
356 : eBufType, static_cast<int>(nPixelSpace), nBufXSize);
357 : else
358 22264 : GDALCopyWords64(static_cast<GByte *>(pData) +
359 22264 : static_cast<GPtrDiff_t>(iBufYOff) *
360 : nLineSpace,
361 : eBufType, static_cast<int>(nPixelSpace),
362 22264 : pabySrcBlock + nSrcByteOffset, eDataType,
363 : nBandDataSize, nBufXSize);
364 : }
365 :
366 6265970 : if (psExtraArg->pfnProgress != nullptr &&
367 73559 : !psExtraArg->pfnProgress(1.0 * (iBufYOff + 1) / nBufYSize, "",
368 : psExtraArg->pProgressData))
369 : {
370 5 : eErr = CE_Failure;
371 5 : break;
372 : }
373 : }
374 :
375 3091640 : if (poBlock)
376 3090740 : poBlock->DropLock();
377 :
378 3091820 : return eErr;
379 : }
380 :
381 : /* ==================================================================== */
382 : /* Do we have overviews that would be appropriate to satisfy */
383 : /* this request? */
384 : /* ==================================================================== */
385 3036290 : if ((nBufXSize < nXSize || nBufYSize < nYSize) && GetOverviewCount() > 0 &&
386 : eRWFlag == GF_Read)
387 : {
388 : GDALRasterIOExtraArg sExtraArg;
389 2902 : GDALCopyRasterIOExtraArg(&sExtraArg, psExtraArg);
390 :
391 : const int nOverview =
392 2902 : GDALBandGetBestOverviewLevel2(this, nXOff, nYOff, nXSize, nYSize,
393 : nBufXSize, nBufYSize, &sExtraArg);
394 2902 : if (nOverview >= 0)
395 : {
396 2827 : GDALRasterBand *poOverviewBand = GetOverview(nOverview);
397 2827 : if (poOverviewBand == nullptr)
398 2827 : return CE_Failure;
399 :
400 2827 : return poOverviewBand->RasterIO(
401 : eRWFlag, nXOff, nYOff, nXSize, nYSize, pData, nBufXSize,
402 2827 : nBufYSize, eBufType, nPixelSpace, nLineSpace, &sExtraArg);
403 : }
404 : }
405 :
406 844556 : if (eRWFlag == GF_Read && nBufXSize < nXSize / 100 &&
407 6 : nBufYSize < nYSize / 100 && nPixelSpace == nBufDataSize &&
408 3877860 : nLineSpace == nPixelSpace * nBufXSize &&
409 6 : CPLTestBool(CPLGetConfigOption("GDAL_NO_COSTLY_OVERVIEW", "NO")))
410 : {
411 0 : memset(pData, 0, static_cast<size_t>(nLineSpace * nBufYSize));
412 0 : return CE_None;
413 : }
414 :
415 : /* ==================================================================== */
416 : /* The second case when we don't need subsample data but likely */
417 : /* need data type conversion. */
418 : /* ==================================================================== */
419 3033290 : if ( // nPixelSpace == nBufDataSize &&
420 3033290 : nXSize == nBufXSize && nYSize == nBufYSize && bUseIntegerRequestCoords)
421 : {
422 : #if DEBUG_VERBOSE
423 : printf("IRasterIO(%d,%d,%d,%d) rw=%d case 2\n", /*ok*/
424 : nXOff, nYOff, nXSize, nYSize, static_cast<int>(eRWFlag));
425 : #endif
426 :
427 : /* --------------------------------------------------------------------
428 : */
429 : /* Loop over buffer computing source locations. */
430 : /* --------------------------------------------------------------------
431 : */
432 : // Calculate starting values out of loop
433 2468070 : const int nLBlockXStart = nXOff / nBlockXSize;
434 2468070 : const int nXSpanEnd = nBufXSize + nXOff;
435 :
436 2468070 : int nYInc = 0;
437 4975040 : for (int iBufYOff = 0, iSrcY = nYOff; iBufYOff < nBufYSize;
438 2506980 : iBufYOff += nYInc, iSrcY += nYInc)
439 : {
440 2507040 : GPtrDiff_t iBufOffset = static_cast<GPtrDiff_t>(iBufYOff) *
441 : static_cast<GPtrDiff_t>(nLineSpace);
442 2507040 : int nLBlockY = iSrcY / nBlockYSize;
443 2507040 : int nLBlockX = nLBlockXStart;
444 2507040 : int iSrcX = nXOff;
445 5233160 : while (iSrcX < nXSpanEnd)
446 : {
447 2726190 : int nXSpan = nLBlockX * nBlockXSize;
448 2726190 : if (nXSpan < INT_MAX - nBlockXSize)
449 2726170 : nXSpan += nBlockXSize;
450 : else
451 14 : nXSpan = INT_MAX;
452 2726190 : const int nXRight = nXSpan;
453 2726190 : nXSpan = (nXSpan < nXSpanEnd ? nXSpan : nXSpanEnd) - iSrcX;
454 :
455 : const size_t nXSpanSize =
456 2726190 : CPLUnsanitizedMul(nXSpan, static_cast<size_t>(nPixelSpace));
457 :
458 2726180 : bool bJustInitialize =
459 2042150 : eRWFlag == GF_Write && nYOff <= nLBlockY * nBlockYSize &&
460 37240 : nYOff + nYSize - nBlockYSize >= nLBlockY * nBlockYSize &&
461 4793900 : nXOff <= nLBlockX * nBlockXSize &&
462 25575 : nXOff + nXSize >= nXRight;
463 :
464 : // Is this a partial tile at right and/or bottom edges of
465 : // the raster, and that is going to be completely written?
466 : // If so, do not load it from storage, but zero it so that
467 : // the content outsize of the validity area is initialized.
468 2726180 : bool bMemZeroBuffer = false;
469 2042150 : if (eRWFlag == GF_Write && !bJustInitialize &&
470 2017820 : nXOff <= nLBlockX * nBlockXSize &&
471 2016190 : nYOff <= nLBlockY * nBlockYSize &&
472 12145 : (nXOff + nXSize >= nXRight ||
473 : // cppcheck-suppress knownConditionTrueFalse
474 4771040 : (nXOff + nXSize == GetXSize() && nXRight > GetXSize())) &&
475 11965 : (nYOff + nYSize - nBlockYSize >= nLBlockY * nBlockYSize ||
476 10743 : (nYOff + nYSize == GetYSize() &&
477 1955 : nLBlockY * nBlockYSize > GetYSize() - nBlockYSize)))
478 : {
479 3177 : bJustInitialize = true;
480 3177 : bMemZeroBuffer = true;
481 : }
482 :
483 : /* --------------------------------------------------------------------
484 : */
485 : /* Ensure we have the appropriate block loaded. */
486 : /* --------------------------------------------------------------------
487 : */
488 2726180 : const GUInt32 nErrorCounter = CPLGetErrorCounter();
489 2726200 : poBlock =
490 2726160 : GetLockedBlockRef(nLBlockX, nLBlockY, bJustInitialize);
491 2726200 : if (!poBlock)
492 : {
493 71 : if (strstr(CPLGetLastErrorMsg(), "IReadBlock failed") ==
494 : nullptr)
495 : {
496 0 : CPLError(CE_Failure, CPLE_AppDefined,
497 : "GetBlockRef failed at X block offset %d, "
498 : "Y block offset %d%s",
499 : nLBlockX, nLBlockY,
500 0 : (nErrorCounter != CPLGetErrorCounter())
501 0 : ? CPLSPrintf(": %s", CPLGetLastErrorMsg())
502 : : "");
503 : }
504 71 : return (CE_Failure);
505 : }
506 :
507 2726130 : if (eRWFlag == GF_Write)
508 2042160 : poBlock->MarkDirty();
509 :
510 2726130 : pabySrcBlock = static_cast<GByte *>(poBlock->GetDataRef());
511 2726120 : if (bMemZeroBuffer)
512 : {
513 3177 : memset(pabySrcBlock, 0,
514 3177 : static_cast<GPtrDiff_t>(nBandDataSize) *
515 3177 : nBlockXSize * nBlockYSize);
516 : }
517 : /* --------------------------------------------------------------------
518 : */
519 : /* Copy over this chunk of data. */
520 : /* --------------------------------------------------------------------
521 : */
522 2726120 : GPtrDiff_t iSrcOffset =
523 2726120 : (static_cast<GPtrDiff_t>(iSrcX) -
524 2726120 : static_cast<GPtrDiff_t>(nLBlockX * nBlockXSize) +
525 2726120 : (static_cast<GPtrDiff_t>(iSrcY) -
526 2726120 : static_cast<GPtrDiff_t>(nLBlockY) * nBlockYSize) *
527 2726120 : nBlockXSize) *
528 2726120 : nBandDataSize;
529 : // Fill up as many rows as possible for the loaded block.
530 5452260 : const int kmax = std::min(nBlockYSize - (iSrcY % nBlockYSize),
531 2726120 : nBufYSize - iBufYOff);
532 58971400 : for (int k = 0; k < kmax; k++)
533 : {
534 56245300 : if (eDataType == eBufType && nPixelSpace == nBufDataSize)
535 : {
536 52288400 : if (eRWFlag == GF_Read)
537 47841600 : memcpy(static_cast<GByte *>(pData) + iBufOffset +
538 47841600 : static_cast<GPtrDiff_t>(k) * nLineSpace,
539 47841600 : pabySrcBlock + iSrcOffset, nXSpanSize);
540 : else
541 4446820 : memcpy(pabySrcBlock + iSrcOffset,
542 4446820 : static_cast<GByte *>(pData) + iBufOffset +
543 4446820 : static_cast<GPtrDiff_t>(k) * nLineSpace,
544 : nXSpanSize);
545 : }
546 : else
547 : {
548 : /* type to type conversion */
549 3956940 : if (eRWFlag == GF_Read)
550 3898020 : GDALCopyWords64(
551 3898020 : pabySrcBlock + iSrcOffset, eDataType,
552 : nBandDataSize,
553 3898020 : static_cast<GByte *>(pData) + iBufOffset +
554 3898020 : static_cast<GPtrDiff_t>(k) * nLineSpace,
555 : eBufType, static_cast<int>(nPixelSpace),
556 : nXSpan);
557 : else
558 58916 : GDALCopyWords64(
559 58916 : static_cast<GByte *>(pData) + iBufOffset +
560 58916 : static_cast<GPtrDiff_t>(k) * nLineSpace,
561 : eBufType, static_cast<int>(nPixelSpace),
562 58916 : pabySrcBlock + iSrcOffset, eDataType,
563 : nBandDataSize, nXSpan);
564 : }
565 :
566 56245300 : iSrcOffset +=
567 56245300 : static_cast<GPtrDiff_t>(nBlockXSize) * nBandDataSize;
568 : }
569 :
570 : iBufOffset =
571 2726090 : CPLUnsanitizedAdd<GPtrDiff_t>(iBufOffset, nXSpanSize);
572 2726100 : nLBlockX++;
573 2726100 : iSrcX += nXSpan;
574 :
575 2726100 : poBlock->DropLock();
576 2726120 : poBlock = nullptr;
577 : }
578 :
579 : /* Compute the increment to go on a block boundary */
580 2506980 : nYInc = nBlockYSize - (iSrcY % nBlockYSize);
581 :
582 2508830 : if (psExtraArg->pfnProgress != nullptr &&
583 1855 : !psExtraArg->pfnProgress(
584 2508830 : 1.0 * std::min(nBufYSize, iBufYOff + nYInc) / nBufYSize, "",
585 : psExtraArg->pProgressData))
586 : {
587 1 : return CE_Failure;
588 : }
589 : }
590 :
591 2468000 : return CE_None;
592 : }
593 :
594 : /* ==================================================================== */
595 : /* Loop reading required source blocks to satisfy output */
596 : /* request. This is the most general implementation. */
597 : /* ==================================================================== */
598 :
599 565225 : double dfXOff = nXOff;
600 565225 : double dfYOff = nYOff;
601 565225 : double dfXSize = nXSize;
602 565225 : double dfYSize = nYSize;
603 565225 : if (psExtraArg->bFloatingPointWindowValidity)
604 : {
605 230510 : dfXOff = psExtraArg->dfXOff;
606 230510 : dfYOff = psExtraArg->dfYOff;
607 230510 : dfXSize = psExtraArg->dfXSize;
608 230510 : dfYSize = psExtraArg->dfYSize;
609 : }
610 :
611 : /* -------------------------------------------------------------------- */
612 : /* Compute stepping increment. */
613 : /* -------------------------------------------------------------------- */
614 565225 : const double dfSrcXInc = dfXSize / static_cast<double>(nBufXSize);
615 565225 : const double dfSrcYInc = dfYSize / static_cast<double>(nBufYSize);
616 565225 : CPLErr eErr = CE_None;
617 :
618 565225 : if (eRWFlag == GF_Write)
619 : {
620 : /* --------------------------------------------------------------------
621 : */
622 : /* Write case */
623 : /* Loop over raster window computing source locations in the buffer.
624 : */
625 : /* --------------------------------------------------------------------
626 : */
627 166651 : GByte *pabyDstBlock = nullptr;
628 166651 : int nLBlockX = -1;
629 166651 : int nLBlockY = -1;
630 :
631 1259990 : for (int iDstY = nYOff; iDstY < nYOff + nYSize; iDstY++)
632 : {
633 1093340 : const int iBufYOff = static_cast<int>((iDstY - nYOff) / dfSrcYInc);
634 :
635 12384000 : for (int iDstX = nXOff; iDstX < nXOff + nXSize; iDstX++)
636 : {
637 11290600 : const int iBufXOff =
638 11290600 : static_cast<int>((iDstX - nXOff) / dfSrcXInc);
639 11290600 : GPtrDiff_t iBufOffset =
640 11290600 : static_cast<GPtrDiff_t>(iBufYOff) *
641 : static_cast<GPtrDiff_t>(nLineSpace) +
642 11290600 : iBufXOff * static_cast<GPtrDiff_t>(nPixelSpace);
643 :
644 : // FIXME: this code likely doesn't work if the dirty block gets
645 : // flushed to disk before being completely written.
646 : // In the meantime, bJustInitialize should probably be set to
647 : // FALSE even if it is not ideal performance wise, and for
648 : // lossy compression.
649 :
650 : /* --------------------------------------------------------------------
651 : */
652 : /* Ensure we have the appropriate block loaded. */
653 : /* --------------------------------------------------------------------
654 : */
655 11290600 : if (iDstX < nLBlockX * nBlockXSize ||
656 11041300 : iDstX - nBlockXSize >= nLBlockX * nBlockXSize ||
657 10584600 : iDstY < nLBlockY * nBlockYSize ||
658 10584600 : iDstY - nBlockYSize >= nLBlockY * nBlockYSize)
659 : {
660 738682 : nLBlockX = iDstX / nBlockXSize;
661 738682 : nLBlockY = iDstY / nBlockYSize;
662 :
663 738682 : const bool bJustInitialize =
664 1065950 : nYOff <= nLBlockY * nBlockYSize &&
665 327271 : nYOff + nYSize - nBlockYSize >=
666 327271 : nLBlockY * nBlockYSize &&
667 1116260 : nXOff <= nLBlockX * nBlockXSize &&
668 50305 : nXOff + nXSize - nBlockXSize >= nLBlockX * nBlockXSize;
669 : /*bool bMemZeroBuffer = FALSE;
670 : if( !bJustInitialize &&
671 : nXOff <= nLBlockX * nBlockXSize &&
672 : nYOff <= nLBlockY * nBlockYSize &&
673 : (nXOff + nXSize >= (nLBlockX+1) * nBlockXSize ||
674 : (nXOff + nXSize == GetXSize() &&
675 : (nLBlockX+1) * nBlockXSize > GetXSize())) &&
676 : (nYOff + nYSize >= (nLBlockY+1) * nBlockYSize ||
677 : (nYOff + nYSize == GetYSize() &&
678 : (nLBlockY+1) * nBlockYSize > GetYSize())) )
679 : {
680 : bJustInitialize = TRUE;
681 : bMemZeroBuffer = TRUE;
682 : }*/
683 738682 : if (poBlock != nullptr)
684 572031 : poBlock->DropLock();
685 :
686 738682 : poBlock =
687 738682 : GetLockedBlockRef(nLBlockX, nLBlockY, bJustInitialize);
688 738682 : if (poBlock == nullptr)
689 : {
690 0 : return (CE_Failure);
691 : }
692 :
693 738682 : poBlock->MarkDirty();
694 :
695 738682 : pabyDstBlock = static_cast<GByte *>(poBlock->GetDataRef());
696 : /*if( bMemZeroBuffer )
697 : {
698 : memset(pabyDstBlock, 0,
699 : static_cast<GPtrDiff_t>(nBandDataSize) * nBlockXSize
700 : * nBlockYSize);
701 : }*/
702 : }
703 :
704 : // To make Coverity happy. Should not happen by design.
705 11290600 : if (pabyDstBlock == nullptr)
706 : {
707 0 : CPLAssert(false);
708 : eErr = CE_Failure;
709 : break;
710 : }
711 :
712 : /* --------------------------------------------------------------------
713 : */
714 : /* Copy over this pixel of data. */
715 : /* --------------------------------------------------------------------
716 : */
717 11290600 : GPtrDiff_t iDstOffset =
718 11290600 : (static_cast<GPtrDiff_t>(iDstX) -
719 11290600 : static_cast<GPtrDiff_t>(nLBlockX) * nBlockXSize +
720 11290600 : (static_cast<GPtrDiff_t>(iDstY) -
721 11290600 : static_cast<GPtrDiff_t>(nLBlockY) * nBlockYSize) *
722 11290600 : nBlockXSize) *
723 11290600 : nBandDataSize;
724 :
725 11290600 : if (eDataType == eBufType)
726 : {
727 11287500 : memcpy(pabyDstBlock + iDstOffset,
728 11287500 : static_cast<GByte *>(pData) + iBufOffset,
729 : nBandDataSize);
730 : }
731 : else
732 : {
733 : /* type to type conversion ... ouch, this is expensive way
734 : of handling single words */
735 3096 : GDALCopyWords64(static_cast<GByte *>(pData) + iBufOffset,
736 3096 : eBufType, 0, pabyDstBlock + iDstOffset,
737 : eDataType, 0, 1);
738 : }
739 : }
740 :
741 1093340 : if (psExtraArg->pfnProgress != nullptr &&
742 0 : !psExtraArg->pfnProgress(1.0 * (iDstY - nYOff + 1) / nYSize, "",
743 : psExtraArg->pProgressData))
744 : {
745 0 : eErr = CE_Failure;
746 0 : break;
747 : }
748 : }
749 : }
750 : else
751 : {
752 398574 : if (psExtraArg->eResampleAlg != GRIORA_NearestNeighbour)
753 : {
754 8687 : if ((psExtraArg->eResampleAlg == GRIORA_Cubic ||
755 2513 : psExtraArg->eResampleAlg == GRIORA_CubicSpline ||
756 2511 : psExtraArg->eResampleAlg == GRIORA_Bilinear ||
757 6179 : psExtraArg->eResampleAlg == GRIORA_Lanczos) &&
758 2968 : GetColorTable() != nullptr)
759 : {
760 0 : CPLError(CE_Warning, CPLE_NotSupported,
761 : "Resampling method not supported on paletted band. "
762 : "Falling back to nearest neighbour");
763 : }
764 3090 : else if (psExtraArg->eResampleAlg == GRIORA_Gauss &&
765 3 : GDALDataTypeIsComplex(eDataType))
766 : {
767 0 : CPLError(CE_Warning, CPLE_NotSupported,
768 : "Resampling method not supported on complex data type "
769 : "band. Falling back to nearest neighbour");
770 : }
771 : else
772 : {
773 3087 : return RasterIOResampled(eRWFlag, nXOff, nYOff, nXSize, nYSize,
774 : pData, nBufXSize, nBufYSize, eBufType,
775 3087 : nPixelSpace, nLineSpace, psExtraArg);
776 : }
777 : }
778 :
779 395483 : int nLimitBlockY = 0;
780 395483 : const bool bByteCopy = eDataType == eBufType && nBandDataSize == 1;
781 395483 : int nStartBlockX = -nBlockXSize;
782 395483 : const double EPS = 1e-10;
783 395483 : int nLBlockY = -1;
784 395483 : const double dfSrcXStart = 0.5 * dfSrcXInc + dfXOff + EPS;
785 395483 : const bool bIntegerXFactor =
786 372806 : bUseIntegerRequestCoords &&
787 669271 : static_cast<int>(dfSrcXInc) == dfSrcXInc &&
788 273788 : static_cast<int>(dfSrcXInc) < INT_MAX / nBandDataSize;
789 :
790 : /* --------------------------------------------------------------------
791 : */
792 : /* Read case */
793 : /* Loop over buffer computing source locations. */
794 : /* --------------------------------------------------------------------
795 : */
796 2451410 : for (int iBufYOff = 0; iBufYOff < nBufYSize; iBufYOff++)
797 : {
798 : // Add small epsilon to avoid some numeric precision issues.
799 2055940 : const double dfSrcY = (iBufYOff + 0.5) * dfSrcYInc + dfYOff + EPS;
800 2055940 : const int iSrcY = static_cast<int>(std::min(
801 2055940 : std::max(0.0, dfSrcY), static_cast<double>(nRasterYSize - 1)));
802 :
803 2055940 : GPtrDiff_t iBufOffset = static_cast<GPtrDiff_t>(iBufYOff) *
804 : static_cast<GPtrDiff_t>(nLineSpace);
805 :
806 2055940 : if (iSrcY >= nLimitBlockY)
807 : {
808 433624 : nLBlockY = iSrcY / nBlockYSize;
809 433624 : nLimitBlockY = nLBlockY * nBlockYSize;
810 433624 : if (nLimitBlockY < INT_MAX - nBlockYSize)
811 433624 : nLimitBlockY += nBlockYSize;
812 : else
813 0 : nLimitBlockY = INT_MAX;
814 : // Make sure a new block is loaded.
815 433624 : nStartBlockX = -nBlockXSize;
816 : }
817 1622320 : else if (static_cast<int>(dfSrcXStart) < nStartBlockX)
818 : {
819 : // Make sure a new block is loaded.
820 441987 : nStartBlockX = -nBlockXSize;
821 : }
822 :
823 2055940 : GPtrDiff_t iSrcOffsetCst = (iSrcY - nLBlockY * nBlockYSize) *
824 2055940 : static_cast<GPtrDiff_t>(nBlockXSize);
825 :
826 2055940 : if (bIntegerXFactor)
827 : {
828 695677 : int iSrcX = static_cast<int>(dfSrcXStart);
829 695677 : const int nSrcXInc = static_cast<int>(dfSrcXInc);
830 695677 : GByte *pabyDstData = static_cast<GByte *>(pData) + iBufOffset;
831 695677 : bool bRet = false;
832 695677 : if (bByteCopy)
833 : {
834 585768 : bRet = DownsamplingIntegerXFactor<true, 1>(
835 : this, iSrcX, nSrcXInc, iSrcOffsetCst, pabyDstData,
836 : static_cast<int>(nPixelSpace), nBufXSize, GDT_Byte,
837 : GDT_Byte, nStartBlockX, nBlockXSize, poBlock, nLBlockY);
838 : }
839 109909 : else if (eDataType == eBufType)
840 : {
841 109704 : switch (nBandDataSize)
842 : {
843 109624 : case 2:
844 109624 : bRet = DownsamplingIntegerXFactor<true, 2>(
845 : this, iSrcX, nSrcXInc, iSrcOffsetCst,
846 : pabyDstData, static_cast<int>(nPixelSpace),
847 : nBufXSize, eDataType, eDataType, nStartBlockX,
848 : nBlockXSize, poBlock, nLBlockY);
849 109624 : break;
850 22 : case 4:
851 22 : bRet = DownsamplingIntegerXFactor<true, 4>(
852 : this, iSrcX, nSrcXInc, iSrcOffsetCst,
853 : pabyDstData, static_cast<int>(nPixelSpace),
854 : nBufXSize, eDataType, eDataType, nStartBlockX,
855 : nBlockXSize, poBlock, nLBlockY);
856 22 : break;
857 56 : case 8:
858 56 : bRet = DownsamplingIntegerXFactor<true, 8>(
859 : this, iSrcX, nSrcXInc, iSrcOffsetCst,
860 : pabyDstData, static_cast<int>(nPixelSpace),
861 : nBufXSize, eDataType, eDataType, nStartBlockX,
862 : nBlockXSize, poBlock, nLBlockY);
863 56 : break;
864 2 : case 16:
865 2 : bRet = DownsamplingIntegerXFactor<true, 16>(
866 : this, iSrcX, nSrcXInc, iSrcOffsetCst,
867 : pabyDstData, static_cast<int>(nPixelSpace),
868 : nBufXSize, eDataType, eDataType, nStartBlockX,
869 : nBlockXSize, poBlock, nLBlockY);
870 2 : break;
871 0 : default:
872 0 : CPLAssert(false);
873 : break;
874 : }
875 : }
876 : else
877 : {
878 205 : bRet = DownsamplingIntegerXFactor<false, 0>(
879 : this, iSrcX, nSrcXInc, iSrcOffsetCst, pabyDstData,
880 : static_cast<int>(nPixelSpace), nBufXSize, eDataType,
881 : eBufType, nStartBlockX, nBlockXSize, poBlock, nLBlockY);
882 : }
883 695677 : if (!bRet)
884 1 : eErr = CE_Failure;
885 : }
886 : else
887 : {
888 1360260 : double dfSrcX = dfSrcXStart;
889 582293000 : for (int iBufXOff = 0; iBufXOff < nBufXSize;
890 580933000 : iBufXOff++, dfSrcX += dfSrcXInc)
891 : {
892 : // TODO?: try to avoid the clamping for most iterations
893 : const int iSrcX = static_cast<int>(
894 1161870000 : std::min(std::max(0.0, dfSrcX),
895 580933000 : static_cast<double>(nRasterXSize - 1)));
896 :
897 : /* --------------------------------------------------------------------
898 : */
899 : /* Ensure we have the appropriate block loaded. */
900 : /* --------------------------------------------------------------------
901 : */
902 580933000 : if (iSrcX >= nBlockXSize + nStartBlockX)
903 : {
904 1702800 : const int nLBlockX = iSrcX / nBlockXSize;
905 1702800 : nStartBlockX = nLBlockX * nBlockXSize;
906 :
907 1702800 : if (poBlock != nullptr)
908 1581100 : poBlock->DropLock();
909 :
910 1702800 : poBlock = GetLockedBlockRef(nLBlockX, nLBlockY, FALSE);
911 1702800 : if (poBlock == nullptr)
912 : {
913 9 : eErr = CE_Failure;
914 9 : break;
915 : }
916 :
917 : pabySrcBlock =
918 1702790 : static_cast<GByte *>(poBlock->GetDataRef());
919 : }
920 580933000 : const GPtrDiff_t nDiffX =
921 580933000 : static_cast<GPtrDiff_t>(iSrcX - nStartBlockX);
922 :
923 : /* --------------------------------------------------------------------
924 : */
925 : /* Copy over this pixel of data. */
926 : /* --------------------------------------------------------------------
927 : */
928 :
929 580933000 : if (bByteCopy)
930 : {
931 527231000 : GPtrDiff_t iSrcOffset = nDiffX + iSrcOffsetCst;
932 527231000 : static_cast<GByte *>(pData)[iBufOffset] =
933 527231000 : pabySrcBlock[iSrcOffset];
934 : }
935 53701600 : else if (eDataType == eBufType)
936 : {
937 48225600 : GPtrDiff_t iSrcOffset =
938 48225600 : (nDiffX + iSrcOffsetCst) * nBandDataSize;
939 48225600 : memcpy(static_cast<GByte *>(pData) + iBufOffset,
940 48225600 : pabySrcBlock + iSrcOffset, nBandDataSize);
941 : }
942 : else
943 : {
944 : // Type to type conversion ...
945 5476050 : GPtrDiff_t iSrcOffset =
946 5476050 : (nDiffX + iSrcOffsetCst) * nBandDataSize;
947 5476050 : GDALCopyWords64(pabySrcBlock + iSrcOffset, eDataType, 0,
948 : static_cast<GByte *>(pData) +
949 5476050 : iBufOffset,
950 : eBufType, 0, 1);
951 : }
952 :
953 580933000 : iBufOffset += static_cast<int>(nPixelSpace);
954 : }
955 : }
956 2055940 : if (eErr == CE_Failure)
957 11 : break;
958 :
959 2287020 : if (psExtraArg->pfnProgress != nullptr &&
960 231086 : !psExtraArg->pfnProgress(1.0 * (iBufYOff + 1) / nBufYSize, "",
961 : psExtraArg->pProgressData))
962 : {
963 1 : eErr = CE_Failure;
964 1 : break;
965 : }
966 : }
967 : }
968 :
969 562134 : if (poBlock != nullptr)
970 562124 : poBlock->DropLock();
971 :
972 562134 : return eErr;
973 : }
974 :
975 : /************************************************************************/
976 : /* GDALRasterIOTransformer() */
977 : /************************************************************************/
978 :
979 : struct GDALRasterIOTransformerStruct
980 : {
981 : double dfXOff;
982 : double dfYOff;
983 : double dfXRatioDstToSrc;
984 : double dfYRatioDstToSrc;
985 : };
986 :
987 6748 : static int GDALRasterIOTransformer(void *pTransformerArg, int bDstToSrc,
988 : int nPointCount, double *x, double *y,
989 : double * /* z */, int *panSuccess)
990 : {
991 6748 : GDALRasterIOTransformerStruct *psParams =
992 : static_cast<GDALRasterIOTransformerStruct *>(pTransformerArg);
993 6748 : if (bDstToSrc)
994 : {
995 252996 : for (int i = 0; i < nPointCount; i++)
996 : {
997 246836 : x[i] = x[i] * psParams->dfXRatioDstToSrc + psParams->dfXOff;
998 246836 : y[i] = y[i] * psParams->dfYRatioDstToSrc + psParams->dfYOff;
999 246836 : panSuccess[i] = TRUE;
1000 : }
1001 : }
1002 : else
1003 : {
1004 1176 : for (int i = 0; i < nPointCount; i++)
1005 : {
1006 588 : x[i] = (x[i] - psParams->dfXOff) / psParams->dfXRatioDstToSrc;
1007 588 : y[i] = (y[i] - psParams->dfYOff) / psParams->dfYRatioDstToSrc;
1008 588 : panSuccess[i] = TRUE;
1009 : }
1010 : }
1011 6748 : return TRUE;
1012 : }
1013 :
1014 : /************************************************************************/
1015 : /* RasterIOResampled() */
1016 : /************************************************************************/
1017 :
1018 : //! @cond Doxygen_Suppress
1019 3087 : CPLErr GDALRasterBand::RasterIOResampled(
1020 : GDALRWFlag /* eRWFlag */, int nXOff, int nYOff, int nXSize, int nYSize,
1021 : void *pData, int nBufXSize, int nBufYSize, GDALDataType eBufType,
1022 : GSpacing nPixelSpace, GSpacing nLineSpace, GDALRasterIOExtraArg *psExtraArg)
1023 : {
1024 : // Determine if we use warping resampling or overview resampling
1025 : const bool bUseWarp =
1026 3087 : (GDALDataTypeIsComplex(eDataType) &&
1027 3244 : psExtraArg->eResampleAlg != GRIORA_NearestNeighbour &&
1028 157 : psExtraArg->eResampleAlg != GRIORA_Mode);
1029 :
1030 3087 : double dfXOff = nXOff;
1031 3087 : double dfYOff = nYOff;
1032 3087 : double dfXSize = nXSize;
1033 3087 : double dfYSize = nYSize;
1034 3087 : if (psExtraArg->bFloatingPointWindowValidity)
1035 : {
1036 2628 : dfXOff = psExtraArg->dfXOff;
1037 2628 : dfYOff = psExtraArg->dfYOff;
1038 2628 : dfXSize = psExtraArg->dfXSize;
1039 2628 : dfYSize = psExtraArg->dfYSize;
1040 : }
1041 :
1042 3087 : const double dfXRatioDstToSrc = dfXSize / nBufXSize;
1043 3087 : const double dfYRatioDstToSrc = dfYSize / nBufYSize;
1044 :
1045 : // Determine the coordinates in the "virtual" output raster to see
1046 : // if there are not integers, in which case we will use them as a shift
1047 : // so that subwindow extracts give the exact same results as entire raster
1048 : // scaling.
1049 3087 : double dfDestXOff = dfXOff / dfXRatioDstToSrc;
1050 3087 : bool bHasXOffVirtual = false;
1051 3087 : int nDestXOffVirtual = 0;
1052 3087 : if (fabs(dfDestXOff - static_cast<int>(dfDestXOff + 0.5)) < 1e-8)
1053 : {
1054 2759 : bHasXOffVirtual = true;
1055 2759 : dfXOff = nXOff;
1056 2759 : nDestXOffVirtual = static_cast<int>(dfDestXOff + 0.5);
1057 : }
1058 :
1059 3087 : double dfDestYOff = dfYOff / dfYRatioDstToSrc;
1060 3087 : bool bHasYOffVirtual = false;
1061 3087 : int nDestYOffVirtual = 0;
1062 3087 : if (fabs(dfDestYOff - static_cast<int>(dfDestYOff + 0.5)) < 1e-8)
1063 : {
1064 2755 : bHasYOffVirtual = true;
1065 2755 : dfYOff = nYOff;
1066 2755 : nDestYOffVirtual = static_cast<int>(dfDestYOff + 0.5);
1067 : }
1068 :
1069 : // Create a MEM dataset that wraps the output buffer.
1070 : GDALDataset *poMEMDS;
1071 3087 : void *pTempBuffer = nullptr;
1072 3087 : GSpacing nPSMem = nPixelSpace;
1073 3087 : GSpacing nLSMem = nLineSpace;
1074 3087 : void *pDataMem = pData;
1075 3087 : GDALDataType eDTMem = eBufType;
1076 3087 : if (eBufType != eDataType)
1077 : {
1078 40 : nPSMem = GDALGetDataTypeSizeBytes(eDataType);
1079 40 : nLSMem = nPSMem * nBufXSize;
1080 : pTempBuffer =
1081 40 : VSI_MALLOC2_VERBOSE(nBufYSize, static_cast<size_t>(nLSMem));
1082 40 : if (pTempBuffer == nullptr)
1083 0 : return CE_Failure;
1084 40 : pDataMem = pTempBuffer;
1085 40 : eDTMem = eDataType;
1086 : }
1087 :
1088 : poMEMDS =
1089 3087 : MEMDataset::Create("", nDestXOffVirtual + nBufXSize,
1090 : nDestYOffVirtual + nBufYSize, 0, eDTMem, nullptr);
1091 3087 : GByte *pabyData = static_cast<GByte *>(pDataMem) -
1092 3087 : nPSMem * nDestXOffVirtual - nLSMem * nDestYOffVirtual;
1093 3087 : GDALRasterBandH hMEMBand = MEMCreateRasterBandEx(
1094 : poMEMDS, 1, pabyData, eDTMem, nPSMem, nLSMem, false);
1095 3087 : poMEMDS->SetBand(1, GDALRasterBand::FromHandle(hMEMBand));
1096 :
1097 3087 : const char *pszNBITS = GetMetadataItem("NBITS", "IMAGE_STRUCTURE");
1098 3087 : const int nNBITS = pszNBITS ? atoi(pszNBITS) : 0;
1099 3087 : if (pszNBITS)
1100 6 : GDALRasterBand::FromHandle(hMEMBand)->SetMetadataItem(
1101 6 : "NBITS", pszNBITS, "IMAGE_STRUCTURE");
1102 :
1103 3087 : CPLErr eErr = CE_None;
1104 :
1105 : // Do the resampling.
1106 3087 : if (bUseWarp)
1107 : {
1108 149 : int bHasNoData = FALSE;
1109 149 : double dfNoDataValue = GetNoDataValue(&bHasNoData);
1110 :
1111 149 : VRTDatasetH hVRTDS = nullptr;
1112 149 : GDALRasterBandH hVRTBand = nullptr;
1113 149 : if (GetDataset() == nullptr)
1114 : {
1115 : /* Create VRT dataset that wraps the whole dataset */
1116 0 : hVRTDS = VRTCreate(nRasterXSize, nRasterYSize);
1117 0 : VRTAddBand(hVRTDS, eDataType, nullptr);
1118 0 : hVRTBand = GDALGetRasterBand(hVRTDS, 1);
1119 0 : VRTAddSimpleSource(hVRTBand, this, 0, 0, nRasterXSize, nRasterYSize,
1120 : 0, 0, nRasterXSize, nRasterYSize, nullptr,
1121 : VRT_NODATA_UNSET);
1122 :
1123 : /* Add a mask band if needed */
1124 0 : if (GetMaskFlags() != GMF_ALL_VALID)
1125 : {
1126 0 : GDALDataset::FromHandle(hVRTDS)->CreateMaskBand(0);
1127 : VRTSourcedRasterBand *poVRTMaskBand =
1128 : reinterpret_cast<VRTSourcedRasterBand *>(
1129 : reinterpret_cast<GDALRasterBand *>(hVRTBand)
1130 0 : ->GetMaskBand());
1131 0 : poVRTMaskBand->AddMaskBandSource(this, 0, 0, nRasterXSize,
1132 0 : nRasterYSize, 0, 0,
1133 0 : nRasterXSize, nRasterYSize);
1134 : }
1135 : }
1136 :
1137 149 : GDALWarpOptions *psWarpOptions = GDALCreateWarpOptions();
1138 149 : switch (psExtraArg->eResampleAlg)
1139 : {
1140 0 : case GRIORA_NearestNeighbour:
1141 0 : psWarpOptions->eResampleAlg = GRA_NearestNeighbour;
1142 0 : break;
1143 147 : case GRIORA_Bilinear:
1144 147 : psWarpOptions->eResampleAlg = GRA_Bilinear;
1145 147 : break;
1146 0 : case GRIORA_Cubic:
1147 0 : psWarpOptions->eResampleAlg = GRA_Cubic;
1148 0 : break;
1149 0 : case GRIORA_CubicSpline:
1150 0 : psWarpOptions->eResampleAlg = GRA_CubicSpline;
1151 0 : break;
1152 0 : case GRIORA_Lanczos:
1153 0 : psWarpOptions->eResampleAlg = GRA_Lanczos;
1154 0 : break;
1155 0 : case GRIORA_Average:
1156 0 : psWarpOptions->eResampleAlg = GRA_Average;
1157 0 : break;
1158 2 : case GRIORA_RMS:
1159 2 : psWarpOptions->eResampleAlg = GRA_RMS;
1160 2 : break;
1161 0 : case GRIORA_Mode:
1162 0 : psWarpOptions->eResampleAlg = GRA_Mode;
1163 0 : break;
1164 0 : default:
1165 0 : CPLAssert(false);
1166 : psWarpOptions->eResampleAlg = GRA_NearestNeighbour;
1167 : break;
1168 : }
1169 149 : psWarpOptions->hSrcDS = hVRTDS ? hVRTDS : GetDataset();
1170 149 : psWarpOptions->hDstDS = poMEMDS;
1171 149 : psWarpOptions->nBandCount = 1;
1172 149 : int nSrcBandNumber = hVRTDS ? 1 : nBand;
1173 149 : int nDstBandNumber = 1;
1174 149 : psWarpOptions->panSrcBands = &nSrcBandNumber;
1175 149 : psWarpOptions->panDstBands = &nDstBandNumber;
1176 298 : psWarpOptions->pfnProgress = psExtraArg->pfnProgress
1177 149 : ? psExtraArg->pfnProgress
1178 : : GDALDummyProgress;
1179 149 : psWarpOptions->pProgressArg = psExtraArg->pProgressData;
1180 149 : psWarpOptions->pfnTransformer = GDALRasterIOTransformer;
1181 149 : if (bHasNoData)
1182 : {
1183 0 : psWarpOptions->papszWarpOptions = CSLSetNameValue(
1184 : psWarpOptions->papszWarpOptions, "INIT_DEST", "NO_DATA");
1185 0 : if (psWarpOptions->padfSrcNoDataReal == nullptr)
1186 : {
1187 0 : psWarpOptions->padfSrcNoDataReal =
1188 0 : static_cast<double *>(CPLMalloc(sizeof(double)));
1189 0 : psWarpOptions->padfSrcNoDataReal[0] = dfNoDataValue;
1190 : }
1191 :
1192 0 : if (psWarpOptions->padfDstNoDataReal == nullptr)
1193 : {
1194 0 : psWarpOptions->padfDstNoDataReal =
1195 0 : static_cast<double *>(CPLMalloc(sizeof(double)));
1196 0 : psWarpOptions->padfDstNoDataReal[0] = dfNoDataValue;
1197 : }
1198 : }
1199 :
1200 : GDALRasterIOTransformerStruct sTransformer;
1201 149 : sTransformer.dfXOff = bHasXOffVirtual ? 0 : dfXOff;
1202 149 : sTransformer.dfYOff = bHasYOffVirtual ? 0 : dfYOff;
1203 149 : sTransformer.dfXRatioDstToSrc = dfXRatioDstToSrc;
1204 149 : sTransformer.dfYRatioDstToSrc = dfYRatioDstToSrc;
1205 149 : psWarpOptions->pTransformerArg = &sTransformer;
1206 :
1207 : GDALWarpOperationH hWarpOperation =
1208 149 : GDALCreateWarpOperation(psWarpOptions);
1209 149 : eErr = GDALChunkAndWarpImage(hWarpOperation, nDestXOffVirtual,
1210 : nDestYOffVirtual, nBufXSize, nBufYSize);
1211 149 : GDALDestroyWarpOperation(hWarpOperation);
1212 :
1213 149 : psWarpOptions->panSrcBands = nullptr;
1214 149 : psWarpOptions->panDstBands = nullptr;
1215 149 : GDALDestroyWarpOptions(psWarpOptions);
1216 :
1217 149 : if (hVRTDS)
1218 0 : GDALClose(hVRTDS);
1219 : }
1220 : else
1221 : {
1222 2938 : const char *pszResampling =
1223 3636 : (psExtraArg->eResampleAlg == GRIORA_Bilinear) ? "BILINEAR"
1224 822 : : (psExtraArg->eResampleAlg == GRIORA_Cubic) ? "CUBIC"
1225 246 : : (psExtraArg->eResampleAlg == GRIORA_CubicSpline) ? "CUBICSPLINE"
1226 239 : : (psExtraArg->eResampleAlg == GRIORA_Lanczos) ? "LANCZOS"
1227 172 : : (psExtraArg->eResampleAlg == GRIORA_Average) ? "AVERAGE"
1228 95 : : (psExtraArg->eResampleAlg == GRIORA_RMS) ? "RMS"
1229 43 : : (psExtraArg->eResampleAlg == GRIORA_Mode) ? "MODE"
1230 3 : : (psExtraArg->eResampleAlg == GRIORA_Gauss) ? "GAUSS"
1231 : : "UNKNOWN";
1232 :
1233 2938 : int nKernelRadius = 0;
1234 : GDALResampleFunction pfnResampleFunc =
1235 2938 : GDALGetResampleFunction(pszResampling, &nKernelRadius);
1236 2938 : CPLAssert(pfnResampleFunc);
1237 : GDALDataType eWrkDataType =
1238 2938 : GDALGetOvrWorkDataType(pszResampling, eDataType);
1239 2938 : int nHasNoData = 0;
1240 2938 : double dfNoDataValue = GetNoDataValue(&nHasNoData);
1241 2938 : const bool bHasNoData = CPL_TO_BOOL(nHasNoData);
1242 2938 : if (!bHasNoData)
1243 2848 : dfNoDataValue = 0.0;
1244 :
1245 2938 : int nDstBlockXSize = nBufXSize;
1246 2938 : int nDstBlockYSize = nBufYSize;
1247 2938 : int nFullResXChunk = 0;
1248 2938 : int nFullResYChunk = 0;
1249 : while (true)
1250 : {
1251 2949 : nFullResXChunk =
1252 2949 : 3 + static_cast<int>(nDstBlockXSize * dfXRatioDstToSrc);
1253 2949 : nFullResYChunk =
1254 2949 : 3 + static_cast<int>(nDstBlockYSize * dfYRatioDstToSrc);
1255 2949 : if (nFullResXChunk > nRasterXSize)
1256 2666 : nFullResXChunk = nRasterXSize;
1257 2949 : if (nFullResYChunk > nRasterYSize)
1258 267 : nFullResYChunk = nRasterYSize;
1259 2949 : if ((nDstBlockXSize == 1 && nDstBlockYSize == 1) ||
1260 2895 : (static_cast<GIntBig>(nFullResXChunk) * nFullResYChunk <=
1261 : 1024 * 1024))
1262 : break;
1263 : // When operating on the full width of a raster whose block width is
1264 : // the raster width, prefer doing chunks in height.
1265 11 : if (nFullResXChunk >= nXSize && nXSize == nBlockXSize &&
1266 : nDstBlockYSize > 1)
1267 0 : nDstBlockYSize /= 2;
1268 : /* Otherwise cut the maximal dimension */
1269 11 : else if (nDstBlockXSize > 1 &&
1270 0 : (nFullResXChunk > nFullResYChunk || nDstBlockYSize == 1))
1271 11 : nDstBlockXSize /= 2;
1272 : else
1273 0 : nDstBlockYSize /= 2;
1274 : }
1275 :
1276 2938 : int nOvrXFactor = static_cast<int>(0.5 + dfXRatioDstToSrc);
1277 2938 : int nOvrYFactor = static_cast<int>(0.5 + dfYRatioDstToSrc);
1278 2938 : if (nOvrXFactor == 0)
1279 2029 : nOvrXFactor = 1;
1280 2938 : if (nOvrYFactor == 0)
1281 2028 : nOvrYFactor = 1;
1282 2938 : int nFullResXSizeQueried =
1283 2938 : nFullResXChunk + 2 * nKernelRadius * nOvrXFactor;
1284 2938 : int nFullResYSizeQueried =
1285 2938 : nFullResYChunk + 2 * nKernelRadius * nOvrYFactor;
1286 :
1287 2938 : if (nFullResXSizeQueried > nRasterXSize)
1288 2558 : nFullResXSizeQueried = nRasterXSize;
1289 2938 : if (nFullResYSizeQueried > nRasterYSize)
1290 156 : nFullResYSizeQueried = nRasterYSize;
1291 :
1292 : void *pChunk =
1293 2938 : VSI_MALLOC3_VERBOSE(GDALGetDataTypeSizeBytes(eWrkDataType),
1294 : nFullResXSizeQueried, nFullResYSizeQueried);
1295 2938 : GByte *pabyChunkNoDataMask = nullptr;
1296 :
1297 2938 : GDALRasterBand *poMaskBand = GetMaskBand();
1298 2938 : int l_nMaskFlags = GetMaskFlags();
1299 :
1300 2938 : bool bUseNoDataMask = ((l_nMaskFlags & GMF_ALL_VALID) == 0);
1301 2938 : if (bUseNoDataMask)
1302 : {
1303 158 : pabyChunkNoDataMask = static_cast<GByte *>(VSI_MALLOC2_VERBOSE(
1304 : nFullResXSizeQueried, nFullResYSizeQueried));
1305 : }
1306 2938 : if (pChunk == nullptr ||
1307 158 : (bUseNoDataMask && pabyChunkNoDataMask == nullptr))
1308 : {
1309 0 : GDALClose(poMEMDS);
1310 0 : CPLFree(pChunk);
1311 0 : CPLFree(pabyChunkNoDataMask);
1312 0 : VSIFree(pTempBuffer);
1313 0 : return CE_Failure;
1314 : }
1315 :
1316 2938 : const int nTotalBlocks = DIV_ROUND_UP(nBufXSize, nDstBlockXSize) *
1317 2938 : DIV_ROUND_UP(nBufYSize, nDstBlockYSize);
1318 2938 : int nBlocksDone = 0;
1319 :
1320 : int nDstYOff;
1321 5876 : for (nDstYOff = 0; nDstYOff < nBufYSize && eErr == CE_None;
1322 2938 : nDstYOff += nDstBlockYSize)
1323 : {
1324 : int nDstYCount;
1325 2938 : if (nDstYOff + nDstBlockYSize <= nBufYSize)
1326 2938 : nDstYCount = nDstBlockYSize;
1327 : else
1328 0 : nDstYCount = nBufYSize - nDstYOff;
1329 :
1330 2938 : int nChunkYOff =
1331 2938 : nYOff + static_cast<int>(nDstYOff * dfYRatioDstToSrc);
1332 2938 : int nChunkYOff2 = nYOff + 1 +
1333 2938 : static_cast<int>(ceil((nDstYOff + nDstYCount) *
1334 : dfYRatioDstToSrc));
1335 2938 : if (nChunkYOff2 > nRasterYSize)
1336 391 : nChunkYOff2 = nRasterYSize;
1337 2938 : int nYCount = nChunkYOff2 - nChunkYOff;
1338 2938 : CPLAssert(nYCount <= nFullResYChunk);
1339 :
1340 2938 : int nChunkYOffQueried = nChunkYOff - nKernelRadius * nOvrYFactor;
1341 2938 : int nChunkYSizeQueried = nYCount + 2 * nKernelRadius * nOvrYFactor;
1342 2938 : if (nChunkYOffQueried < 0)
1343 : {
1344 291 : nChunkYSizeQueried += nChunkYOffQueried;
1345 291 : nChunkYOffQueried = 0;
1346 : }
1347 2938 : if (nChunkYSizeQueried + nChunkYOffQueried > nRasterYSize)
1348 394 : nChunkYSizeQueried = nRasterYSize - nChunkYOffQueried;
1349 2938 : CPLAssert(nChunkYSizeQueried <= nFullResYSizeQueried);
1350 :
1351 2938 : int nDstXOff = 0;
1352 5876 : for (nDstXOff = 0; nDstXOff < nBufXSize && eErr == CE_None;
1353 2938 : nDstXOff += nDstBlockXSize)
1354 : {
1355 2938 : int nDstXCount = 0;
1356 2938 : if (nDstXOff + nDstBlockXSize <= nBufXSize)
1357 2938 : nDstXCount = nDstBlockXSize;
1358 : else
1359 0 : nDstXCount = nBufXSize - nDstXOff;
1360 :
1361 2938 : int nChunkXOff =
1362 2938 : nXOff + static_cast<int>(nDstXOff * dfXRatioDstToSrc);
1363 2938 : int nChunkXOff2 =
1364 2938 : nXOff + 1 +
1365 2938 : static_cast<int>(
1366 2938 : ceil((nDstXOff + nDstXCount) * dfXRatioDstToSrc));
1367 2938 : if (nChunkXOff2 > nRasterXSize)
1368 2691 : nChunkXOff2 = nRasterXSize;
1369 2938 : int nXCount = nChunkXOff2 - nChunkXOff;
1370 2938 : CPLAssert(nXCount <= nFullResXChunk);
1371 :
1372 2938 : int nChunkXOffQueried =
1373 2938 : nChunkXOff - nKernelRadius * nOvrXFactor;
1374 2938 : int nChunkXSizeQueried =
1375 2938 : nXCount + 2 * nKernelRadius * nOvrXFactor;
1376 2938 : if (nChunkXOffQueried < 0)
1377 : {
1378 2595 : nChunkXSizeQueried += nChunkXOffQueried;
1379 2595 : nChunkXOffQueried = 0;
1380 : }
1381 2938 : if (nChunkXSizeQueried + nChunkXOffQueried > nRasterXSize)
1382 2581 : nChunkXSizeQueried = nRasterXSize - nChunkXOffQueried;
1383 2938 : CPLAssert(nChunkXSizeQueried <= nFullResXSizeQueried);
1384 :
1385 : // Read the source buffers.
1386 2938 : eErr = RasterIO(GF_Read, nChunkXOffQueried, nChunkYOffQueried,
1387 : nChunkXSizeQueried, nChunkYSizeQueried, pChunk,
1388 : nChunkXSizeQueried, nChunkYSizeQueried,
1389 : eWrkDataType, 0, 0, nullptr);
1390 :
1391 2938 : bool bSkipResample = false;
1392 2938 : bool bNoDataMaskFullyOpaque = false;
1393 2938 : if (eErr == CE_None && bUseNoDataMask)
1394 : {
1395 158 : eErr = poMaskBand->RasterIO(
1396 : GF_Read, nChunkXOffQueried, nChunkYOffQueried,
1397 : nChunkXSizeQueried, nChunkYSizeQueried,
1398 : pabyChunkNoDataMask, nChunkXSizeQueried,
1399 : nChunkYSizeQueried, GDT_Byte, 0, 0, nullptr);
1400 :
1401 : /* Optimizations if mask if fully opaque or transparent */
1402 158 : int nPixels = nChunkXSizeQueried * nChunkYSizeQueried;
1403 158 : GByte bVal = pabyChunkNoDataMask[0];
1404 158 : int i = 1;
1405 3751650 : for (; i < nPixels; i++)
1406 : {
1407 3751590 : if (pabyChunkNoDataMask[i] != bVal)
1408 104 : break;
1409 : }
1410 158 : if (i == nPixels)
1411 : {
1412 54 : if (bVal == 0)
1413 : {
1414 712 : for (int j = 0; j < nDstYCount; j++)
1415 : {
1416 686 : GDALCopyWords64(&dfNoDataValue, GDT_Float64, 0,
1417 : static_cast<GByte *>(pDataMem) +
1418 686 : nLSMem * (j + nDstYOff) +
1419 686 : nDstXOff * nPSMem,
1420 : eDTMem,
1421 : static_cast<int>(nPSMem),
1422 : nDstXCount);
1423 : }
1424 26 : bSkipResample = true;
1425 : }
1426 : else
1427 : {
1428 28 : bNoDataMaskFullyOpaque = true;
1429 : }
1430 : }
1431 : }
1432 :
1433 2938 : if (!bSkipResample && eErr == CE_None)
1434 : {
1435 2909 : const bool bPropagateNoData = false;
1436 2909 : void *pDstBuffer = nullptr;
1437 2909 : GDALDataType eDstBufferDataType = GDT_Unknown;
1438 : GDALRasterBand *poMEMBand =
1439 2909 : GDALRasterBand::FromHandle(hMEMBand);
1440 2909 : GDALOverviewResampleArgs args;
1441 2909 : args.eSrcDataType = eDataType;
1442 2909 : args.eOvrDataType = poMEMBand->GetRasterDataType();
1443 2909 : args.nOvrXSize = poMEMBand->GetXSize();
1444 2909 : args.nOvrYSize = poMEMBand->GetYSize();
1445 2909 : args.nOvrNBITS = nNBITS;
1446 2909 : args.dfXRatioDstToSrc = dfXRatioDstToSrc;
1447 2909 : args.dfYRatioDstToSrc = dfYRatioDstToSrc;
1448 2909 : args.dfSrcXDelta =
1449 2909 : dfXOff - nXOff; /* == 0 if bHasXOffVirtual */
1450 2909 : args.dfSrcYDelta =
1451 2909 : dfYOff - nYOff; /* == 0 if bHasYOffVirtual */
1452 2909 : args.eWrkDataType = eWrkDataType;
1453 2909 : args.pabyChunkNodataMask =
1454 2909 : bNoDataMaskFullyOpaque ? nullptr : pabyChunkNoDataMask;
1455 2909 : args.nChunkXOff =
1456 2909 : nChunkXOffQueried - (bHasXOffVirtual ? 0 : nXOff);
1457 2909 : args.nChunkXSize = nChunkXSizeQueried;
1458 2909 : args.nChunkYOff =
1459 2909 : nChunkYOffQueried - (bHasYOffVirtual ? 0 : nYOff);
1460 2909 : args.nChunkYSize = nChunkYSizeQueried;
1461 2909 : args.nDstXOff = nDstXOff + nDestXOffVirtual;
1462 2909 : args.nDstXOff2 = nDstXOff + nDestXOffVirtual + nDstXCount;
1463 2909 : args.nDstYOff = nDstYOff + nDestYOffVirtual;
1464 2909 : args.nDstYOff2 = nDstYOff + nDestYOffVirtual + nDstYCount;
1465 2909 : args.pszResampling = pszResampling;
1466 2909 : args.bHasNoData = bHasNoData;
1467 2909 : args.dfNoDataValue = dfNoDataValue;
1468 2909 : args.poColorTable = GetColorTable();
1469 2909 : args.bPropagateNoData = bPropagateNoData;
1470 2909 : eErr = pfnResampleFunc(args, pChunk, &pDstBuffer,
1471 : &eDstBufferDataType);
1472 2909 : if (eErr == CE_None)
1473 : {
1474 2909 : eErr = poMEMBand->RasterIO(
1475 : GF_Write, nDstXOff + nDestXOffVirtual,
1476 : nDstYOff + nDestYOffVirtual, nDstXCount, nDstYCount,
1477 : pDstBuffer, nDstXCount, nDstYCount,
1478 : eDstBufferDataType, 0, 0, nullptr);
1479 : }
1480 2909 : CPLFree(pDstBuffer);
1481 : }
1482 :
1483 2938 : nBlocksDone++;
1484 3363 : if (eErr == CE_None && psExtraArg->pfnProgress != nullptr &&
1485 425 : !psExtraArg->pfnProgress(1.0 * nBlocksDone / nTotalBlocks,
1486 : "", psExtraArg->pProgressData))
1487 : {
1488 1 : eErr = CE_Failure;
1489 : }
1490 : }
1491 : }
1492 :
1493 2938 : CPLFree(pChunk);
1494 2938 : CPLFree(pabyChunkNoDataMask);
1495 : }
1496 :
1497 3087 : if (eBufType != eDataType)
1498 : {
1499 40 : CPL_IGNORE_RET_VAL(poMEMDS->GetRasterBand(1)->RasterIO(
1500 : GF_Read, nDestXOffVirtual, nDestYOffVirtual, nBufXSize, nBufYSize,
1501 : pData, nBufXSize, nBufYSize, eBufType, nPixelSpace, nLineSpace,
1502 : nullptr));
1503 : }
1504 3087 : GDALClose(poMEMDS);
1505 3087 : VSIFree(pTempBuffer);
1506 :
1507 3087 : return eErr;
1508 : }
1509 :
1510 : /************************************************************************/
1511 : /* RasterIOResampled() */
1512 : /************************************************************************/
1513 :
1514 798 : CPLErr GDALDataset::RasterIOResampled(
1515 : GDALRWFlag /* eRWFlag */, int nXOff, int nYOff, int nXSize, int nYSize,
1516 : void *pData, int nBufXSize, int nBufYSize, GDALDataType eBufType,
1517 : int nBandCount, const int *panBandMap, GSpacing nPixelSpace,
1518 : GSpacing nLineSpace, GSpacing nBandSpace, GDALRasterIOExtraArg *psExtraArg)
1519 :
1520 : {
1521 : #if 0
1522 : // Determine if we use warping resampling or overview resampling
1523 : bool bUseWarp = false;
1524 : if( GDALDataTypeIsComplex( eDataType ) )
1525 : bUseWarp = true;
1526 : #endif
1527 :
1528 798 : double dfXOff = nXOff;
1529 798 : double dfYOff = nYOff;
1530 798 : double dfXSize = nXSize;
1531 798 : double dfYSize = nYSize;
1532 798 : if (psExtraArg->bFloatingPointWindowValidity)
1533 : {
1534 678 : dfXOff = psExtraArg->dfXOff;
1535 678 : dfYOff = psExtraArg->dfYOff;
1536 678 : dfXSize = psExtraArg->dfXSize;
1537 678 : dfYSize = psExtraArg->dfYSize;
1538 : }
1539 :
1540 798 : const double dfXRatioDstToSrc = dfXSize / nBufXSize;
1541 798 : const double dfYRatioDstToSrc = dfYSize / nBufYSize;
1542 :
1543 : // Determine the coordinates in the "virtual" output raster to see
1544 : // if there are not integers, in which case we will use them as a shift
1545 : // so that subwindow extracts give the exact same results as entire raster
1546 : // scaling.
1547 798 : double dfDestXOff = dfXOff / dfXRatioDstToSrc;
1548 798 : bool bHasXOffVirtual = false;
1549 798 : int nDestXOffVirtual = 0;
1550 798 : if (fabs(dfDestXOff - static_cast<int>(dfDestXOff + 0.5)) < 1e-8)
1551 : {
1552 673 : bHasXOffVirtual = true;
1553 673 : dfXOff = nXOff;
1554 673 : nDestXOffVirtual = static_cast<int>(dfDestXOff + 0.5);
1555 : }
1556 :
1557 798 : double dfDestYOff = dfYOff / dfYRatioDstToSrc;
1558 798 : bool bHasYOffVirtual = false;
1559 798 : int nDestYOffVirtual = 0;
1560 798 : if (fabs(dfDestYOff - static_cast<int>(dfDestYOff + 0.5)) < 1e-8)
1561 : {
1562 634 : bHasYOffVirtual = true;
1563 634 : dfYOff = nYOff;
1564 634 : nDestYOffVirtual = static_cast<int>(dfDestYOff + 0.5);
1565 : }
1566 :
1567 : // Create a MEM dataset that wraps the output buffer.
1568 : GDALDataset *poMEMDS =
1569 798 : MEMDataset::Create("", nDestXOffVirtual + nBufXSize,
1570 : nDestYOffVirtual + nBufYSize, 0, eBufType, nullptr);
1571 : GDALRasterBand **papoDstBands = static_cast<GDALRasterBand **>(
1572 799 : CPLMalloc(nBandCount * sizeof(GDALRasterBand *)));
1573 802 : int nNBITS = 0;
1574 2513 : for (int i = 0; i < nBandCount; i++)
1575 : {
1576 1721 : char szBuffer[32] = {'\0'};
1577 3444 : int nRet = CPLPrintPointer(
1578 : szBuffer,
1579 1721 : static_cast<GByte *>(pData) - nPixelSpace * nDestXOffVirtual -
1580 1721 : nLineSpace * nDestYOffVirtual + nBandSpace * i,
1581 : sizeof(szBuffer));
1582 1723 : szBuffer[nRet] = 0;
1583 :
1584 1723 : char szBuffer0[64] = {'\0'};
1585 1723 : snprintf(szBuffer0, sizeof(szBuffer0), "DATAPOINTER=%s", szBuffer);
1586 :
1587 1723 : char szBuffer1[64] = {'\0'};
1588 1723 : snprintf(szBuffer1, sizeof(szBuffer1), "PIXELOFFSET=" CPL_FRMT_GIB,
1589 : static_cast<GIntBig>(nPixelSpace));
1590 :
1591 1723 : char szBuffer2[64] = {'\0'};
1592 1723 : snprintf(szBuffer2, sizeof(szBuffer2), "LINEOFFSET=" CPL_FRMT_GIB,
1593 : static_cast<GIntBig>(nLineSpace));
1594 :
1595 1723 : char *apszOptions[4] = {szBuffer0, szBuffer1, szBuffer2, nullptr};
1596 :
1597 1723 : poMEMDS->AddBand(eBufType, apszOptions);
1598 :
1599 1722 : GDALRasterBand *poSrcBand = GetRasterBand(panBandMap[i]);
1600 1716 : papoDstBands[i] = poMEMDS->GetRasterBand(i + 1);
1601 : const char *pszNBITS =
1602 1722 : poSrcBand->GetMetadataItem("NBITS", "IMAGE_STRUCTURE");
1603 1703 : if (pszNBITS)
1604 : {
1605 0 : nNBITS = atoi(pszNBITS);
1606 0 : poMEMDS->GetRasterBand(i + 1)->SetMetadataItem("NBITS", pszNBITS,
1607 0 : "IMAGE_STRUCTURE");
1608 : }
1609 : }
1610 :
1611 792 : CPLErr eErr = CE_None;
1612 :
1613 : // TODO(schwehr): Why disabled? Why not just delete?
1614 : // Looks like this code was initially added as disable by copying
1615 : // from RasterIO here:
1616 : // https://trac.osgeo.org/gdal/changeset/29572
1617 : #if 0
1618 : // Do the resampling.
1619 : if( bUseWarp )
1620 : {
1621 : VRTDatasetH hVRTDS = nullptr;
1622 : GDALRasterBandH hVRTBand = nullptr;
1623 : if( GetDataset() == nullptr )
1624 : {
1625 : /* Create VRT dataset that wraps the whole dataset */
1626 : hVRTDS = VRTCreate(nRasterXSize, nRasterYSize);
1627 : VRTAddBand( hVRTDS, eDataType, nullptr );
1628 : hVRTBand = GDALGetRasterBand(hVRTDS, 1);
1629 : VRTAddSimpleSource( (VRTSourcedRasterBandH)hVRTBand,
1630 : (GDALRasterBandH)this,
1631 : 0, 0,
1632 : nRasterXSize, nRasterYSize,
1633 : 0, 0,
1634 : nRasterXSize, nRasterYSize,
1635 : nullptr, VRT_NODATA_UNSET );
1636 :
1637 : /* Add a mask band if needed */
1638 : if( GetMaskFlags() != GMF_ALL_VALID )
1639 : {
1640 : ((GDALDataset*)hVRTDS)->CreateMaskBand(0);
1641 : VRTSourcedRasterBand* poVRTMaskBand =
1642 : (VRTSourcedRasterBand*)(((GDALRasterBand*)hVRTBand)->GetMaskBand());
1643 : poVRTMaskBand->
1644 : AddMaskBandSource( this,
1645 : 0, 0,
1646 : nRasterXSize, nRasterYSize,
1647 : 0, 0,
1648 : nRasterXSize, nRasterYSize);
1649 : }
1650 : }
1651 :
1652 : GDALWarpOptions* psWarpOptions = GDALCreateWarpOptions();
1653 : psWarpOptions->eResampleAlg = (GDALResampleAlg)psExtraArg->eResampleAlg;
1654 : psWarpOptions->hSrcDS = (GDALDatasetH) (hVRTDS ? hVRTDS : GetDataset());
1655 : psWarpOptions->hDstDS = (GDALDatasetH) poMEMDS;
1656 : psWarpOptions->nBandCount = 1;
1657 : int nSrcBandNumber = (hVRTDS ? 1 : nBand);
1658 : int nDstBandNumber = 1;
1659 : psWarpOptions->panSrcBands = &nSrcBandNumber;
1660 : psWarpOptions->panDstBands = &nDstBandNumber;
1661 : psWarpOptions->pfnProgress = psExtraArg->pfnProgress ?
1662 : psExtraArg->pfnProgress : GDALDummyProgress;
1663 : psWarpOptions->pProgressArg = psExtraArg->pProgressData;
1664 : psWarpOptions->pfnTransformer = GDALRasterIOTransformer;
1665 : GDALRasterIOTransformerStruct sTransformer;
1666 : sTransformer.dfXOff = bHasXOffVirtual ? 0 : dfXOff;
1667 : sTransformer.dfYOff = bHasYOffVirtual ? 0 : dfYOff;
1668 : sTransformer.dfXRatioDstToSrc = dfXRatioDstToSrc;
1669 : sTransformer.dfYRatioDstToSrc = dfYRatioDstToSrc;
1670 : psWarpOptions->pTransformerArg = &sTransformer;
1671 :
1672 : GDALWarpOperationH hWarpOperation = GDALCreateWarpOperation(psWarpOptions);
1673 : eErr = GDALChunkAndWarpImage( hWarpOperation,
1674 : nDestXOffVirtual, nDestYOffVirtual,
1675 : nBufXSize, nBufYSize );
1676 : GDALDestroyWarpOperation( hWarpOperation );
1677 :
1678 : psWarpOptions->panSrcBands = nullptr;
1679 : psWarpOptions->panDstBands = nullptr;
1680 : GDALDestroyWarpOptions( psWarpOptions );
1681 :
1682 : if( hVRTDS )
1683 : GDALClose(hVRTDS);
1684 : }
1685 : else
1686 : #endif
1687 : {
1688 792 : const char *pszResampling =
1689 1466 : (psExtraArg->eResampleAlg == GRIORA_Bilinear) ? "BILINEAR"
1690 674 : : (psExtraArg->eResampleAlg == GRIORA_Cubic) ? "CUBIC"
1691 0 : : (psExtraArg->eResampleAlg == GRIORA_CubicSpline) ? "CUBICSPLINE"
1692 0 : : (psExtraArg->eResampleAlg == GRIORA_Lanczos) ? "LANCZOS"
1693 0 : : (psExtraArg->eResampleAlg == GRIORA_Average) ? "AVERAGE"
1694 0 : : (psExtraArg->eResampleAlg == GRIORA_RMS) ? "RMS"
1695 0 : : (psExtraArg->eResampleAlg == GRIORA_Mode) ? "MODE"
1696 0 : : (psExtraArg->eResampleAlg == GRIORA_Gauss) ? "GAUSS"
1697 : : "UNKNOWN";
1698 :
1699 792 : GDALRasterBand *poFirstSrcBand = GetRasterBand(panBandMap[0]);
1700 790 : GDALDataType eDataType = poFirstSrcBand->GetRasterDataType();
1701 : int nBlockXSize, nBlockYSize;
1702 791 : poFirstSrcBand->GetBlockSize(&nBlockXSize, &nBlockYSize);
1703 :
1704 : int nKernelRadius;
1705 : GDALResampleFunction pfnResampleFunc =
1706 787 : GDALGetResampleFunction(pszResampling, &nKernelRadius);
1707 789 : CPLAssert(pfnResampleFunc);
1708 : #ifdef GDAL_ENABLE_RESAMPLING_MULTIBAND
1709 : GDALResampleFunctionMultiBands pfnResampleFuncMultiBands =
1710 : GDALGetResampleFunctionMultiBands(pszResampling, &nKernelRadius);
1711 : #endif
1712 : GDALDataType eWrkDataType =
1713 789 : GDALGetOvrWorkDataType(pszResampling, eDataType);
1714 :
1715 786 : int nDstBlockXSize = nBufXSize;
1716 786 : int nDstBlockYSize = nBufYSize;
1717 : int nFullResXChunk, nFullResYChunk;
1718 : while (true)
1719 : {
1720 786 : nFullResXChunk =
1721 786 : 3 + static_cast<int>(nDstBlockXSize * dfXRatioDstToSrc);
1722 786 : nFullResYChunk =
1723 786 : 3 + static_cast<int>(nDstBlockYSize * dfYRatioDstToSrc);
1724 786 : if (nFullResXChunk > nRasterXSize)
1725 567 : nFullResXChunk = nRasterXSize;
1726 786 : if (nFullResYChunk > nRasterYSize)
1727 45 : nFullResYChunk = nRasterYSize;
1728 786 : if ((nDstBlockXSize == 1 && nDstBlockYSize == 1) ||
1729 784 : (static_cast<GIntBig>(nFullResXChunk) * nFullResYChunk <=
1730 : 1024 * 1024))
1731 : break;
1732 : // When operating on the full width of a raster whose block width is
1733 : // the raster width, prefer doing chunks in height.
1734 0 : if (nFullResXChunk >= nXSize && nXSize == nBlockXSize &&
1735 : nDstBlockYSize > 1)
1736 0 : nDstBlockYSize /= 2;
1737 : /* Otherwise cut the maximal dimension */
1738 0 : else if (nDstBlockXSize > 1 &&
1739 0 : (nFullResXChunk > nFullResYChunk || nDstBlockYSize == 1))
1740 0 : nDstBlockXSize /= 2;
1741 : else
1742 0 : nDstBlockYSize /= 2;
1743 : }
1744 :
1745 1580 : int nOvrFactor = std::max(static_cast<int>(0.5 + dfXRatioDstToSrc),
1746 786 : static_cast<int>(0.5 + dfYRatioDstToSrc));
1747 794 : if (nOvrFactor == 0)
1748 95 : nOvrFactor = 1;
1749 794 : int nFullResXSizeQueried =
1750 794 : nFullResXChunk + 2 * nKernelRadius * nOvrFactor;
1751 794 : int nFullResYSizeQueried =
1752 794 : nFullResYChunk + 2 * nKernelRadius * nOvrFactor;
1753 :
1754 794 : if (nFullResXSizeQueried > nRasterXSize)
1755 596 : nFullResXSizeQueried = nRasterXSize;
1756 794 : if (nFullResYSizeQueried > nRasterYSize)
1757 48 : nFullResYSizeQueried = nRasterYSize;
1758 :
1759 794 : void *pChunk = VSI_MALLOC3_VERBOSE(
1760 : cpl::fits_on<int>(GDALGetDataTypeSizeBytes(eWrkDataType) *
1761 : nBandCount),
1762 : nFullResXSizeQueried, nFullResYSizeQueried);
1763 796 : GByte *pabyChunkNoDataMask = nullptr;
1764 :
1765 796 : GDALRasterBand *poMaskBand = poFirstSrcBand->GetMaskBand();
1766 799 : int nMaskFlags = poFirstSrcBand->GetMaskFlags();
1767 :
1768 796 : bool bUseNoDataMask = ((nMaskFlags & GMF_ALL_VALID) == 0);
1769 796 : if (bUseNoDataMask)
1770 : {
1771 531 : pabyChunkNoDataMask = static_cast<GByte *>(VSI_MALLOC2_VERBOSE(
1772 : nFullResXSizeQueried, nFullResYSizeQueried));
1773 : }
1774 796 : if (pChunk == nullptr ||
1775 531 : (bUseNoDataMask && pabyChunkNoDataMask == nullptr))
1776 : {
1777 6 : GDALClose(poMEMDS);
1778 0 : CPLFree(pChunk);
1779 0 : CPLFree(pabyChunkNoDataMask);
1780 0 : CPLFree(papoDstBands);
1781 0 : return CE_Failure;
1782 : }
1783 :
1784 790 : const int nTotalBlocks = DIV_ROUND_UP(nBufXSize, nDstBlockXSize) *
1785 790 : DIV_ROUND_UP(nBufYSize, nDstBlockYSize);
1786 790 : int nBlocksDone = 0;
1787 :
1788 : int nDstYOff;
1789 1592 : for (nDstYOff = 0; nDstYOff < nBufYSize && eErr == CE_None;
1790 802 : nDstYOff += nDstBlockYSize)
1791 : {
1792 : int nDstYCount;
1793 788 : if (nDstYOff + nDstBlockYSize <= nBufYSize)
1794 790 : nDstYCount = nDstBlockYSize;
1795 : else
1796 0 : nDstYCount = nBufYSize - nDstYOff;
1797 :
1798 788 : int nChunkYOff =
1799 788 : nYOff + static_cast<int>(nDstYOff * dfYRatioDstToSrc);
1800 788 : int nChunkYOff2 = nYOff + 1 +
1801 788 : static_cast<int>(ceil((nDstYOff + nDstYCount) *
1802 : dfYRatioDstToSrc));
1803 788 : if (nChunkYOff2 > nRasterYSize)
1804 103 : nChunkYOff2 = nRasterYSize;
1805 788 : int nYCount = nChunkYOff2 - nChunkYOff;
1806 788 : CPLAssert(nYCount <= nFullResYChunk);
1807 :
1808 788 : int nChunkYOffQueried = nChunkYOff - nKernelRadius * nOvrFactor;
1809 788 : int nChunkYSizeQueried = nYCount + 2 * nKernelRadius * nOvrFactor;
1810 788 : if (nChunkYOffQueried < 0)
1811 : {
1812 106 : nChunkYSizeQueried += nChunkYOffQueried;
1813 106 : nChunkYOffQueried = 0;
1814 : }
1815 788 : if (nChunkYSizeQueried + nChunkYOffQueried > nRasterYSize)
1816 121 : nChunkYSizeQueried = nRasterYSize - nChunkYOffQueried;
1817 788 : CPLAssert(nChunkYSizeQueried <= nFullResYSizeQueried);
1818 :
1819 : int nDstXOff;
1820 1587 : for (nDstXOff = 0; nDstXOff < nBufXSize && eErr == CE_None;
1821 799 : nDstXOff += nDstBlockXSize)
1822 : {
1823 : int nDstXCount;
1824 785 : if (nDstXOff + nDstBlockXSize <= nBufXSize)
1825 789 : nDstXCount = nDstBlockXSize;
1826 : else
1827 0 : nDstXCount = nBufXSize - nDstXOff;
1828 :
1829 785 : int nChunkXOff =
1830 785 : nXOff + static_cast<int>(nDstXOff * dfXRatioDstToSrc);
1831 785 : int nChunkXOff2 =
1832 785 : nXOff + 1 +
1833 785 : static_cast<int>(
1834 785 : ceil((nDstXOff + nDstXCount) * dfXRatioDstToSrc));
1835 785 : if (nChunkXOff2 > nRasterXSize)
1836 604 : nChunkXOff2 = nRasterXSize;
1837 785 : int nXCount = nChunkXOff2 - nChunkXOff;
1838 785 : CPLAssert(nXCount <= nFullResXChunk);
1839 :
1840 785 : int nChunkXOffQueried = nChunkXOff - nKernelRadius * nOvrFactor;
1841 785 : int nChunkXSizeQueried =
1842 785 : nXCount + 2 * nKernelRadius * nOvrFactor;
1843 785 : if (nChunkXOffQueried < 0)
1844 : {
1845 600 : nChunkXSizeQueried += nChunkXOffQueried;
1846 600 : nChunkXOffQueried = 0;
1847 : }
1848 785 : if (nChunkXSizeQueried + nChunkXOffQueried > nRasterXSize)
1849 604 : nChunkXSizeQueried = nRasterXSize - nChunkXOffQueried;
1850 785 : CPLAssert(nChunkXSizeQueried <= nFullResXSizeQueried);
1851 :
1852 785 : bool bSkipResample = false;
1853 785 : bool bNoDataMaskFullyOpaque = false;
1854 785 : if (eErr == CE_None && bUseNoDataMask)
1855 : {
1856 531 : eErr = poMaskBand->RasterIO(
1857 : GF_Read, nChunkXOffQueried, nChunkYOffQueried,
1858 : nChunkXSizeQueried, nChunkYSizeQueried,
1859 : pabyChunkNoDataMask, nChunkXSizeQueried,
1860 : nChunkYSizeQueried, GDT_Byte, 0, 0, nullptr);
1861 :
1862 : /* Optimizations if mask if fully opaque or transparent */
1863 531 : const int nPixels = nChunkXSizeQueried * nChunkYSizeQueried;
1864 531 : const GByte bVal = pabyChunkNoDataMask[0];
1865 531 : int i = 1; // Used after for.
1866 23113400 : for (; i < nPixels; i++)
1867 : {
1868 23112900 : if (pabyChunkNoDataMask[i] != bVal)
1869 72 : break;
1870 : }
1871 531 : if (i == nPixels)
1872 : {
1873 459 : if (bVal == 0)
1874 : {
1875 373 : GByte abyZero[16] = {0};
1876 780 : for (int iBand = 0; iBand < nBandCount; iBand++)
1877 : {
1878 3499 : for (int j = 0; j < nDstYCount; j++)
1879 : {
1880 3092 : GDALCopyWords64(
1881 : abyZero, GDT_Byte, 0,
1882 : static_cast<GByte *>(pData) +
1883 3092 : iBand * nBandSpace +
1884 3092 : nLineSpace * (j + nDstYOff) +
1885 3092 : nDstXOff * nPixelSpace,
1886 : eBufType, static_cast<int>(nPixelSpace),
1887 : nDstXCount);
1888 : }
1889 : }
1890 373 : bSkipResample = true;
1891 : }
1892 : else
1893 : {
1894 86 : bNoDataMaskFullyOpaque = true;
1895 : }
1896 : }
1897 : }
1898 :
1899 785 : if (!bSkipResample && eErr == CE_None)
1900 : {
1901 : /* Read the source buffers */
1902 414 : eErr = RasterIO(
1903 : GF_Read, nChunkXOffQueried, nChunkYOffQueried,
1904 : nChunkXSizeQueried, nChunkYSizeQueried, pChunk,
1905 : nChunkXSizeQueried, nChunkYSizeQueried, eWrkDataType,
1906 : nBandCount, panBandMap, 0, 0, 0, nullptr);
1907 : }
1908 :
1909 : #ifdef GDAL_ENABLE_RESAMPLING_MULTIBAND
1910 : if (pfnResampleFuncMultiBands && !bSkipResample &&
1911 : eErr == CE_None)
1912 : {
1913 : eErr = pfnResampleFuncMultiBands(
1914 : dfXRatioDstToSrc, dfYRatioDstToSrc,
1915 : dfXOff - nXOff, /* == 0 if bHasXOffVirtual */
1916 : dfYOff - nYOff, /* == 0 if bHasYOffVirtual */
1917 : eWrkDataType, (GByte *)pChunk, nBandCount,
1918 : bNoDataMaskFullyOpaque ? nullptr : pabyChunkNoDataMask,
1919 : nChunkXOffQueried - (bHasXOffVirtual ? 0 : nXOff),
1920 : nChunkXSizeQueried,
1921 : nChunkYOffQueried - (bHasYOffVirtual ? 0 : nYOff),
1922 : nChunkYSizeQueried, nDstXOff + nDestXOffVirtual,
1923 : nDstXOff + nDestXOffVirtual + nDstXCount,
1924 : nDstYOff + nDestYOffVirtual,
1925 : nDstYOff + nDestYOffVirtual + nDstYCount, papoDstBands,
1926 : pszResampling, FALSE /*bHasNoData*/,
1927 : 0.0 /* dfNoDataValue */, nullptr /* color table*/,
1928 : eDataType);
1929 : }
1930 : else
1931 : #endif
1932 : {
1933 : size_t nChunkBandOffset =
1934 792 : static_cast<size_t>(nChunkXSizeQueried) *
1935 792 : nChunkYSizeQueried *
1936 792 : GDALGetDataTypeSizeBytes(eWrkDataType);
1937 2116 : for (int i = 0;
1938 2116 : i < nBandCount && !bSkipResample && eErr == CE_None;
1939 : i++)
1940 : {
1941 1317 : const bool bPropagateNoData = false;
1942 1317 : void *pDstBuffer = nullptr;
1943 1317 : GDALDataType eDstBufferDataType = GDT_Unknown;
1944 : GDALRasterBand *poMEMBand =
1945 1317 : poMEMDS->GetRasterBand(i + 1);
1946 1318 : GDALOverviewResampleArgs args;
1947 1318 : args.eSrcDataType = eDataType;
1948 1318 : args.eOvrDataType = poMEMBand->GetRasterDataType();
1949 1317 : args.nOvrXSize = poMEMBand->GetXSize();
1950 1317 : args.nOvrYSize = poMEMBand->GetYSize();
1951 1316 : args.nOvrNBITS = nNBITS;
1952 1316 : args.dfXRatioDstToSrc = dfXRatioDstToSrc;
1953 1316 : args.dfYRatioDstToSrc = dfYRatioDstToSrc;
1954 1316 : args.dfSrcXDelta =
1955 1316 : dfXOff - nXOff; /* == 0 if bHasXOffVirtual */
1956 1316 : args.dfSrcYDelta =
1957 1316 : dfYOff - nYOff; /* == 0 if bHasYOffVirtual */
1958 1316 : args.eWrkDataType = eWrkDataType;
1959 1316 : args.pabyChunkNodataMask = bNoDataMaskFullyOpaque
1960 1316 : ? nullptr
1961 : : pabyChunkNoDataMask;
1962 1316 : args.nChunkXOff =
1963 1316 : nChunkXOffQueried - (bHasXOffVirtual ? 0 : nXOff);
1964 1316 : args.nChunkXSize = nChunkXSizeQueried;
1965 1316 : args.nChunkYOff =
1966 1316 : nChunkYOffQueried - (bHasYOffVirtual ? 0 : nYOff);
1967 1316 : args.nChunkYSize = nChunkYSizeQueried;
1968 1316 : args.nDstXOff = nDstXOff + nDestXOffVirtual;
1969 1316 : args.nDstXOff2 =
1970 1316 : nDstXOff + nDestXOffVirtual + nDstXCount;
1971 1316 : args.nDstYOff = nDstYOff + nDestYOffVirtual;
1972 1316 : args.nDstYOff2 =
1973 1316 : nDstYOff + nDestYOffVirtual + nDstYCount;
1974 1316 : args.pszResampling = pszResampling;
1975 1316 : args.bHasNoData = false;
1976 1316 : args.dfNoDataValue = 0.0;
1977 1316 : args.poColorTable = nullptr;
1978 1316 : args.bPropagateNoData = bPropagateNoData;
1979 :
1980 : eErr =
1981 2634 : pfnResampleFunc(args,
1982 1316 : reinterpret_cast<GByte *>(pChunk) +
1983 1316 : i * nChunkBandOffset,
1984 : &pDstBuffer, &eDstBufferDataType);
1985 1318 : if (eErr == CE_None)
1986 : {
1987 1318 : eErr = poMEMBand->RasterIO(
1988 : GF_Write, nDstXOff + nDestXOffVirtual,
1989 : nDstYOff + nDestYOffVirtual, nDstXCount,
1990 : nDstYCount, pDstBuffer, nDstXCount, nDstYCount,
1991 : eDstBufferDataType, 0, 0, nullptr);
1992 : }
1993 1318 : CPLFree(pDstBuffer);
1994 : }
1995 : }
1996 :
1997 799 : nBlocksDone++;
1998 1188 : if (eErr == CE_None && psExtraArg->pfnProgress != nullptr &&
1999 389 : !psExtraArg->pfnProgress(1.0 * nBlocksDone / nTotalBlocks,
2000 : "", psExtraArg->pProgressData))
2001 : {
2002 0 : eErr = CE_Failure;
2003 : }
2004 : }
2005 : }
2006 :
2007 804 : CPLFree(pChunk);
2008 800 : CPLFree(pabyChunkNoDataMask);
2009 : }
2010 :
2011 800 : CPLFree(papoDstBands);
2012 800 : GDALClose(poMEMDS);
2013 :
2014 800 : return eErr;
2015 : }
2016 :
2017 : //! @endcond
2018 :
2019 : /************************************************************************/
2020 : /* GDALSwapWords() */
2021 : /************************************************************************/
2022 :
2023 : /**
2024 : * Byte swap words in-place.
2025 : *
2026 : * This function will byte swap a set of 2, 4 or 8 byte words "in place" in
2027 : * a memory array. No assumption is made that the words being swapped are
2028 : * word aligned in memory. Use the CPL_LSB and CPL_MSB macros from cpl_port.h
2029 : * to determine if the current platform is big endian or little endian. Use
2030 : * The macros like CPL_SWAP32() to byte swap single values without the overhead
2031 : * of a function call.
2032 : *
2033 : * @param pData pointer to start of data buffer.
2034 : * @param nWordSize size of words being swapped in bytes. Normally 2, 4 or 8.
2035 : * @param nWordCount the number of words to be swapped in this call.
2036 : * @param nWordSkip the byte offset from the start of one word to the start of
2037 : * the next. For packed buffers this is the same as nWordSize.
2038 : */
2039 :
2040 497137 : void CPL_STDCALL GDALSwapWords(void *pData, int nWordSize, int nWordCount,
2041 : int nWordSkip)
2042 :
2043 : {
2044 497137 : if (nWordCount > 0)
2045 497137 : VALIDATE_POINTER0(pData, "GDALSwapWords");
2046 :
2047 497137 : GByte *pabyData = static_cast<GByte *>(pData);
2048 :
2049 497137 : switch (nWordSize)
2050 : {
2051 7234 : case 1:
2052 7234 : break;
2053 :
2054 476903 : case 2:
2055 476903 : CPLAssert(nWordSkip >= 2 || nWordCount == 1);
2056 228062000 : for (int i = 0; i < nWordCount; i++)
2057 : {
2058 227585000 : CPL_SWAP16PTR(pabyData);
2059 227585000 : pabyData += nWordSkip;
2060 : }
2061 476903 : break;
2062 :
2063 10574 : case 4:
2064 10574 : CPLAssert(nWordSkip >= 4 || nWordCount == 1);
2065 10574 : if (CPL_IS_ALIGNED(pabyData, 4) && (nWordSkip % 4) == 0)
2066 : {
2067 29140500 : for (int i = 0; i < nWordCount; i++)
2068 : {
2069 29130000 : *reinterpret_cast<GUInt32 *>(pabyData) = CPL_SWAP32(
2070 : *reinterpret_cast<const GUInt32 *>(pabyData));
2071 29130000 : pabyData += nWordSkip;
2072 10571 : }
2073 : }
2074 : else
2075 : {
2076 9 : for (int i = 0; i < nWordCount; i++)
2077 : {
2078 6 : CPL_SWAP32PTR(pabyData);
2079 6 : pabyData += nWordSkip;
2080 : }
2081 : }
2082 10574 : break;
2083 :
2084 2426 : case 8:
2085 2426 : CPLAssert(nWordSkip >= 8 || nWordCount == 1);
2086 2426 : if (CPL_IS_ALIGNED(pabyData, 8) && (nWordSkip % 8) == 0)
2087 : {
2088 3356900 : for (int i = 0; i < nWordCount; i++)
2089 : {
2090 3354480 : *reinterpret_cast<GUInt64 *>(pabyData) = CPL_SWAP64(
2091 : *reinterpret_cast<const GUInt64 *>(pabyData));
2092 3354480 : pabyData += nWordSkip;
2093 2425 : }
2094 : }
2095 : else
2096 : {
2097 3 : for (int i = 0; i < nWordCount; i++)
2098 : {
2099 2 : CPL_SWAP64PTR(pabyData);
2100 2 : pabyData += nWordSkip;
2101 : }
2102 : }
2103 2426 : break;
2104 :
2105 0 : default:
2106 0 : CPLAssert(false);
2107 : }
2108 : }
2109 :
2110 : /************************************************************************/
2111 : /* GDALSwapWordsEx() */
2112 : /************************************************************************/
2113 :
2114 : /**
2115 : * Byte swap words in-place.
2116 : *
2117 : * This function will byte swap a set of 2, 4 or 8 byte words "in place" in
2118 : * a memory array. No assumption is made that the words being swapped are
2119 : * word aligned in memory. Use the CPL_LSB and CPL_MSB macros from cpl_port.h
2120 : * to determine if the current platform is big endian or little endian. Use
2121 : * The macros like CPL_SWAP32() to byte swap single values without the overhead
2122 : * of a function call.
2123 : *
2124 : * @param pData pointer to start of data buffer.
2125 : * @param nWordSize size of words being swapped in bytes. Normally 2, 4 or 8.
2126 : * @param nWordCount the number of words to be swapped in this call.
2127 : * @param nWordSkip the byte offset from the start of one word to the start of
2128 : * the next. For packed buffers this is the same as nWordSize.
2129 : * @since GDAL 2.1
2130 : */
2131 6118 : void CPL_STDCALL GDALSwapWordsEx(void *pData, int nWordSize, size_t nWordCount,
2132 : int nWordSkip)
2133 : {
2134 6118 : GByte *pabyData = static_cast<GByte *>(pData);
2135 12236 : while (nWordCount)
2136 : {
2137 : // Pick-up a multiple of 8 as max chunk size.
2138 6118 : const int nWordCountSmall =
2139 6118 : (nWordCount > (1 << 30)) ? (1 << 30) : static_cast<int>(nWordCount);
2140 6118 : GDALSwapWords(pabyData, nWordSize, nWordCountSmall, nWordSkip);
2141 6118 : pabyData += static_cast<size_t>(nWordSkip) * nWordCountSmall;
2142 6118 : nWordCount -= nWordCountSmall;
2143 : }
2144 6118 : }
2145 :
2146 : // Place the new GDALCopyWords helpers in an anonymous namespace
2147 : namespace
2148 : {
2149 :
2150 : /************************************************************************/
2151 : /* GDALCopyWordsT() */
2152 : /************************************************************************/
2153 : /**
2154 : * Template function, used to copy data from pSrcData into buffer
2155 : * pDstData, with stride nSrcPixelStride in the source data and
2156 : * stride nDstPixelStride in the destination data. This template can
2157 : * deal with the case where the input data type is real or complex and
2158 : * the output is real.
2159 : *
2160 : * @param pSrcData the source data buffer
2161 : * @param nSrcPixelStride the stride, in the buffer pSrcData for pixels
2162 : * of interest.
2163 : * @param pDstData the destination buffer.
2164 : * @param nDstPixelStride the stride in the buffer pDstData for pixels of
2165 : * interest.
2166 : * @param nWordCount the total number of pixel words to copy
2167 : *
2168 : * @code
2169 : * // Assume an input buffer of type GUInt16 named pBufferIn
2170 : * GByte *pBufferOut = new GByte[numBytesOut];
2171 : * GDALCopyWordsT<GUInt16, GByte>(pSrcData, 2, pDstData, 1, numBytesOut);
2172 : * @endcode
2173 : * @note
2174 : * This is a private function, and should not be exposed outside of
2175 : * rasterio.cpp. External users should call the GDALCopyWords driver function.
2176 : */
2177 :
2178 : template <class Tin, class Tout>
2179 42356155 : static void inline GDALCopyWordsGenericT(const Tin *const CPL_RESTRICT pSrcData,
2180 : int nSrcPixelStride,
2181 : Tout *const CPL_RESTRICT pDstData,
2182 : int nDstPixelStride,
2183 : GPtrDiff_t nWordCount)
2184 : {
2185 42356155 : decltype(nWordCount) nDstOffset = 0;
2186 :
2187 42356155 : const char *const pSrcDataPtr = reinterpret_cast<const char *>(pSrcData);
2188 42356155 : char *const pDstDataPtr = reinterpret_cast<char *>(pDstData);
2189 515103798 : for (decltype(nWordCount) n = 0; n < nWordCount; n++)
2190 : {
2191 472751852 : const Tin tValue =
2192 472751852 : *reinterpret_cast<const Tin *>(pSrcDataPtr + (n * nSrcPixelStride));
2193 472751852 : Tout *const pOutPixel =
2194 472751852 : reinterpret_cast<Tout *>(pDstDataPtr + nDstOffset);
2195 :
2196 472751852 : GDALCopyWord(tValue, *pOutPixel);
2197 :
2198 472747752 : nDstOffset += nDstPixelStride;
2199 : }
2200 42352097 : }
2201 :
2202 : template <class Tin, class Tout>
2203 29693935 : static void CPL_NOINLINE GDALCopyWordsT(const Tin *const CPL_RESTRICT pSrcData,
2204 : int nSrcPixelStride,
2205 : Tout *const CPL_RESTRICT pDstData,
2206 : int nDstPixelStride,
2207 : GPtrDiff_t nWordCount)
2208 : {
2209 29693935 : GDALCopyWordsGenericT(pSrcData, nSrcPixelStride, pDstData, nDstPixelStride,
2210 : nWordCount);
2211 29693936 : }
2212 :
2213 : template <class Tin, class Tout>
2214 4912365 : static void inline GDALCopyWordsT_8atatime(
2215 : const Tin *const CPL_RESTRICT pSrcData, int nSrcPixelStride,
2216 : Tout *const CPL_RESTRICT pDstData, int nDstPixelStride,
2217 : GPtrDiff_t nWordCount)
2218 : {
2219 4912365 : decltype(nWordCount) nDstOffset = 0;
2220 :
2221 4912365 : const char *const pSrcDataPtr = reinterpret_cast<const char *>(pSrcData);
2222 4912365 : char *const pDstDataPtr = reinterpret_cast<char *>(pDstData);
2223 4912365 : decltype(nWordCount) n = 0;
2224 4912365 : if (nSrcPixelStride == static_cast<int>(sizeof(Tin)) &&
2225 : nDstPixelStride == static_cast<int>(sizeof(Tout)))
2226 : {
2227 34931659 : for (; n < nWordCount - 7; n += 8)
2228 : {
2229 34518102 : const Tin *pInValues = reinterpret_cast<const Tin *>(
2230 34518102 : pSrcDataPtr + (n * nSrcPixelStride));
2231 34518102 : Tout *const pOutPixels =
2232 34518102 : reinterpret_cast<Tout *>(pDstDataPtr + nDstOffset);
2233 :
2234 34518102 : GDALCopy8Words(pInValues, pOutPixels);
2235 :
2236 34516771 : nDstOffset += 8 * nDstPixelStride;
2237 : }
2238 : }
2239 10211400 : for (; n < nWordCount; n++)
2240 : {
2241 5299046 : const Tin tValue =
2242 5299046 : *reinterpret_cast<const Tin *>(pSrcDataPtr + (n * nSrcPixelStride));
2243 5299046 : Tout *const pOutPixel =
2244 5299046 : reinterpret_cast<Tout *>(pDstDataPtr + nDstOffset);
2245 :
2246 5299046 : GDALCopyWord(tValue, *pOutPixel);
2247 :
2248 5300358 : nDstOffset += nDstPixelStride;
2249 : }
2250 4912354 : }
2251 :
2252 : #ifdef HAVE_SSE2
2253 :
2254 : template <class Tout>
2255 39675 : void GDALCopyWordsByteTo16Bit(const GByte *const CPL_RESTRICT pSrcData,
2256 : int nSrcPixelStride,
2257 : Tout *const CPL_RESTRICT pDstData,
2258 : int nDstPixelStride, GPtrDiff_t nWordCount)
2259 : {
2260 : static_assert(std::is_integral<Tout>::value &&
2261 : sizeof(Tout) == sizeof(uint16_t),
2262 : "Bad Tout");
2263 39675 : if (nSrcPixelStride == static_cast<int>(sizeof(*pSrcData)) &&
2264 : nDstPixelStride == static_cast<int>(sizeof(*pDstData)))
2265 : {
2266 33324 : decltype(nWordCount) n = 0;
2267 33324 : const __m128i xmm_zero = _mm_setzero_si128();
2268 33324 : GByte *CPL_RESTRICT pabyDstDataPtr =
2269 : reinterpret_cast<GByte *>(pDstData);
2270 1415668 : for (; n < nWordCount - 15; n += 16)
2271 : {
2272 1382344 : __m128i xmm = _mm_loadu_si128(
2273 1382344 : reinterpret_cast<const __m128i *>(pSrcData + n));
2274 1382344 : __m128i xmm0 = _mm_unpacklo_epi8(xmm, xmm_zero);
2275 1382344 : __m128i xmm1 = _mm_unpackhi_epi8(xmm, xmm_zero);
2276 : _mm_storeu_si128(
2277 1382344 : reinterpret_cast<__m128i *>(pabyDstDataPtr + n * 2), xmm0);
2278 : _mm_storeu_si128(
2279 1382344 : reinterpret_cast<__m128i *>(pabyDstDataPtr + n * 2 + 16), xmm1);
2280 : }
2281 109209 : for (; n < nWordCount; n++)
2282 : {
2283 75885 : pDstData[n] = pSrcData[n];
2284 33324 : }
2285 : }
2286 : else
2287 : {
2288 6351 : GDALCopyWordsGenericT(pSrcData, nSrcPixelStride, pDstData,
2289 : nDstPixelStride, nWordCount);
2290 : }
2291 39675 : }
2292 :
2293 : template <>
2294 26942 : CPL_NOINLINE void GDALCopyWordsT(const GByte *const CPL_RESTRICT pSrcData,
2295 : int nSrcPixelStride,
2296 : GUInt16 *const CPL_RESTRICT pDstData,
2297 : int nDstPixelStride, GPtrDiff_t nWordCount)
2298 : {
2299 26942 : GDALCopyWordsByteTo16Bit(pSrcData, nSrcPixelStride, pDstData,
2300 : nDstPixelStride, nWordCount);
2301 26942 : }
2302 :
2303 : template <>
2304 12733 : CPL_NOINLINE void GDALCopyWordsT(const GByte *const CPL_RESTRICT pSrcData,
2305 : int nSrcPixelStride,
2306 : GInt16 *const CPL_RESTRICT pDstData,
2307 : int nDstPixelStride, GPtrDiff_t nWordCount)
2308 : {
2309 12733 : GDALCopyWordsByteTo16Bit(pSrcData, nSrcPixelStride, pDstData,
2310 : nDstPixelStride, nWordCount);
2311 12733 : }
2312 :
2313 : template <class Tout>
2314 12813965 : void GDALCopyWordsByteTo32Bit(const GByte *const CPL_RESTRICT pSrcData,
2315 : int nSrcPixelStride,
2316 : Tout *const CPL_RESTRICT pDstData,
2317 : int nDstPixelStride, GPtrDiff_t nWordCount)
2318 : {
2319 : static_assert(std::is_integral<Tout>::value &&
2320 : sizeof(Tout) == sizeof(uint32_t),
2321 : "Bad Tout");
2322 12813965 : if (nSrcPixelStride == static_cast<int>(sizeof(*pSrcData)) &&
2323 : nDstPixelStride == static_cast<int>(sizeof(*pDstData)))
2324 : {
2325 6243375 : decltype(nWordCount) n = 0;
2326 6243375 : const __m128i xmm_zero = _mm_setzero_si128();
2327 6243375 : GByte *CPL_RESTRICT pabyDstDataPtr =
2328 : reinterpret_cast<GByte *>(pDstData);
2329 70530300 : for (; n < nWordCount - 15; n += 16)
2330 : {
2331 64265945 : __m128i xmm = _mm_loadu_si128(
2332 64265945 : reinterpret_cast<const __m128i *>(pSrcData + n));
2333 64275345 : __m128i xmm_low = _mm_unpacklo_epi8(xmm, xmm_zero);
2334 64312445 : __m128i xmm_high = _mm_unpackhi_epi8(xmm, xmm_zero);
2335 64225745 : __m128i xmm0 = _mm_unpacklo_epi16(xmm_low, xmm_zero);
2336 64259645 : __m128i xmm1 = _mm_unpackhi_epi16(xmm_low, xmm_zero);
2337 63996645 : __m128i xmm2 = _mm_unpacklo_epi16(xmm_high, xmm_zero);
2338 64286945 : __m128i xmm3 = _mm_unpackhi_epi16(xmm_high, xmm_zero);
2339 : _mm_storeu_si128(
2340 64286945 : reinterpret_cast<__m128i *>(pabyDstDataPtr + n * 4), xmm0);
2341 : _mm_storeu_si128(
2342 64286945 : reinterpret_cast<__m128i *>(pabyDstDataPtr + n * 4 + 16), xmm1);
2343 : _mm_storeu_si128(
2344 64286945 : reinterpret_cast<__m128i *>(pabyDstDataPtr + n * 4 + 32), xmm2);
2345 : _mm_storeu_si128(
2346 64286945 : reinterpret_cast<__m128i *>(pabyDstDataPtr + n * 4 + 48), xmm3);
2347 : }
2348 14520279 : for (; n < nWordCount; n++)
2349 : {
2350 8255854 : pDstData[n] = pSrcData[n];
2351 6264415 : }
2352 : }
2353 : else
2354 : {
2355 6570640 : GDALCopyWordsGenericT(pSrcData, nSrcPixelStride, pDstData,
2356 : nDstPixelStride, nWordCount);
2357 : }
2358 12816465 : }
2359 :
2360 : template <>
2361 465 : CPL_NOINLINE void GDALCopyWordsT(const GByte *const CPL_RESTRICT pSrcData,
2362 : int nSrcPixelStride,
2363 : GUInt32 *const CPL_RESTRICT pDstData,
2364 : int nDstPixelStride, GPtrDiff_t nWordCount)
2365 : {
2366 465 : GDALCopyWordsByteTo32Bit(pSrcData, nSrcPixelStride, pDstData,
2367 : nDstPixelStride, nWordCount);
2368 465 : }
2369 :
2370 : template <>
2371 12825900 : CPL_NOINLINE void GDALCopyWordsT(const GByte *const CPL_RESTRICT pSrcData,
2372 : int nSrcPixelStride,
2373 : GInt32 *const CPL_RESTRICT pDstData,
2374 : int nDstPixelStride, GPtrDiff_t nWordCount)
2375 : {
2376 12825900 : GDALCopyWordsByteTo32Bit(pSrcData, nSrcPixelStride, pDstData,
2377 : nDstPixelStride, nWordCount);
2378 12841200 : }
2379 :
2380 : template <>
2381 2471810 : CPL_NOINLINE void GDALCopyWordsT(const GByte *const CPL_RESTRICT pSrcData,
2382 : int nSrcPixelStride,
2383 : float *const CPL_RESTRICT pDstData,
2384 : int nDstPixelStride, GPtrDiff_t nWordCount)
2385 : {
2386 2471810 : if (nSrcPixelStride == static_cast<int>(sizeof(*pSrcData)) &&
2387 : nDstPixelStride == static_cast<int>(sizeof(*pDstData)))
2388 : {
2389 112368 : decltype(nWordCount) n = 0;
2390 112368 : const __m128i xmm_zero = _mm_setzero_si128();
2391 112368 : GByte *CPL_RESTRICT pabyDstDataPtr =
2392 : reinterpret_cast<GByte *>(pDstData);
2393 3261800 : for (; n < nWordCount - 15; n += 16)
2394 : {
2395 3149440 : __m128i xmm = _mm_loadu_si128(
2396 3149440 : reinterpret_cast<const __m128i *>(pSrcData + n));
2397 3149440 : __m128i xmm_low = _mm_unpacklo_epi8(xmm, xmm_zero);
2398 3149440 : __m128i xmm_high = _mm_unpackhi_epi8(xmm, xmm_zero);
2399 3149440 : __m128i xmm0 = _mm_unpacklo_epi16(xmm_low, xmm_zero);
2400 3149440 : __m128i xmm1 = _mm_unpackhi_epi16(xmm_low, xmm_zero);
2401 3149440 : __m128i xmm2 = _mm_unpacklo_epi16(xmm_high, xmm_zero);
2402 3149440 : __m128i xmm3 = _mm_unpackhi_epi16(xmm_high, xmm_zero);
2403 3149440 : __m128 xmm0_f = _mm_cvtepi32_ps(xmm0);
2404 3149440 : __m128 xmm1_f = _mm_cvtepi32_ps(xmm1);
2405 3149440 : __m128 xmm2_f = _mm_cvtepi32_ps(xmm2);
2406 3149440 : __m128 xmm3_f = _mm_cvtepi32_ps(xmm3);
2407 3149440 : _mm_storeu_ps(reinterpret_cast<float *>(pabyDstDataPtr + n * 4),
2408 : xmm0_f);
2409 : _mm_storeu_ps(
2410 3149440 : reinterpret_cast<float *>(pabyDstDataPtr + n * 4 + 16), xmm1_f);
2411 : _mm_storeu_ps(
2412 3149440 : reinterpret_cast<float *>(pabyDstDataPtr + n * 4 + 32), xmm2_f);
2413 : _mm_storeu_ps(
2414 3149440 : reinterpret_cast<float *>(pabyDstDataPtr + n * 4 + 48), xmm3_f);
2415 : }
2416 480044 : for (; n < nWordCount; n++)
2417 : {
2418 367676 : pDstData[n] = pSrcData[n];
2419 112368 : }
2420 : }
2421 : else
2422 : {
2423 2359440 : GDALCopyWordsGenericT(pSrcData, nSrcPixelStride, pDstData,
2424 : nDstPixelStride, nWordCount);
2425 : }
2426 2471810 : }
2427 :
2428 : template <>
2429 147675 : CPL_NOINLINE void GDALCopyWordsT(const GByte *const CPL_RESTRICT pSrcData,
2430 : int nSrcPixelStride,
2431 : double *const CPL_RESTRICT pDstData,
2432 : int nDstPixelStride, GPtrDiff_t nWordCount)
2433 : {
2434 147675 : if (nSrcPixelStride == static_cast<int>(sizeof(*pSrcData)) &&
2435 : nDstPixelStride == static_cast<int>(sizeof(*pDstData)))
2436 : {
2437 124619 : decltype(nWordCount) n = 0;
2438 124619 : const __m128i xmm_zero = _mm_setzero_si128();
2439 124619 : GByte *CPL_RESTRICT pabyDstDataPtr =
2440 : reinterpret_cast<GByte *>(pDstData);
2441 1424200 : for (; n < nWordCount - 15; n += 16)
2442 : {
2443 1299580 : __m128i xmm = _mm_loadu_si128(
2444 1299580 : reinterpret_cast<const __m128i *>(pSrcData + n));
2445 1299580 : __m128i xmm_low = _mm_unpacklo_epi8(xmm, xmm_zero);
2446 1299580 : __m128i xmm_high = _mm_unpackhi_epi8(xmm, xmm_zero);
2447 1299580 : __m128i xmm0 = _mm_unpacklo_epi16(xmm_low, xmm_zero);
2448 1299580 : __m128i xmm1 = _mm_unpackhi_epi16(xmm_low, xmm_zero);
2449 1299580 : __m128i xmm2 = _mm_unpacklo_epi16(xmm_high, xmm_zero);
2450 1299580 : __m128i xmm3 = _mm_unpackhi_epi16(xmm_high, xmm_zero);
2451 :
2452 : #if defined(__AVX2__) && defined(slightly_slower_than_SSE2)
2453 : _mm256_storeu_pd(reinterpret_cast<double *>(pabyDstDataPtr + n * 8),
2454 : _mm256_cvtepi32_pd(xmm0));
2455 : _mm256_storeu_pd(
2456 : reinterpret_cast<double *>(pabyDstDataPtr + n * 8 + 32),
2457 : _mm256_cvtepi32_pd(xmm1));
2458 : _mm256_storeu_pd(
2459 : reinterpret_cast<double *>(pabyDstDataPtr + n * 8 + 64),
2460 : _mm256_cvtepi32_pd(xmm2));
2461 : _mm256_storeu_pd(
2462 : reinterpret_cast<double *>(pabyDstDataPtr + n * 8 + 96),
2463 : _mm256_cvtepi32_pd(xmm3));
2464 : #else
2465 1299580 : __m128d xmm0_low_d = _mm_cvtepi32_pd(xmm0);
2466 1299580 : __m128d xmm1_low_d = _mm_cvtepi32_pd(xmm1);
2467 1299580 : __m128d xmm2_low_d = _mm_cvtepi32_pd(xmm2);
2468 1299580 : __m128d xmm3_low_d = _mm_cvtepi32_pd(xmm3);
2469 1299580 : xmm0 = _mm_srli_si128(xmm0, 8);
2470 1299580 : xmm1 = _mm_srli_si128(xmm1, 8);
2471 1299580 : xmm2 = _mm_srli_si128(xmm2, 8);
2472 1299580 : xmm3 = _mm_srli_si128(xmm3, 8);
2473 1299580 : __m128d xmm0_high_d = _mm_cvtepi32_pd(xmm0);
2474 1299580 : __m128d xmm1_high_d = _mm_cvtepi32_pd(xmm1);
2475 1299580 : __m128d xmm2_high_d = _mm_cvtepi32_pd(xmm2);
2476 1299580 : __m128d xmm3_high_d = _mm_cvtepi32_pd(xmm3);
2477 :
2478 1299580 : _mm_storeu_pd(reinterpret_cast<double *>(pabyDstDataPtr + n * 8),
2479 : xmm0_low_d);
2480 : _mm_storeu_pd(
2481 1299580 : reinterpret_cast<double *>(pabyDstDataPtr + n * 8 + 16),
2482 : xmm0_high_d);
2483 : _mm_storeu_pd(
2484 1299580 : reinterpret_cast<double *>(pabyDstDataPtr + n * 8 + 32),
2485 : xmm1_low_d);
2486 : _mm_storeu_pd(
2487 1299580 : reinterpret_cast<double *>(pabyDstDataPtr + n * 8 + 48),
2488 : xmm1_high_d);
2489 : _mm_storeu_pd(
2490 1299580 : reinterpret_cast<double *>(pabyDstDataPtr + n * 8 + 64),
2491 : xmm2_low_d);
2492 : _mm_storeu_pd(
2493 1299580 : reinterpret_cast<double *>(pabyDstDataPtr + n * 8 + 80),
2494 : xmm2_high_d);
2495 : _mm_storeu_pd(
2496 1299580 : reinterpret_cast<double *>(pabyDstDataPtr + n * 8 + 96),
2497 : xmm3_low_d);
2498 : _mm_storeu_pd(
2499 1299580 : reinterpret_cast<double *>(pabyDstDataPtr + n * 8 + 112),
2500 : xmm3_high_d);
2501 : #endif
2502 : }
2503 236049 : for (; n < nWordCount; n++)
2504 : {
2505 111430 : pDstData[n] = pSrcData[n];
2506 124619 : }
2507 : }
2508 : else
2509 : {
2510 23056 : GDALCopyWordsGenericT(pSrcData, nSrcPixelStride, pDstData,
2511 : nDstPixelStride, nWordCount);
2512 : }
2513 147675 : }
2514 :
2515 : template <>
2516 148 : CPL_NOINLINE void GDALCopyWordsT(const uint8_t *const CPL_RESTRICT pSrcData,
2517 : int nSrcPixelStride,
2518 : int8_t *const CPL_RESTRICT pDstData,
2519 : int nDstPixelStride, GPtrDiff_t nWordCount)
2520 : {
2521 148 : if (nSrcPixelStride == static_cast<int>(sizeof(*pSrcData)) &&
2522 : nDstPixelStride == static_cast<int>(sizeof(*pDstData)))
2523 : {
2524 142 : decltype(nWordCount) n = 0;
2525 142 : const __m128i xmm_127 = _mm_set1_epi8(127);
2526 146 : for (; n < nWordCount - 31; n += 32)
2527 : {
2528 8 : __m128i xmm0 = _mm_loadu_si128(
2529 4 : reinterpret_cast<const __m128i *>(pSrcData + n));
2530 4 : __m128i xmm1 = _mm_loadu_si128(
2531 4 : reinterpret_cast<const __m128i *>(pSrcData + n + 16));
2532 4 : xmm0 = _mm_min_epu8(xmm0, xmm_127);
2533 4 : xmm1 = _mm_min_epu8(xmm1, xmm_127);
2534 4 : _mm_storeu_si128(reinterpret_cast<__m128i *>(pDstData + n), xmm0);
2535 4 : _mm_storeu_si128(reinterpret_cast<__m128i *>(pDstData + n + 16),
2536 : xmm1);
2537 : }
2538 2422 : for (; n < nWordCount; n++)
2539 : {
2540 2280 : pDstData[n] =
2541 2280 : pSrcData[n] >= 127 ? 127 : static_cast<int8_t>(pSrcData[n]);
2542 142 : }
2543 : }
2544 : else
2545 : {
2546 6 : GDALCopyWordsGenericT(pSrcData, nSrcPixelStride, pDstData,
2547 : nDstPixelStride, nWordCount);
2548 : }
2549 148 : }
2550 :
2551 : template <>
2552 82 : CPL_NOINLINE void GDALCopyWordsT(const int8_t *const CPL_RESTRICT pSrcData,
2553 : int nSrcPixelStride,
2554 : uint8_t *const CPL_RESTRICT pDstData,
2555 : int nDstPixelStride, GPtrDiff_t nWordCount)
2556 : {
2557 82 : if (nSrcPixelStride == static_cast<int>(sizeof(*pSrcData)) &&
2558 : nDstPixelStride == static_cast<int>(sizeof(*pDstData)))
2559 : {
2560 56 : decltype(nWordCount) n = 0;
2561 : #if !(defined(__SSE4_1__) || defined(USE_NEON_OPTIMIZATIONS))
2562 56 : const __m128i xmm_INT8_to_UINT8 = _mm_set1_epi8(-128);
2563 : #endif
2564 117 : for (; n < nWordCount - 31; n += 32)
2565 : {
2566 122 : __m128i xmm0 = _mm_loadu_si128(
2567 61 : reinterpret_cast<const __m128i *>(pSrcData + n));
2568 61 : __m128i xmm1 = _mm_loadu_si128(
2569 61 : reinterpret_cast<const __m128i *>(pSrcData + n + 16));
2570 : #if defined(__SSE4_1__) || defined(USE_NEON_OPTIMIZATIONS)
2571 : xmm0 = _mm_max_epi8(xmm0, _mm_setzero_si128());
2572 : xmm1 = _mm_max_epi8(xmm1, _mm_setzero_si128());
2573 : #else
2574 61 : xmm0 = _mm_add_epi8(xmm0, xmm_INT8_to_UINT8);
2575 61 : xmm1 = _mm_add_epi8(xmm1, xmm_INT8_to_UINT8);
2576 61 : xmm0 = _mm_max_epu8(xmm0, xmm_INT8_to_UINT8);
2577 61 : xmm1 = _mm_max_epu8(xmm1, xmm_INT8_to_UINT8);
2578 61 : xmm0 = _mm_sub_epi8(xmm0, xmm_INT8_to_UINT8);
2579 61 : xmm1 = _mm_sub_epi8(xmm1, xmm_INT8_to_UINT8);
2580 : #endif
2581 61 : _mm_storeu_si128(reinterpret_cast<__m128i *>(pDstData + n), xmm0);
2582 61 : _mm_storeu_si128(reinterpret_cast<__m128i *>(pDstData + n + 16),
2583 : xmm1);
2584 : }
2585 352 : for (; n < nWordCount; n++)
2586 : {
2587 296 : pDstData[n] =
2588 296 : pSrcData[n] < 0 ? 0 : static_cast<uint8_t>(pSrcData[n]);
2589 56 : }
2590 : }
2591 : else
2592 : {
2593 26 : GDALCopyWordsGenericT(pSrcData, nSrcPixelStride, pDstData,
2594 : nDstPixelStride, nWordCount);
2595 : }
2596 82 : }
2597 :
2598 : template <>
2599 6037 : CPL_NOINLINE void GDALCopyWordsT(const uint16_t *const CPL_RESTRICT pSrcData,
2600 : int nSrcPixelStride,
2601 : uint8_t *const CPL_RESTRICT pDstData,
2602 : int nDstPixelStride, GPtrDiff_t nWordCount)
2603 : {
2604 6037 : if (nSrcPixelStride == static_cast<int>(sizeof(*pSrcData)) &&
2605 : nDstPixelStride == static_cast<int>(sizeof(*pDstData)))
2606 : {
2607 5062 : decltype(nWordCount) n = 0;
2608 : #if defined(__SSE4_1__) || defined(USE_NEON_OPTIMIZATIONS)
2609 : const auto xmm_MAX_INT16 = _mm_set1_epi16(32767);
2610 : #else
2611 : // In SSE2, min_epu16 does not exist, so shift from
2612 : // UInt16 to SInt16 to be able to use min_epi16
2613 5062 : const __m128i xmm_UINT16_to_INT16 = _mm_set1_epi16(-32768);
2614 5062 : const __m128i xmm_m255_shifted = _mm_set1_epi16(255 - 32768);
2615 : #endif
2616 71888 : for (; n < nWordCount - 15; n += 16)
2617 : {
2618 133652 : __m128i xmm0 = _mm_loadu_si128(
2619 66826 : reinterpret_cast<const __m128i *>(pSrcData + n));
2620 66826 : __m128i xmm1 = _mm_loadu_si128(
2621 66826 : reinterpret_cast<const __m128i *>(pSrcData + n + 8));
2622 : #if defined(__SSE4_1__) || defined(USE_NEON_OPTIMIZATIONS)
2623 : xmm0 = _mm_min_epu16(xmm0, xmm_MAX_INT16);
2624 : xmm1 = _mm_min_epu16(xmm1, xmm_MAX_INT16);
2625 : #else
2626 66826 : xmm0 = _mm_add_epi16(xmm0, xmm_UINT16_to_INT16);
2627 66826 : xmm1 = _mm_add_epi16(xmm1, xmm_UINT16_to_INT16);
2628 66826 : xmm0 = _mm_min_epi16(xmm0, xmm_m255_shifted);
2629 66826 : xmm1 = _mm_min_epi16(xmm1, xmm_m255_shifted);
2630 66826 : xmm0 = _mm_sub_epi16(xmm0, xmm_UINT16_to_INT16);
2631 66826 : xmm1 = _mm_sub_epi16(xmm1, xmm_UINT16_to_INT16);
2632 : #endif
2633 66826 : xmm0 = _mm_packus_epi16(xmm0, xmm1);
2634 66826 : _mm_storeu_si128(reinterpret_cast<__m128i *>(pDstData + n), xmm0);
2635 : }
2636 16403 : for (; n < nWordCount; n++)
2637 : {
2638 11341 : pDstData[n] =
2639 11341 : pSrcData[n] >= 255 ? 255 : static_cast<uint8_t>(pSrcData[n]);
2640 5062 : }
2641 : }
2642 : else
2643 : {
2644 975 : GDALCopyWordsGenericT(pSrcData, nSrcPixelStride, pDstData,
2645 : nDstPixelStride, nWordCount);
2646 : }
2647 6037 : }
2648 :
2649 : template <>
2650 46 : CPL_NOINLINE void GDALCopyWordsT(const uint16_t *const CPL_RESTRICT pSrcData,
2651 : int nSrcPixelStride,
2652 : int16_t *const CPL_RESTRICT pDstData,
2653 : int nDstPixelStride, GPtrDiff_t nWordCount)
2654 : {
2655 46 : if (nSrcPixelStride == static_cast<int>(sizeof(*pSrcData)) &&
2656 : nDstPixelStride == static_cast<int>(sizeof(*pDstData)))
2657 : {
2658 40 : decltype(nWordCount) n = 0;
2659 : #if defined(__SSE4_1__) || defined(USE_NEON_OPTIMIZATIONS)
2660 : const __m128i xmm_MAX_INT16 = _mm_set1_epi16(32767);
2661 : #else
2662 : // In SSE2, min_epu16 does not exist, so shift from
2663 : // UInt16 to SInt16 to be able to use min_epi16
2664 40 : const __m128i xmm_UINT16_to_INT16 = _mm_set1_epi16(-32768);
2665 40 : const __m128i xmm_32767_shifted = _mm_set1_epi16(32767 - 32768);
2666 : #endif
2667 169 : for (; n < nWordCount - 15; n += 16)
2668 : {
2669 258 : __m128i xmm0 = _mm_loadu_si128(
2670 129 : reinterpret_cast<const __m128i *>(pSrcData + n));
2671 129 : __m128i xmm1 = _mm_loadu_si128(
2672 129 : reinterpret_cast<const __m128i *>(pSrcData + n + 8));
2673 : #if defined(__SSE4_1__) || defined(USE_NEON_OPTIMIZATIONS)
2674 : xmm0 = _mm_min_epu16(xmm0, xmm_MAX_INT16);
2675 : xmm1 = _mm_min_epu16(xmm1, xmm_MAX_INT16);
2676 : #else
2677 129 : xmm0 = _mm_add_epi16(xmm0, xmm_UINT16_to_INT16);
2678 129 : xmm1 = _mm_add_epi16(xmm1, xmm_UINT16_to_INT16);
2679 129 : xmm0 = _mm_min_epi16(xmm0, xmm_32767_shifted);
2680 129 : xmm1 = _mm_min_epi16(xmm1, xmm_32767_shifted);
2681 129 : xmm0 = _mm_sub_epi16(xmm0, xmm_UINT16_to_INT16);
2682 129 : xmm1 = _mm_sub_epi16(xmm1, xmm_UINT16_to_INT16);
2683 : #endif
2684 129 : _mm_storeu_si128(reinterpret_cast<__m128i *>(pDstData + n), xmm0);
2685 129 : _mm_storeu_si128(reinterpret_cast<__m128i *>(pDstData + n + 8),
2686 : xmm1);
2687 : }
2688 191 : for (; n < nWordCount; n++)
2689 : {
2690 282 : pDstData[n] = pSrcData[n] >= 32767
2691 : ? 32767
2692 131 : : static_cast<int16_t>(pSrcData[n]);
2693 40 : }
2694 : }
2695 : else
2696 : {
2697 6 : GDALCopyWordsGenericT(pSrcData, nSrcPixelStride, pDstData,
2698 : nDstPixelStride, nWordCount);
2699 : }
2700 46 : }
2701 :
2702 : template <>
2703 135 : CPL_NOINLINE void GDALCopyWordsT(const int16_t *const CPL_RESTRICT pSrcData,
2704 : int nSrcPixelStride,
2705 : uint16_t *const CPL_RESTRICT pDstData,
2706 : int nDstPixelStride, GPtrDiff_t nWordCount)
2707 : {
2708 135 : if (nSrcPixelStride == static_cast<int>(sizeof(*pSrcData)) &&
2709 : nDstPixelStride == static_cast<int>(sizeof(*pDstData)))
2710 : {
2711 92 : decltype(nWordCount) n = 0;
2712 92 : const __m128i xmm_zero = _mm_setzero_si128();
2713 277 : for (; n < nWordCount - 15; n += 16)
2714 : {
2715 370 : __m128i xmm0 = _mm_loadu_si128(
2716 185 : reinterpret_cast<const __m128i *>(pSrcData + n));
2717 185 : __m128i xmm1 = _mm_loadu_si128(
2718 185 : reinterpret_cast<const __m128i *>(pSrcData + n + 8));
2719 185 : xmm0 = _mm_max_epi16(xmm0, xmm_zero);
2720 185 : xmm1 = _mm_max_epi16(xmm1, xmm_zero);
2721 185 : _mm_storeu_si128(reinterpret_cast<__m128i *>(pDstData + n), xmm0);
2722 185 : _mm_storeu_si128(reinterpret_cast<__m128i *>(pDstData + n + 8),
2723 : xmm1);
2724 : }
2725 468 : for (; n < nWordCount; n++)
2726 : {
2727 376 : pDstData[n] =
2728 376 : pSrcData[n] < 0 ? 0 : static_cast<uint16_t>(pSrcData[n]);
2729 92 : }
2730 : }
2731 : else
2732 : {
2733 43 : GDALCopyWordsGenericT(pSrcData, nSrcPixelStride, pDstData,
2734 : nDstPixelStride, nWordCount);
2735 : }
2736 135 : }
2737 :
2738 : #if defined(__SSE4_1__) || defined(USE_NEON_OPTIMIZATIONS)
2739 :
2740 : template <>
2741 : CPL_NOINLINE void GDALCopyWordsT(const uint32_t *const CPL_RESTRICT pSrcData,
2742 : int nSrcPixelStride,
2743 : int32_t *const CPL_RESTRICT pDstData,
2744 : int nDstPixelStride, GPtrDiff_t nWordCount)
2745 : {
2746 : if (nSrcPixelStride == static_cast<int>(sizeof(*pSrcData)) &&
2747 : nDstPixelStride == static_cast<int>(sizeof(*pDstData)))
2748 : {
2749 : decltype(nWordCount) n = 0;
2750 : const __m128i xmm_MAX_INT = _mm_set1_epi32(INT_MAX);
2751 : for (; n < nWordCount - 8; n += 7)
2752 : {
2753 : __m128i xmm0 = _mm_loadu_si128(
2754 : reinterpret_cast<const __m128i *>(pSrcData + n));
2755 : __m128i xmm1 = _mm_loadu_si128(
2756 : reinterpret_cast<const __m128i *>(pSrcData + n + 4));
2757 : xmm0 = _mm_min_epu32(xmm0, xmm_MAX_INT);
2758 : xmm1 = _mm_min_epu32(xmm1, xmm_MAX_INT);
2759 : _mm_storeu_si128(reinterpret_cast<__m128i *>(pDstData + n), xmm0);
2760 : _mm_storeu_si128(reinterpret_cast<__m128i *>(pDstData + n + 4),
2761 : xmm1);
2762 : }
2763 : for (; n < nWordCount; n++)
2764 : {
2765 : pDstData[n] = pSrcData[n] >= INT_MAX
2766 : ? INT_MAX
2767 : : static_cast<int32_t>(pSrcData[n]);
2768 : }
2769 : }
2770 : else
2771 : {
2772 : GDALCopyWordsGenericT(pSrcData, nSrcPixelStride, pDstData,
2773 : nDstPixelStride, nWordCount);
2774 : }
2775 : }
2776 :
2777 : template <>
2778 : CPL_NOINLINE void GDALCopyWordsT(const int32_t *const CPL_RESTRICT pSrcData,
2779 : int nSrcPixelStride,
2780 : uint32_t *const CPL_RESTRICT pDstData,
2781 : int nDstPixelStride, GPtrDiff_t nWordCount)
2782 : {
2783 : if (nSrcPixelStride == static_cast<int>(sizeof(*pSrcData)) &&
2784 : nDstPixelStride == static_cast<int>(sizeof(*pDstData)))
2785 : {
2786 : decltype(nWordCount) n = 0;
2787 : const __m128i xmm_zero = _mm_setzero_si128();
2788 : for (; n < nWordCount - 7; n += 8)
2789 : {
2790 : __m128i xmm0 = _mm_loadu_si128(
2791 : reinterpret_cast<const __m128i *>(pSrcData + n));
2792 : __m128i xmm1 = _mm_loadu_si128(
2793 : reinterpret_cast<const __m128i *>(pSrcData + n + 4));
2794 : xmm0 = _mm_max_epi32(xmm0, xmm_zero);
2795 : xmm1 = _mm_max_epi32(xmm1, xmm_zero);
2796 : _mm_storeu_si128(reinterpret_cast<__m128i *>(pDstData + n), xmm0);
2797 : _mm_storeu_si128(reinterpret_cast<__m128i *>(pDstData + n + 4),
2798 : xmm1);
2799 : }
2800 : for (; n < nWordCount; n++)
2801 : {
2802 : pDstData[n] =
2803 : pSrcData[n] < 0 ? 0 : static_cast<uint32_t>(pSrcData[n]);
2804 : }
2805 : }
2806 : else
2807 : {
2808 : GDALCopyWordsGenericT(pSrcData, nSrcPixelStride, pDstData,
2809 : nDstPixelStride, nWordCount);
2810 : }
2811 : }
2812 :
2813 : #endif // defined(__SSE4_1__) || defined(USE_NEON_OPTIMIZATIONS)
2814 :
2815 : template <>
2816 339 : CPL_NOINLINE void GDALCopyWordsT(const uint16_t *const CPL_RESTRICT pSrcData,
2817 : int nSrcPixelStride,
2818 : float *const CPL_RESTRICT pDstData,
2819 : int nDstPixelStride, GPtrDiff_t nWordCount)
2820 : {
2821 339 : if (nSrcPixelStride == static_cast<int>(sizeof(*pSrcData)) &&
2822 : nDstPixelStride == static_cast<int>(sizeof(*pDstData)))
2823 : {
2824 333 : decltype(nWordCount) n = 0;
2825 333 : const __m128i xmm_zero = _mm_setzero_si128();
2826 333 : GByte *CPL_RESTRICT pabyDstDataPtr =
2827 : reinterpret_cast<GByte *>(pDstData);
2828 1472 : for (; n < nWordCount - 7; n += 8)
2829 : {
2830 1139 : __m128i xmm = _mm_loadu_si128(
2831 1139 : reinterpret_cast<const __m128i *>(pSrcData + n));
2832 1139 : __m128i xmm0 = _mm_unpacklo_epi16(xmm, xmm_zero);
2833 1139 : __m128i xmm1 = _mm_unpackhi_epi16(xmm, xmm_zero);
2834 1139 : __m128 xmm0_f = _mm_cvtepi32_ps(xmm0);
2835 1139 : __m128 xmm1_f = _mm_cvtepi32_ps(xmm1);
2836 1139 : _mm_storeu_ps(reinterpret_cast<float *>(pabyDstDataPtr + n * 4),
2837 : xmm0_f);
2838 : _mm_storeu_ps(
2839 1139 : reinterpret_cast<float *>(pabyDstDataPtr + n * 4 + 16), xmm1_f);
2840 : }
2841 1099 : for (; n < nWordCount; n++)
2842 : {
2843 766 : pDstData[n] = pSrcData[n];
2844 333 : }
2845 : }
2846 : else
2847 : {
2848 6 : GDALCopyWordsGenericT(pSrcData, nSrcPixelStride, pDstData,
2849 : nDstPixelStride, nWordCount);
2850 : }
2851 339 : }
2852 :
2853 : template <>
2854 1072750 : CPL_NOINLINE void GDALCopyWordsT(const int16_t *const CPL_RESTRICT pSrcData,
2855 : int nSrcPixelStride,
2856 : float *const CPL_RESTRICT pDstData,
2857 : int nDstPixelStride, GPtrDiff_t nWordCount)
2858 : {
2859 1072750 : if (nSrcPixelStride == static_cast<int>(sizeof(*pSrcData)) &&
2860 : nDstPixelStride == static_cast<int>(sizeof(*pDstData)))
2861 : {
2862 82850 : decltype(nWordCount) n = 0;
2863 82850 : GByte *CPL_RESTRICT pabyDstDataPtr =
2864 : reinterpret_cast<GByte *>(pDstData);
2865 553615 : for (; n < nWordCount - 7; n += 8)
2866 : {
2867 470765 : __m128i xmm = _mm_loadu_si128(
2868 470765 : reinterpret_cast<const __m128i *>(pSrcData + n));
2869 470765 : const auto sign = _mm_srai_epi16(xmm, 15);
2870 470765 : __m128i xmm0 = _mm_unpacklo_epi16(xmm, sign);
2871 470765 : __m128i xmm1 = _mm_unpackhi_epi16(xmm, sign);
2872 470765 : __m128 xmm0_f = _mm_cvtepi32_ps(xmm0);
2873 470765 : __m128 xmm1_f = _mm_cvtepi32_ps(xmm1);
2874 470765 : _mm_storeu_ps(reinterpret_cast<float *>(pabyDstDataPtr + n * 4),
2875 : xmm0_f);
2876 : _mm_storeu_ps(
2877 470765 : reinterpret_cast<float *>(pabyDstDataPtr + n * 4 + 16), xmm1_f);
2878 : }
2879 242713 : for (; n < nWordCount; n++)
2880 : {
2881 159863 : pDstData[n] = pSrcData[n];
2882 82850 : }
2883 : }
2884 : else
2885 : {
2886 989901 : GDALCopyWordsGenericT(pSrcData, nSrcPixelStride, pDstData,
2887 : nDstPixelStride, nWordCount);
2888 : }
2889 1072750 : }
2890 :
2891 : template <>
2892 380 : CPL_NOINLINE void GDALCopyWordsT(const uint16_t *const CPL_RESTRICT pSrcData,
2893 : int nSrcPixelStride,
2894 : double *const CPL_RESTRICT pDstData,
2895 : int nDstPixelStride, GPtrDiff_t nWordCount)
2896 : {
2897 380 : if (nSrcPixelStride == static_cast<int>(sizeof(*pSrcData)) &&
2898 : nDstPixelStride == static_cast<int>(sizeof(*pDstData)))
2899 : {
2900 269 : decltype(nWordCount) n = 0;
2901 269 : const __m128i xmm_zero = _mm_setzero_si128();
2902 269 : GByte *CPL_RESTRICT pabyDstDataPtr =
2903 : reinterpret_cast<GByte *>(pDstData);
2904 713 : for (; n < nWordCount - 7; n += 8)
2905 : {
2906 444 : __m128i xmm = _mm_loadu_si128(
2907 444 : reinterpret_cast<const __m128i *>(pSrcData + n));
2908 444 : __m128i xmm0 = _mm_unpacklo_epi16(xmm, xmm_zero);
2909 444 : __m128i xmm1 = _mm_unpackhi_epi16(xmm, xmm_zero);
2910 :
2911 444 : __m128d xmm0_low_d = _mm_cvtepi32_pd(xmm0);
2912 444 : __m128d xmm1_low_d = _mm_cvtepi32_pd(xmm1);
2913 444 : xmm0 = _mm_srli_si128(xmm0, 8);
2914 444 : xmm1 = _mm_srli_si128(xmm1, 8);
2915 444 : __m128d xmm0_high_d = _mm_cvtepi32_pd(xmm0);
2916 444 : __m128d xmm1_high_d = _mm_cvtepi32_pd(xmm1);
2917 :
2918 444 : _mm_storeu_pd(reinterpret_cast<double *>(pabyDstDataPtr + n * 8),
2919 : xmm0_low_d);
2920 : _mm_storeu_pd(
2921 444 : reinterpret_cast<double *>(pabyDstDataPtr + n * 8 + 16),
2922 : xmm0_high_d);
2923 : _mm_storeu_pd(
2924 444 : reinterpret_cast<double *>(pabyDstDataPtr + n * 8 + 32),
2925 : xmm1_low_d);
2926 : _mm_storeu_pd(
2927 444 : reinterpret_cast<double *>(pabyDstDataPtr + n * 8 + 48),
2928 : xmm1_high_d);
2929 : }
2930 918 : for (; n < nWordCount; n++)
2931 : {
2932 649 : pDstData[n] = pSrcData[n];
2933 269 : }
2934 : }
2935 : else
2936 : {
2937 111 : GDALCopyWordsGenericT(pSrcData, nSrcPixelStride, pDstData,
2938 : nDstPixelStride, nWordCount);
2939 : }
2940 380 : }
2941 :
2942 : template <>
2943 2760110 : CPL_NOINLINE void GDALCopyWordsT(const int16_t *const CPL_RESTRICT pSrcData,
2944 : int nSrcPixelStride,
2945 : double *const CPL_RESTRICT pDstData,
2946 : int nDstPixelStride, GPtrDiff_t nWordCount)
2947 : {
2948 2760110 : if (nSrcPixelStride == static_cast<int>(sizeof(*pSrcData)) &&
2949 : nDstPixelStride == static_cast<int>(sizeof(*pDstData)))
2950 : {
2951 34420 : decltype(nWordCount) n = 0;
2952 34420 : GByte *CPL_RESTRICT pabyDstDataPtr =
2953 : reinterpret_cast<GByte *>(pDstData);
2954 400969 : for (; n < nWordCount - 7; n += 8)
2955 : {
2956 366668 : __m128i xmm = _mm_loadu_si128(
2957 366668 : reinterpret_cast<const __m128i *>(pSrcData + n));
2958 366570 : const auto sign = _mm_srai_epi16(xmm, 15);
2959 366536 : __m128i xmm0 = _mm_unpacklo_epi16(xmm, sign);
2960 366505 : __m128i xmm1 = _mm_unpackhi_epi16(xmm, sign);
2961 :
2962 366493 : __m128d xmm0_low_d = _mm_cvtepi32_pd(xmm0);
2963 366360 : __m128d xmm1_low_d = _mm_cvtepi32_pd(xmm1);
2964 366360 : xmm0 = _mm_srli_si128(xmm0, 8);
2965 366431 : xmm1 = _mm_srli_si128(xmm1, 8);
2966 366524 : __m128d xmm0_high_d = _mm_cvtepi32_pd(xmm0);
2967 366549 : __m128d xmm1_high_d = _mm_cvtepi32_pd(xmm1);
2968 :
2969 366549 : _mm_storeu_pd(reinterpret_cast<double *>(pabyDstDataPtr + n * 8),
2970 : xmm0_low_d);
2971 : _mm_storeu_pd(
2972 366549 : reinterpret_cast<double *>(pabyDstDataPtr + n * 8 + 16),
2973 : xmm0_high_d);
2974 : _mm_storeu_pd(
2975 366549 : reinterpret_cast<double *>(pabyDstDataPtr + n * 8 + 32),
2976 : xmm1_low_d);
2977 : _mm_storeu_pd(
2978 366549 : reinterpret_cast<double *>(pabyDstDataPtr + n * 8 + 48),
2979 : xmm1_high_d);
2980 : }
2981 252098 : for (; n < nWordCount; n++)
2982 : {
2983 217797 : pDstData[n] = pSrcData[n];
2984 34301 : }
2985 : }
2986 : else
2987 : {
2988 2725690 : GDALCopyWordsGenericT(pSrcData, nSrcPixelStride, pDstData,
2989 : nDstPixelStride, nWordCount);
2990 : }
2991 2759990 : }
2992 :
2993 : template <>
2994 4388770 : CPL_NOINLINE void GDALCopyWordsT(const double *const CPL_RESTRICT pSrcData,
2995 : int nSrcPixelStride,
2996 : GByte *const CPL_RESTRICT pDstData,
2997 : int nDstPixelStride, GPtrDiff_t nWordCount)
2998 : {
2999 4388770 : GDALCopyWordsT_8atatime(pSrcData, nSrcPixelStride, pDstData,
3000 : nDstPixelStride, nWordCount);
3001 4388800 : }
3002 :
3003 : template <>
3004 38353 : CPL_NOINLINE void GDALCopyWordsT(const double *const CPL_RESTRICT pSrcData,
3005 : int nSrcPixelStride,
3006 : GUInt16 *const CPL_RESTRICT pDstData,
3007 : int nDstPixelStride, GPtrDiff_t nWordCount)
3008 : {
3009 38353 : GDALCopyWordsT_8atatime(pSrcData, nSrcPixelStride, pDstData,
3010 : nDstPixelStride, nWordCount);
3011 38353 : }
3012 :
3013 : template <>
3014 51557 : CPL_NOINLINE void GDALCopyWordsT(const float *const CPL_RESTRICT pSrcData,
3015 : int nSrcPixelStride,
3016 : double *const CPL_RESTRICT pDstData,
3017 : int nDstPixelStride, GPtrDiff_t nWordCount)
3018 : {
3019 51557 : GDALCopyWordsT_8atatime(pSrcData, nSrcPixelStride, pDstData,
3020 : nDstPixelStride, nWordCount);
3021 51557 : }
3022 :
3023 : template <>
3024 122467 : CPL_NOINLINE void GDALCopyWordsT(const double *const CPL_RESTRICT pSrcData,
3025 : int nSrcPixelStride,
3026 : float *const CPL_RESTRICT pDstData,
3027 : int nDstPixelStride, GPtrDiff_t nWordCount)
3028 : {
3029 122467 : GDALCopyWordsT_8atatime(pSrcData, nSrcPixelStride, pDstData,
3030 : nDstPixelStride, nWordCount);
3031 122466 : }
3032 :
3033 : template <>
3034 111 : CPL_NOINLINE void GDALCopyWordsT(const GFloat16 *const CPL_RESTRICT pSrcData,
3035 : int nSrcPixelStride,
3036 : float *const CPL_RESTRICT pDstData,
3037 : int nDstPixelStride, GPtrDiff_t nWordCount)
3038 : {
3039 111 : GDALCopyWordsT_8atatime(pSrcData, nSrcPixelStride, pDstData,
3040 : nDstPixelStride, nWordCount);
3041 111 : }
3042 :
3043 : template <>
3044 532 : CPL_NOINLINE void GDALCopyWordsT(const GFloat16 *const CPL_RESTRICT pSrcData,
3045 : int nSrcPixelStride,
3046 : double *const CPL_RESTRICT pDstData,
3047 : int nDstPixelStride, GPtrDiff_t nWordCount)
3048 : {
3049 532 : GDALCopyWordsT_8atatime(pSrcData, nSrcPixelStride, pDstData,
3050 : nDstPixelStride, nWordCount);
3051 532 : }
3052 :
3053 : #ifdef __F16C__
3054 :
3055 : template <>
3056 : CPL_NOINLINE void GDALCopyWordsT(const float *const CPL_RESTRICT pSrcData,
3057 : int nSrcPixelStride,
3058 : GFloat16 *const CPL_RESTRICT pDstData,
3059 : int nDstPixelStride, GPtrDiff_t nWordCount)
3060 : {
3061 : GDALCopyWordsT_8atatime(pSrcData, nSrcPixelStride, pDstData,
3062 : nDstPixelStride, nWordCount);
3063 : }
3064 :
3065 : template <>
3066 : CPL_NOINLINE void GDALCopyWordsT(const double *const CPL_RESTRICT pSrcData,
3067 : int nSrcPixelStride,
3068 : GFloat16 *const CPL_RESTRICT pDstData,
3069 : int nDstPixelStride, GPtrDiff_t nWordCount)
3070 : {
3071 : GDALCopyWordsT_8atatime(pSrcData, nSrcPixelStride, pDstData,
3072 : nDstPixelStride, nWordCount);
3073 : }
3074 :
3075 : #endif // __F16C__
3076 :
3077 : #endif // HAVE_SSE2
3078 :
3079 : template <>
3080 233093 : CPL_NOINLINE void GDALCopyWordsT(const float *const CPL_RESTRICT pSrcData,
3081 : int nSrcPixelStride,
3082 : GByte *const CPL_RESTRICT pDstData,
3083 : int nDstPixelStride, GPtrDiff_t nWordCount)
3084 : {
3085 233093 : GDALCopyWordsT_8atatime(pSrcData, nSrcPixelStride, pDstData,
3086 : nDstPixelStride, nWordCount);
3087 233093 : }
3088 :
3089 : template <>
3090 15775 : CPL_NOINLINE void GDALCopyWordsT(const float *const CPL_RESTRICT pSrcData,
3091 : int nSrcPixelStride,
3092 : GInt16 *const CPL_RESTRICT pDstData,
3093 : int nDstPixelStride, GPtrDiff_t nWordCount)
3094 : {
3095 15775 : GDALCopyWordsT_8atatime(pSrcData, nSrcPixelStride, pDstData,
3096 : nDstPixelStride, nWordCount);
3097 15775 : }
3098 :
3099 : template <>
3100 61689 : CPL_NOINLINE void GDALCopyWordsT(const float *const CPL_RESTRICT pSrcData,
3101 : int nSrcPixelStride,
3102 : GUInt16 *const CPL_RESTRICT pDstData,
3103 : int nDstPixelStride, GPtrDiff_t nWordCount)
3104 : {
3105 61689 : GDALCopyWordsT_8atatime(pSrcData, nSrcPixelStride, pDstData,
3106 : nDstPixelStride, nWordCount);
3107 61685 : }
3108 :
3109 : /************************************************************************/
3110 : /* GDALCopyWordsComplexT() */
3111 : /************************************************************************/
3112 : /**
3113 : * Template function, used to copy data from pSrcData into buffer
3114 : * pDstData, with stride nSrcPixelStride in the source data and
3115 : * stride nDstPixelStride in the destination data. Deals with the
3116 : * complex case, where input is complex and output is complex.
3117 : *
3118 : * @param pSrcData the source data buffer
3119 : * @param nSrcPixelStride the stride, in the buffer pSrcData for pixels
3120 : * of interest.
3121 : * @param pDstData the destination buffer.
3122 : * @param nDstPixelStride the stride in the buffer pDstData for pixels of
3123 : * interest.
3124 : * @param nWordCount the total number of pixel words to copy
3125 : *
3126 : */
3127 : template <class Tin, class Tout>
3128 96717 : inline void GDALCopyWordsComplexT(const Tin *const CPL_RESTRICT pSrcData,
3129 : int nSrcPixelStride,
3130 : Tout *const CPL_RESTRICT pDstData,
3131 : int nDstPixelStride, GPtrDiff_t nWordCount)
3132 : {
3133 96717 : decltype(nWordCount) nDstOffset = 0;
3134 96717 : const char *const pSrcDataPtr = reinterpret_cast<const char *>(pSrcData);
3135 96717 : char *const pDstDataPtr = reinterpret_cast<char *>(pDstData);
3136 :
3137 5243171 : for (decltype(nWordCount) n = 0; n < nWordCount; n++)
3138 : {
3139 5146449 : const Tin *const pPixelIn =
3140 5146449 : reinterpret_cast<const Tin *>(pSrcDataPtr + n * nSrcPixelStride);
3141 5146449 : Tout *const pPixelOut =
3142 5146449 : reinterpret_cast<Tout *>(pDstDataPtr + nDstOffset);
3143 :
3144 5146449 : GDALCopyWord(pPixelIn[0], pPixelOut[0]);
3145 5146449 : GDALCopyWord(pPixelIn[1], pPixelOut[1]);
3146 :
3147 5146449 : nDstOffset += nDstPixelStride;
3148 : }
3149 96717 : }
3150 :
3151 : /************************************************************************/
3152 : /* GDALCopyWordsComplexOutT() */
3153 : /************************************************************************/
3154 : /**
3155 : * Template function, used to copy data from pSrcData into buffer
3156 : * pDstData, with stride nSrcPixelStride in the source data and
3157 : * stride nDstPixelStride in the destination data. Deals with the
3158 : * case where the value is real coming in, but complex going out.
3159 : *
3160 : * @param pSrcData the source data buffer
3161 : * @param nSrcPixelStride the stride, in the buffer pSrcData for pixels
3162 : * of interest, in bytes.
3163 : * @param pDstData the destination buffer.
3164 : * @param nDstPixelStride the stride in the buffer pDstData for pixels of
3165 : * interest, in bytes.
3166 : * @param nWordCount the total number of pixel words to copy
3167 : *
3168 : */
3169 : template <class Tin, class Tout>
3170 3877 : inline void GDALCopyWordsComplexOutT(const Tin *const CPL_RESTRICT pSrcData,
3171 : int nSrcPixelStride,
3172 : Tout *const CPL_RESTRICT pDstData,
3173 : int nDstPixelStride, GPtrDiff_t nWordCount)
3174 : {
3175 3877 : decltype(nWordCount) nDstOffset = 0;
3176 :
3177 3877 : const Tout tOutZero = static_cast<Tout>(0);
3178 :
3179 3877 : const char *const pSrcDataPtr = reinterpret_cast<const char *>(pSrcData);
3180 3877 : char *const pDstDataPtr = reinterpret_cast<char *>(pDstData);
3181 :
3182 1155414 : for (decltype(nWordCount) n = 0; n < nWordCount; n++)
3183 : {
3184 1151537 : const Tin tValue =
3185 1151537 : *reinterpret_cast<const Tin *>(pSrcDataPtr + n * nSrcPixelStride);
3186 1151537 : Tout *const pPixelOut =
3187 1151537 : reinterpret_cast<Tout *>(pDstDataPtr + nDstOffset);
3188 1151537 : GDALCopyWord(tValue, *pPixelOut);
3189 :
3190 1151537 : pPixelOut[1] = tOutZero;
3191 :
3192 1151537 : nDstOffset += nDstPixelStride;
3193 : }
3194 3877 : }
3195 :
3196 : /************************************************************************/
3197 : /* GDALCopyWordsFromT() */
3198 : /************************************************************************/
3199 : /**
3200 : * Template driver function. Given the input type T, call the appropriate
3201 : * GDALCopyWordsT function template for the desired output type. You should
3202 : * never call this function directly (call GDALCopyWords instead).
3203 : *
3204 : * @param pSrcData source data buffer
3205 : * @param nSrcPixelStride pixel stride in input buffer, in pixel words
3206 : * @param bInComplex input is complex
3207 : * @param pDstData destination data buffer
3208 : * @param eDstType destination data type
3209 : * @param nDstPixelStride pixel stride in output buffer, in pixel words
3210 : * @param nWordCount number of pixel words to be copied
3211 : */
3212 : template <class T>
3213 54037508 : inline void GDALCopyWordsFromT(const T *const CPL_RESTRICT pSrcData,
3214 : int nSrcPixelStride, bool bInComplex,
3215 : void *CPL_RESTRICT pDstData,
3216 : GDALDataType eDstType, int nDstPixelStride,
3217 : GPtrDiff_t nWordCount)
3218 : {
3219 54037508 : switch (eDstType)
3220 : {
3221 4666524 : case GDT_Byte:
3222 4666524 : GDALCopyWordsT(pSrcData, nSrcPixelStride,
3223 : static_cast<unsigned char *>(pDstData),
3224 : nDstPixelStride, nWordCount);
3225 4666606 : break;
3226 751 : case GDT_Int8:
3227 751 : GDALCopyWordsT(pSrcData, nSrcPixelStride,
3228 : static_cast<signed char *>(pDstData),
3229 : nDstPixelStride, nWordCount);
3230 751 : break;
3231 140705 : case GDT_UInt16:
3232 140705 : GDALCopyWordsT(pSrcData, nSrcPixelStride,
3233 : static_cast<unsigned short *>(pDstData),
3234 : nDstPixelStride, nWordCount);
3235 140702 : break;
3236 4162813 : case GDT_Int16:
3237 4162813 : GDALCopyWordsT(pSrcData, nSrcPixelStride,
3238 : static_cast<short *>(pDstData), nDstPixelStride,
3239 : nWordCount);
3240 4162813 : break;
3241 22239 : case GDT_UInt32:
3242 22239 : GDALCopyWordsT(pSrcData, nSrcPixelStride,
3243 : static_cast<unsigned int *>(pDstData),
3244 : nDstPixelStride, nWordCount);
3245 22239 : break;
3246 26034766 : case GDT_Int32:
3247 26034766 : GDALCopyWordsT(pSrcData, nSrcPixelStride,
3248 : static_cast<int *>(pDstData), nDstPixelStride,
3249 : nWordCount);
3250 26044869 : break;
3251 809 : case GDT_UInt64:
3252 809 : GDALCopyWordsT(pSrcData, nSrcPixelStride,
3253 : static_cast<std::uint64_t *>(pDstData),
3254 : nDstPixelStride, nWordCount);
3255 809 : break;
3256 5181 : case GDT_Int64:
3257 5181 : GDALCopyWordsT(pSrcData, nSrcPixelStride,
3258 : static_cast<std::int64_t *>(pDstData),
3259 : nDstPixelStride, nWordCount);
3260 5181 : break;
3261 940 : case GDT_Float16:
3262 940 : GDALCopyWordsT(pSrcData, nSrcPixelStride,
3263 : static_cast<GFloat16 *>(pDstData), nDstPixelStride,
3264 : nWordCount);
3265 940 : break;
3266 3700690 : case GDT_Float32:
3267 3700690 : GDALCopyWordsT(pSrcData, nSrcPixelStride,
3268 : static_cast<float *>(pDstData), nDstPixelStride,
3269 : nWordCount);
3270 3700690 : break;
3271 15195964 : case GDT_Float64:
3272 15195964 : GDALCopyWordsT(pSrcData, nSrcPixelStride,
3273 : static_cast<double *>(pDstData), nDstPixelStride,
3274 : nWordCount);
3275 15195984 : break;
3276 94123 : case GDT_CInt16:
3277 94123 : if (bInComplex)
3278 : {
3279 92870 : GDALCopyWordsComplexT(pSrcData, nSrcPixelStride,
3280 : static_cast<short *>(pDstData),
3281 : nDstPixelStride, nWordCount);
3282 : }
3283 : else // input is not complex, so we need to promote to a complex
3284 : // buffer
3285 : {
3286 1253 : GDALCopyWordsComplexOutT(pSrcData, nSrcPixelStride,
3287 : static_cast<short *>(pDstData),
3288 : nDstPixelStride, nWordCount);
3289 : }
3290 94123 : break;
3291 1052 : case GDT_CInt32:
3292 1052 : if (bInComplex)
3293 : {
3294 421 : GDALCopyWordsComplexT(pSrcData, nSrcPixelStride,
3295 : static_cast<int *>(pDstData),
3296 : nDstPixelStride, nWordCount);
3297 : }
3298 : else // input is not complex, so we need to promote to a complex
3299 : // buffer
3300 : {
3301 631 : GDALCopyWordsComplexOutT(pSrcData, nSrcPixelStride,
3302 : static_cast<int *>(pDstData),
3303 : nDstPixelStride, nWordCount);
3304 : }
3305 1052 : break;
3306 281 : case GDT_CFloat16:
3307 281 : if (bInComplex)
3308 : {
3309 16 : GDALCopyWordsComplexT(pSrcData, nSrcPixelStride,
3310 : static_cast<GFloat16 *>(pDstData),
3311 : nDstPixelStride, nWordCount);
3312 : }
3313 : else // input is not complex, so we need to promote to a complex
3314 : // buffer
3315 : {
3316 265 : GDALCopyWordsComplexOutT(pSrcData, nSrcPixelStride,
3317 : static_cast<GFloat16 *>(pDstData),
3318 : nDstPixelStride, nWordCount);
3319 : }
3320 281 : break;
3321 3359 : case GDT_CFloat32:
3322 3359 : if (bInComplex)
3323 : {
3324 2564 : GDALCopyWordsComplexT(pSrcData, nSrcPixelStride,
3325 : static_cast<float *>(pDstData),
3326 : nDstPixelStride, nWordCount);
3327 : }
3328 : else // input is not complex, so we need to promote to a complex
3329 : // buffer
3330 : {
3331 795 : GDALCopyWordsComplexOutT(pSrcData, nSrcPixelStride,
3332 : static_cast<float *>(pDstData),
3333 : nDstPixelStride, nWordCount);
3334 : }
3335 3359 : break;
3336 1779 : case GDT_CFloat64:
3337 1779 : if (bInComplex)
3338 : {
3339 846 : GDALCopyWordsComplexT(pSrcData, nSrcPixelStride,
3340 : static_cast<double *>(pDstData),
3341 : nDstPixelStride, nWordCount);
3342 : }
3343 : else // input is not complex, so we need to promote to a complex
3344 : // buffer
3345 : {
3346 933 : GDALCopyWordsComplexOutT(pSrcData, nSrcPixelStride,
3347 : static_cast<double *>(pDstData),
3348 : nDstPixelStride, nWordCount);
3349 : }
3350 1779 : break;
3351 0 : case GDT_Unknown:
3352 : case GDT_TypeCount:
3353 0 : CPLAssert(false);
3354 : }
3355 54047731 : }
3356 :
3357 : } // end anonymous namespace
3358 :
3359 : /************************************************************************/
3360 : /* GDALReplicateWord() */
3361 : /************************************************************************/
3362 :
3363 : template <class T>
3364 589658 : inline void GDALReplicateWordT(void *pDstData, int nDstPixelStride,
3365 : GPtrDiff_t nWordCount)
3366 : {
3367 589658 : const T valSet = *static_cast<const T *>(pDstData);
3368 589658 : if (nDstPixelStride == static_cast<int>(sizeof(T)))
3369 : {
3370 559925 : T *pDstPtr = static_cast<T *>(pDstData) + 1;
3371 20888585 : while (nWordCount >= 4)
3372 : {
3373 20328621 : nWordCount -= 4;
3374 20328621 : pDstPtr[0] = valSet;
3375 20328621 : pDstPtr[1] = valSet;
3376 20328621 : pDstPtr[2] = valSet;
3377 20328621 : pDstPtr[3] = valSet;
3378 20328621 : pDstPtr += 4;
3379 : }
3380 1460669 : while (nWordCount > 0)
3381 : {
3382 900744 : --nWordCount;
3383 900744 : *pDstPtr = valSet;
3384 900744 : pDstPtr++;
3385 : }
3386 : }
3387 : else
3388 : {
3389 29751 : GByte *pabyDstPtr = static_cast<GByte *>(pDstData) + nDstPixelStride;
3390 1040338 : while (nWordCount > 0)
3391 : {
3392 1010587 : --nWordCount;
3393 1010587 : *reinterpret_cast<T *>(pabyDstPtr) = valSet;
3394 1010587 : pabyDstPtr += nDstPixelStride;
3395 : }
3396 : }
3397 589658 : }
3398 :
3399 1011200 : static void GDALReplicateWord(const void *CPL_RESTRICT pSrcData,
3400 : GDALDataType eSrcType,
3401 : void *CPL_RESTRICT pDstData,
3402 : GDALDataType eDstType, int nDstPixelStride,
3403 : GPtrDiff_t nWordCount)
3404 : {
3405 : /* -----------------------------------------------------------------------
3406 : */
3407 : /* Special case when the source data is always the same value */
3408 : /* (for VRTSourcedRasterBand::IRasterIO and
3409 : * VRTDerivedRasterBand::IRasterIO*/
3410 : /* for example) */
3411 : /* -----------------------------------------------------------------------
3412 : */
3413 : // Let the general translation case do the necessary conversions
3414 : // on the first destination element.
3415 1011200 : GDALCopyWords64(pSrcData, eSrcType, 0, pDstData, eDstType, 0, 1);
3416 :
3417 : // Now copy the first element to the nWordCount - 1 following destination
3418 : // elements.
3419 1005450 : nWordCount--;
3420 1005450 : GByte *pabyDstWord = reinterpret_cast<GByte *>(pDstData) + nDstPixelStride;
3421 :
3422 1005450 : switch (eDstType)
3423 : {
3424 415594 : case GDT_Byte:
3425 : case GDT_Int8:
3426 : {
3427 415594 : if (nDstPixelStride == 1)
3428 : {
3429 375176 : if (nWordCount > 0)
3430 375176 : memset(pabyDstWord,
3431 375176 : *reinterpret_cast<const GByte *>(pDstData),
3432 : nWordCount);
3433 : }
3434 : else
3435 : {
3436 40418 : GByte valSet = *reinterpret_cast<const GByte *>(pDstData);
3437 31372000 : while (nWordCount > 0)
3438 : {
3439 31331600 : --nWordCount;
3440 31331600 : *pabyDstWord = valSet;
3441 31331600 : pabyDstWord += nDstPixelStride;
3442 : }
3443 : }
3444 415594 : break;
3445 : }
3446 :
3447 : #define CASE_DUPLICATE_SIMPLE(enum_type, c_type) \
3448 : case enum_type: \
3449 : { \
3450 : GDALReplicateWordT<c_type>(pDstData, nDstPixelStride, nWordCount); \
3451 : break; \
3452 : }
3453 :
3454 34497 : CASE_DUPLICATE_SIMPLE(GDT_UInt16, GUInt16)
3455 202438 : CASE_DUPLICATE_SIMPLE(GDT_Int16, GInt16)
3456 56 : CASE_DUPLICATE_SIMPLE(GDT_UInt32, GUInt32)
3457 293993 : CASE_DUPLICATE_SIMPLE(GDT_Int32, GInt32)
3458 21 : CASE_DUPLICATE_SIMPLE(GDT_UInt64, std::uint64_t)
3459 1064 : CASE_DUPLICATE_SIMPLE(GDT_Int64, std::int64_t)
3460 0 : CASE_DUPLICATE_SIMPLE(GDT_Float16, GFloat16)
3461 52564 : CASE_DUPLICATE_SIMPLE(GDT_Float32, float)
3462 5171 : CASE_DUPLICATE_SIMPLE(GDT_Float64, double)
3463 :
3464 : #define CASE_DUPLICATE_COMPLEX(enum_type, c_type) \
3465 : case enum_type: \
3466 : { \
3467 : c_type valSet1 = reinterpret_cast<const c_type *>(pDstData)[0]; \
3468 : c_type valSet2 = reinterpret_cast<const c_type *>(pDstData)[1]; \
3469 : while (nWordCount > 0) \
3470 : { \
3471 : --nWordCount; \
3472 : reinterpret_cast<c_type *>(pabyDstWord)[0] = valSet1; \
3473 : reinterpret_cast<c_type *>(pabyDstWord)[1] = valSet2; \
3474 : pabyDstWord += nDstPixelStride; \
3475 : } \
3476 : break; \
3477 : }
3478 :
3479 784 : CASE_DUPLICATE_COMPLEX(GDT_CInt16, GInt16)
3480 784 : CASE_DUPLICATE_COMPLEX(GDT_CInt32, GInt32)
3481 0 : CASE_DUPLICATE_COMPLEX(GDT_CFloat16, GFloat16)
3482 784 : CASE_DUPLICATE_COMPLEX(GDT_CFloat32, float)
3483 784 : CASE_DUPLICATE_COMPLEX(GDT_CFloat64, double)
3484 :
3485 0 : case GDT_Unknown:
3486 : case GDT_TypeCount:
3487 0 : CPLAssert(false);
3488 : }
3489 1011750 : }
3490 :
3491 : /************************************************************************/
3492 : /* GDALUnrolledCopy() */
3493 : /************************************************************************/
3494 :
3495 : template <class T, int srcStride, int dstStride>
3496 3266280 : static inline void GDALUnrolledCopyGeneric(T *CPL_RESTRICT pDest,
3497 : const T *CPL_RESTRICT pSrc,
3498 : GPtrDiff_t nIters)
3499 : {
3500 3266280 : if (nIters >= 16)
3501 : {
3502 137568057 : for (GPtrDiff_t i = nIters / 16; i != 0; i--)
3503 : {
3504 134428850 : pDest[0 * dstStride] = pSrc[0 * srcStride];
3505 134428850 : pDest[1 * dstStride] = pSrc[1 * srcStride];
3506 134428850 : pDest[2 * dstStride] = pSrc[2 * srcStride];
3507 134428850 : pDest[3 * dstStride] = pSrc[3 * srcStride];
3508 134428850 : pDest[4 * dstStride] = pSrc[4 * srcStride];
3509 134428850 : pDest[5 * dstStride] = pSrc[5 * srcStride];
3510 134428850 : pDest[6 * dstStride] = pSrc[6 * srcStride];
3511 134428850 : pDest[7 * dstStride] = pSrc[7 * srcStride];
3512 134428850 : pDest[8 * dstStride] = pSrc[8 * srcStride];
3513 134428850 : pDest[9 * dstStride] = pSrc[9 * srcStride];
3514 134428850 : pDest[10 * dstStride] = pSrc[10 * srcStride];
3515 134428850 : pDest[11 * dstStride] = pSrc[11 * srcStride];
3516 134428850 : pDest[12 * dstStride] = pSrc[12 * srcStride];
3517 134428850 : pDest[13 * dstStride] = pSrc[13 * srcStride];
3518 134428850 : pDest[14 * dstStride] = pSrc[14 * srcStride];
3519 134428850 : pDest[15 * dstStride] = pSrc[15 * srcStride];
3520 134428850 : pDest += 16 * dstStride;
3521 134428850 : pSrc += 16 * srcStride;
3522 : }
3523 3139175 : nIters = nIters % 16;
3524 : }
3525 5508138 : for (GPtrDiff_t i = 0; i < nIters; i++)
3526 : {
3527 2241845 : pDest[i * dstStride] = *pSrc;
3528 2241845 : pSrc += srcStride;
3529 : }
3530 3266280 : }
3531 :
3532 : template <class T, int srcStride, int dstStride>
3533 3260179 : static inline void GDALUnrolledCopy(T *CPL_RESTRICT pDest,
3534 : const T *CPL_RESTRICT pSrc,
3535 : GPtrDiff_t nIters)
3536 : {
3537 3260179 : GDALUnrolledCopyGeneric<T, srcStride, dstStride>(pDest, pSrc, nIters);
3538 3260179 : }
3539 :
3540 : #ifdef HAVE_SSE2
3541 :
3542 : template <>
3543 352920 : void GDALUnrolledCopy<GByte, 2, 1>(GByte *CPL_RESTRICT pDest,
3544 : const GByte *CPL_RESTRICT pSrc,
3545 : GPtrDiff_t nIters)
3546 : {
3547 352920 : decltype(nIters) i = 0;
3548 352920 : if (nIters > 16)
3549 : {
3550 194667 : const __m128i xmm_mask = _mm_set1_epi16(0xff);
3551 : // If we were sure that there would always be 1 trailing byte, we could
3552 : // check against nIters - 15
3553 2988110 : for (; i < nIters - 16; i += 16)
3554 : {
3555 : __m128i xmm0 =
3556 2793440 : _mm_loadu_si128(reinterpret_cast<__m128i const *>(pSrc + 0));
3557 : __m128i xmm1 =
3558 5586890 : _mm_loadu_si128(reinterpret_cast<__m128i const *>(pSrc + 16));
3559 : // Set higher 8bit of each int16 packed word to 0
3560 2793440 : xmm0 = _mm_and_si128(xmm0, xmm_mask);
3561 2793440 : xmm1 = _mm_and_si128(xmm1, xmm_mask);
3562 : // Pack int16 to uint8 and merge back both vector
3563 2793440 : xmm0 = _mm_packus_epi16(xmm0, xmm1);
3564 :
3565 : // Store result
3566 2793440 : _mm_storeu_si128(reinterpret_cast<__m128i *>(pDest + i), xmm0);
3567 :
3568 2793440 : pSrc += 2 * 16;
3569 : }
3570 : }
3571 4619940 : for (; i < nIters; i++)
3572 : {
3573 4267020 : pDest[i] = *pSrc;
3574 4267020 : pSrc += 2;
3575 : }
3576 352920 : }
3577 :
3578 : #ifdef HAVE_SSSE3_AT_COMPILE_TIME
3579 :
3580 : template <>
3581 191860 : void GDALUnrolledCopy<GByte, 3, 1>(GByte *CPL_RESTRICT pDest,
3582 : const GByte *CPL_RESTRICT pSrc,
3583 : GPtrDiff_t nIters)
3584 : {
3585 191860 : if (nIters > 16 && CPLHaveRuntimeSSSE3())
3586 : {
3587 185760 : GDALUnrolledCopy_GByte_3_1_SSSE3(pDest, pSrc, nIters);
3588 : }
3589 : else
3590 : {
3591 6100 : GDALUnrolledCopyGeneric<GByte, 3, 1>(pDest, pSrc, nIters);
3592 : }
3593 191860 : }
3594 :
3595 : #endif
3596 :
3597 : template <>
3598 106508 : void GDALUnrolledCopy<GByte, 4, 1>(GByte *CPL_RESTRICT pDest,
3599 : const GByte *CPL_RESTRICT pSrc,
3600 : GPtrDiff_t nIters)
3601 : {
3602 106508 : decltype(nIters) i = 0;
3603 106508 : if (nIters > 16)
3604 : {
3605 101215 : const __m128i xmm_mask = _mm_set1_epi32(0xff);
3606 : // If we were sure that there would always be 3 trailing bytes, we could
3607 : // check against nIters - 15
3608 10396100 : for (; i < nIters - 16; i += 16)
3609 : {
3610 : __m128i xmm0 =
3611 10295000 : _mm_loadu_si128(reinterpret_cast<__m128i const *>(pSrc + 0));
3612 : __m128i xmm1 =
3613 10295000 : _mm_loadu_si128(reinterpret_cast<__m128i const *>(pSrc + 16));
3614 : __m128i xmm2 =
3615 10295000 : _mm_loadu_si128(reinterpret_cast<__m128i const *>(pSrc + 32));
3616 : __m128i xmm3 =
3617 20590100 : _mm_loadu_si128(reinterpret_cast<__m128i const *>(pSrc + 48));
3618 : // Set higher 24bit of each int32 packed word to 0
3619 10295000 : xmm0 = _mm_and_si128(xmm0, xmm_mask);
3620 10295000 : xmm1 = _mm_and_si128(xmm1, xmm_mask);
3621 10295000 : xmm2 = _mm_and_si128(xmm2, xmm_mask);
3622 10295000 : xmm3 = _mm_and_si128(xmm3, xmm_mask);
3623 : // Pack int32 to int16
3624 10294700 : xmm0 = _mm_packs_epi32(xmm0, xmm1);
3625 10294900 : xmm2 = _mm_packs_epi32(xmm2, xmm3);
3626 : // Pack int16 to uint8
3627 10294900 : xmm0 = _mm_packus_epi16(xmm0, xmm2);
3628 :
3629 : // Store result
3630 10294900 : _mm_storeu_si128(reinterpret_cast<__m128i *>(pDest + i), xmm0);
3631 :
3632 10294900 : pSrc += 4 * 16;
3633 : }
3634 : }
3635 1138980 : for (; i < nIters; i++)
3636 : {
3637 1032610 : pDest[i] = *pSrc;
3638 1032610 : pSrc += 4;
3639 : }
3640 106371 : }
3641 : #endif // HAVE_SSE2
3642 :
3643 : /************************************************************************/
3644 : /* GDALFastCopy() */
3645 : /************************************************************************/
3646 :
3647 : template <class T>
3648 40423200 : static inline void GDALFastCopy(T *CPL_RESTRICT pDest, int nDestStride,
3649 : const T *CPL_RESTRICT pSrc, int nSrcStride,
3650 : GPtrDiff_t nIters)
3651 : {
3652 40423200 : constexpr int sizeofT = static_cast<int>(sizeof(T));
3653 40423200 : if (nIters == 1)
3654 : {
3655 22109490 : *pDest = *pSrc;
3656 : }
3657 18313761 : else if (nDestStride == sizeofT)
3658 : {
3659 14980862 : if (nSrcStride == sizeofT)
3660 : {
3661 14114105 : memcpy(pDest, pSrc, nIters * sizeof(T));
3662 : }
3663 866758 : else if (nSrcStride == 2 * sizeofT)
3664 : {
3665 355879 : GDALUnrolledCopy<T, 2, 1>(pDest, pSrc, nIters);
3666 : }
3667 510879 : else if (nSrcStride == 3 * sizeofT)
3668 : {
3669 288432 : GDALUnrolledCopy<T, 3, 1>(pDest, pSrc, nIters);
3670 : }
3671 222447 : else if (nSrcStride == 4 * sizeofT)
3672 : {
3673 110489 : GDALUnrolledCopy<T, 4, 1>(pDest, pSrc, nIters);
3674 : }
3675 : else
3676 : {
3677 17228190 : while (nIters-- > 0)
3678 : {
3679 17116250 : *pDest = *pSrc;
3680 17116250 : pSrc += nSrcStride / sizeofT;
3681 17116250 : pDest++;
3682 : }
3683 : }
3684 : }
3685 3332869 : else if (nSrcStride == sizeofT)
3686 : {
3687 3320274 : if (nDestStride == 2 * sizeofT)
3688 : {
3689 158669 : GDALUnrolledCopy<T, 1, 2>(pDest, pSrc, nIters);
3690 : }
3691 3161603 : else if (nDestStride == 3 * sizeofT)
3692 : {
3693 2275339 : GDALUnrolledCopy<T, 1, 3>(pDest, pSrc, nIters);
3694 : }
3695 886268 : else if (nDestStride == 4 * sizeofT)
3696 : {
3697 722657 : GDALUnrolledCopy<T, 1, 4>(pDest, pSrc, nIters);
3698 : }
3699 : else
3700 : {
3701 17105960 : while (nIters-- > 0)
3702 : {
3703 16942410 : *pDest = *pSrc;
3704 16942410 : pSrc++;
3705 16942410 : pDest += nDestStride / sizeofT;
3706 : }
3707 : }
3708 : }
3709 : else
3710 : {
3711 1219697 : while (nIters-- > 0)
3712 : {
3713 1207102 : *pDest = *pSrc;
3714 1207102 : pSrc += nSrcStride / sizeofT;
3715 1207102 : pDest += nDestStride / sizeofT;
3716 : }
3717 : }
3718 40423200 : }
3719 :
3720 : /************************************************************************/
3721 : /* GDALFastCopyByte() */
3722 : /************************************************************************/
3723 :
3724 326246 : static void GDALFastCopyByte(const GByte *CPL_RESTRICT pSrcData,
3725 : int nSrcPixelStride, GByte *CPL_RESTRICT pDstData,
3726 : int nDstPixelStride, GPtrDiff_t nWordCount)
3727 : {
3728 326246 : GDALFastCopy(pDstData, nDstPixelStride, pSrcData, nSrcPixelStride,
3729 : nWordCount);
3730 326246 : }
3731 :
3732 : /************************************************************************/
3733 : /* GDALCopyWords() */
3734 : /************************************************************************/
3735 :
3736 : /**
3737 : * Copy pixel words from buffer to buffer.
3738 : *
3739 : * @see GDALCopyWords64()
3740 : */
3741 87224800 : void CPL_STDCALL GDALCopyWords(const void *CPL_RESTRICT pSrcData,
3742 : GDALDataType eSrcType, int nSrcPixelStride,
3743 : void *CPL_RESTRICT pDstData,
3744 : GDALDataType eDstType, int nDstPixelStride,
3745 : int nWordCount)
3746 : {
3747 87224800 : GDALCopyWords64(pSrcData, eSrcType, nSrcPixelStride, pDstData, eDstType,
3748 : nDstPixelStride, nWordCount);
3749 87212600 : }
3750 :
3751 : /************************************************************************/
3752 : /* GDALCopyWords64() */
3753 : /************************************************************************/
3754 :
3755 : /**
3756 : * Copy pixel words from buffer to buffer.
3757 : *
3758 : * This function is used to copy pixel word values from one memory buffer
3759 : * to another, with support for conversion between data types, and differing
3760 : * step factors. The data type conversion is done using the following
3761 : * rules:
3762 : * <ul>
3763 : * <li>Values assigned to a lower range integer type are clipped. For
3764 : * instance assigning GDT_Int16 values to a GDT_Byte buffer will cause values
3765 : * less the 0 to be set to 0, and values larger than 255 to be set to 255.
3766 : * </li>
3767 : * <li>
3768 : * Assignment from floating point to integer rounds to closest integer.
3769 : * +Infinity is mapped to the largest integer. -Infinity is mapped to the
3770 : * smallest integer. NaN is mapped to 0.
3771 : * </li>
3772 : * <li>
3773 : * Assignment from non-complex to complex will result in the imaginary part
3774 : * being set to zero on output.
3775 : * </li>
3776 : * <li> Assignment from complex to
3777 : * non-complex will result in the complex portion being lost and the real
3778 : * component being preserved (<i>not magnitude!</i>).
3779 : * </li>
3780 : * </ul>
3781 : *
3782 : * No assumptions are made about the source or destination words occurring
3783 : * on word boundaries. It is assumed that all values are in native machine
3784 : * byte order.
3785 : *
3786 : * @param pSrcData Pointer to source data to be converted.
3787 : * @param eSrcType the source data type (see GDALDataType enum)
3788 : * @param nSrcPixelStride Source pixel stride (i.e. distance between 2 words),
3789 : * in bytes
3790 : * @param pDstData Pointer to buffer where destination data should go
3791 : * @param eDstType the destination data type (see GDALDataType enum)
3792 : * @param nDstPixelStride Destination pixel stride (i.e. distance between 2
3793 : * words), in bytes
3794 : * @param nWordCount number of words to be copied
3795 : *
3796 : * @note
3797 : * When adding a new data type to GDAL, you must do the following to
3798 : * support it properly within the GDALCopyWords function:
3799 : * 1. Add the data type to the switch on eSrcType in GDALCopyWords.
3800 : * This should invoke the appropriate GDALCopyWordsFromT wrapper.
3801 : * 2. Add the data type to the switch on eDstType in GDALCopyWordsFromT.
3802 : * This should call the appropriate GDALCopyWordsT template.
3803 : * 3. If appropriate, overload the appropriate CopyWord template in the
3804 : * above namespace. This will ensure that any conversion issues are
3805 : * handled (cases like the float -> int32 case, where the min/max)
3806 : * values are subject to roundoff error.
3807 : */
3808 :
3809 109306000 : void CPL_STDCALL GDALCopyWords64(const void *CPL_RESTRICT pSrcData,
3810 : GDALDataType eSrcType, int nSrcPixelStride,
3811 : void *CPL_RESTRICT pDstData,
3812 : GDALDataType eDstType, int nDstPixelStride,
3813 : GPtrDiff_t nWordCount)
3814 :
3815 : {
3816 : // On platforms where alignment matters, be careful
3817 109306000 : const int nSrcDataTypeSize = GDALGetDataTypeSizeBytes(eSrcType);
3818 109279000 : const int nDstDataTypeSize = GDALGetDataTypeSizeBytes(eDstType);
3819 109280000 : if (CPL_UNLIKELY(nSrcDataTypeSize == 0 || nDstDataTypeSize == 0))
3820 : {
3821 2 : CPLError(CE_Failure, CPLE_NotSupported,
3822 : "GDALCopyWords64(): unsupported GDT_Unknown/GDT_TypeCount "
3823 : "argument");
3824 2 : return;
3825 : }
3826 109280000 : if (!(eSrcType == eDstType && nSrcPixelStride == nDstPixelStride) &&
3827 59015300 : ((reinterpret_cast<uintptr_t>(pSrcData) % nSrcDataTypeSize) != 0 ||
3828 59019900 : (reinterpret_cast<uintptr_t>(pDstData) % nDstDataTypeSize) != 0 ||
3829 59018100 : (nSrcPixelStride % nSrcDataTypeSize) != 0 ||
3830 59017600 : (nDstPixelStride % nDstDataTypeSize) != 0))
3831 : {
3832 905 : if (eSrcType == eDstType)
3833 : {
3834 34800 : for (decltype(nWordCount) i = 0; i < nWordCount; i++)
3835 : {
3836 34000 : memcpy(static_cast<GByte *>(pDstData) + nDstPixelStride * i,
3837 : static_cast<const GByte *>(pSrcData) +
3838 34000 : nSrcPixelStride * i,
3839 : nDstDataTypeSize);
3840 : }
3841 : }
3842 : else
3843 : {
3844 210 : const auto getAlignedPtr = [](GByte *ptr, int align)
3845 : {
3846 : return ptr +
3847 210 : ((align - (reinterpret_cast<uintptr_t>(ptr) % align)) %
3848 210 : align);
3849 : };
3850 :
3851 : // The largest we need is for CFloat64 (16 bytes), so 32 bytes to
3852 : // be sure to get correctly aligned pointer.
3853 105 : constexpr size_t SIZEOF_CFLOAT64 = 2 * sizeof(double);
3854 : GByte abySrcBuffer[2 * SIZEOF_CFLOAT64];
3855 : GByte abyDstBuffer[2 * SIZEOF_CFLOAT64];
3856 : GByte *pabySrcBuffer =
3857 105 : getAlignedPtr(abySrcBuffer, nSrcDataTypeSize);
3858 : GByte *pabyDstBuffer =
3859 105 : getAlignedPtr(abyDstBuffer, nDstDataTypeSize);
3860 3360 : for (decltype(nWordCount) i = 0; i < nWordCount; i++)
3861 : {
3862 3255 : memcpy(pabySrcBuffer,
3863 : static_cast<const GByte *>(pSrcData) +
3864 3255 : nSrcPixelStride * i,
3865 : nSrcDataTypeSize);
3866 3255 : GDALCopyWords64(pabySrcBuffer, eSrcType, 0, pabyDstBuffer,
3867 : eDstType, 0, 1);
3868 3255 : memcpy(static_cast<GByte *>(pDstData) + nDstPixelStride * i,
3869 : pabyDstBuffer, nDstDataTypeSize);
3870 : }
3871 : }
3872 905 : return;
3873 : }
3874 :
3875 : // Deal with the case where we're replicating a single word into the
3876 : // provided buffer
3877 109279000 : if (nSrcPixelStride == 0 && nWordCount > 1)
3878 : {
3879 1011310 : GDALReplicateWord(pSrcData, eSrcType, pDstData, eDstType,
3880 : nDstPixelStride, nWordCount);
3881 1011780 : return;
3882 : }
3883 :
3884 108268000 : if (eSrcType == eDstType)
3885 : {
3886 54380700 : if (eSrcType == GDT_Byte || eSrcType == GDT_Int8)
3887 : {
3888 19287100 : GDALFastCopy(static_cast<GByte *>(pDstData), nDstPixelStride,
3889 : static_cast<const GByte *>(pSrcData), nSrcPixelStride,
3890 : nWordCount);
3891 19287000 : return;
3892 : }
3893 :
3894 35093600 : if (nSrcDataTypeSize == 2 && (nSrcPixelStride % 2) == 0 &&
3895 20809000 : (nDstPixelStride % 2) == 0)
3896 : {
3897 20808900 : GDALFastCopy(static_cast<short *>(pDstData), nDstPixelStride,
3898 : static_cast<const short *>(pSrcData), nSrcPixelStride,
3899 : nWordCount);
3900 20808700 : return;
3901 : }
3902 :
3903 14284700 : if (nWordCount == 1)
3904 : {
3905 : #if defined(CSA_BUILD) || defined(__COVERITY__)
3906 : // Avoid false positives...
3907 : memcpy(pDstData, pSrcData, nSrcDataTypeSize);
3908 : #else
3909 13892900 : if (nSrcDataTypeSize == 2)
3910 0 : memcpy(pDstData, pSrcData, 2);
3911 13892900 : else if (nSrcDataTypeSize == 4)
3912 13809200 : memcpy(pDstData, pSrcData, 4);
3913 83682 : else if (nSrcDataTypeSize == 8)
3914 67162 : memcpy(pDstData, pSrcData, 8);
3915 : else /* if( eSrcType == GDT_CFloat64 ) */
3916 16520 : memcpy(pDstData, pSrcData, 16);
3917 : #endif
3918 13892900 : return;
3919 : }
3920 :
3921 : // Let memcpy() handle the case where we're copying a packed buffer
3922 : // of pixels.
3923 391849 : if (nSrcPixelStride == nDstPixelStride)
3924 : {
3925 263675 : if (nSrcPixelStride == nSrcDataTypeSize)
3926 : {
3927 263604 : memcpy(pDstData, pSrcData, nWordCount * nSrcDataTypeSize);
3928 263604 : return;
3929 : }
3930 : }
3931 : }
3932 :
3933 : // Handle the more general case -- deals with conversion of data types
3934 : // directly.
3935 54015300 : switch (eSrcType)
3936 : {
3937 15494900 : case GDT_Byte:
3938 15494900 : GDALCopyWordsFromT<unsigned char>(
3939 : static_cast<const unsigned char *>(pSrcData), nSrcPixelStride,
3940 : false, pDstData, eDstType, nDstPixelStride, nWordCount);
3941 15502000 : break;
3942 1225 : case GDT_Int8:
3943 1225 : GDALCopyWordsFromT<signed char>(
3944 : static_cast<const signed char *>(pSrcData), nSrcPixelStride,
3945 : false, pDstData, eDstType, nDstPixelStride, nWordCount);
3946 1225 : break;
3947 53348 : case GDT_UInt16:
3948 53348 : GDALCopyWordsFromT<unsigned short>(
3949 : static_cast<const unsigned short *>(pSrcData), nSrcPixelStride,
3950 : false, pDstData, eDstType, nDstPixelStride, nWordCount);
3951 53348 : break;
3952 4350270 : case GDT_Int16:
3953 4350270 : GDALCopyWordsFromT<short>(static_cast<const short *>(pSrcData),
3954 : nSrcPixelStride, false, pDstData,
3955 : eDstType, nDstPixelStride, nWordCount);
3956 4350260 : break;
3957 7092 : case GDT_UInt32:
3958 7092 : GDALCopyWordsFromT<unsigned int>(
3959 : static_cast<const unsigned int *>(pSrcData), nSrcPixelStride,
3960 : false, pDstData, eDstType, nDstPixelStride, nWordCount);
3961 7092 : break;
3962 12255000 : case GDT_Int32:
3963 12255000 : GDALCopyWordsFromT<int>(static_cast<const int *>(pSrcData),
3964 : nSrcPixelStride, false, pDstData, eDstType,
3965 : nDstPixelStride, nWordCount);
3966 12255000 : break;
3967 1635 : case GDT_UInt64:
3968 1635 : GDALCopyWordsFromT<std::uint64_t>(
3969 : static_cast<const std::uint64_t *>(pSrcData), nSrcPixelStride,
3970 : false, pDstData, eDstType, nDstPixelStride, nWordCount);
3971 1635 : break;
3972 10978 : case GDT_Int64:
3973 10978 : GDALCopyWordsFromT<std::int64_t>(
3974 : static_cast<const std::int64_t *>(pSrcData), nSrcPixelStride,
3975 : false, pDstData, eDstType, nDstPixelStride, nWordCount);
3976 10978 : break;
3977 1074 : case GDT_Float16:
3978 1074 : GDALCopyWordsFromT<GFloat16>(
3979 : static_cast<const GFloat16 *>(pSrcData), nSrcPixelStride, false,
3980 : pDstData, eDstType, nDstPixelStride, nWordCount);
3981 1074 : break;
3982 438203 : case GDT_Float32:
3983 438203 : GDALCopyWordsFromT<float>(static_cast<const float *>(pSrcData),
3984 : nSrcPixelStride, false, pDstData,
3985 : eDstType, nDstPixelStride, nWordCount);
3986 438213 : break;
3987 20666700 : case GDT_Float64:
3988 20666700 : GDALCopyWordsFromT<double>(static_cast<const double *>(pSrcData),
3989 : nSrcPixelStride, false, pDstData,
3990 : eDstType, nDstPixelStride, nWordCount);
3991 20666800 : break;
3992 478141 : case GDT_CInt16:
3993 478141 : GDALCopyWordsFromT<short>(static_cast<const short *>(pSrcData),
3994 : nSrcPixelStride, true, pDstData, eDstType,
3995 : nDstPixelStride, nWordCount);
3996 478141 : break;
3997 556 : case GDT_CInt32:
3998 556 : GDALCopyWordsFromT<int>(static_cast<const int *>(pSrcData),
3999 : nSrcPixelStride, true, pDstData, eDstType,
4000 : nDstPixelStride, nWordCount);
4001 556 : break;
4002 396 : case GDT_CFloat16:
4003 396 : GDALCopyWordsFromT<GFloat16>(
4004 : static_cast<const GFloat16 *>(pSrcData), nSrcPixelStride, true,
4005 : pDstData, eDstType, nDstPixelStride, nWordCount);
4006 396 : break;
4007 1547 : case GDT_CFloat32:
4008 1547 : GDALCopyWordsFromT<float>(static_cast<const float *>(pSrcData),
4009 : nSrcPixelStride, true, pDstData, eDstType,
4010 : nDstPixelStride, nWordCount);
4011 1547 : break;
4012 276418 : case GDT_CFloat64:
4013 276418 : GDALCopyWordsFromT<double>(static_cast<const double *>(pSrcData),
4014 : nSrcPixelStride, true, pDstData,
4015 : eDstType, nDstPixelStride, nWordCount);
4016 276418 : break;
4017 0 : case GDT_Unknown:
4018 : case GDT_TypeCount:
4019 0 : CPLAssert(false);
4020 : }
4021 : }
4022 :
4023 : /************************************************************************/
4024 : /* GDALCopyBits() */
4025 : /************************************************************************/
4026 :
4027 : /**
4028 : * Bitwise word copying.
4029 : *
4030 : * A function for moving sets of partial bytes around. Loosely
4031 : * speaking this is a bitwise analog to GDALCopyWords().
4032 : *
4033 : * It copies nStepCount "words" where each word is nBitCount bits long.
4034 : * The nSrcStep and nDstStep are the number of bits from the start of one
4035 : * word to the next (same as nBitCount if they are packed). The nSrcOffset
4036 : * and nDstOffset are the offset into the source and destination buffers
4037 : * to start at, also measured in bits.
4038 : *
4039 : * All bit offsets are assumed to start from the high order bit in a byte
4040 : * (i.e. most significant bit first). Currently this function is not very
4041 : * optimized, but it may be improved for some common cases in the future
4042 : * as needed.
4043 : *
4044 : * @param pabySrcData the source data buffer.
4045 : * @param nSrcOffset the offset (in bits) in pabySrcData to the start of the
4046 : * first word to copy.
4047 : * @param nSrcStep the offset in bits from the start one source word to the
4048 : * start of the next.
4049 : * @param pabyDstData the destination data buffer.
4050 : * @param nDstOffset the offset (in bits) in pabyDstData to the start of the
4051 : * first word to copy over.
4052 : * @param nDstStep the offset in bits from the start one word to the
4053 : * start of the next.
4054 : * @param nBitCount the number of bits in a word to be copied.
4055 : * @param nStepCount the number of words to copy.
4056 : */
4057 :
4058 0 : void GDALCopyBits(const GByte *pabySrcData, int nSrcOffset, int nSrcStep,
4059 : GByte *pabyDstData, int nDstOffset, int nDstStep,
4060 : int nBitCount, int nStepCount)
4061 :
4062 : {
4063 0 : VALIDATE_POINTER0(pabySrcData, "GDALCopyBits");
4064 :
4065 0 : for (int iStep = 0; iStep < nStepCount; iStep++)
4066 : {
4067 0 : for (int iBit = 0; iBit < nBitCount; iBit++)
4068 : {
4069 0 : if (pabySrcData[nSrcOffset >> 3] & (0x80 >> (nSrcOffset & 7)))
4070 0 : pabyDstData[nDstOffset >> 3] |= (0x80 >> (nDstOffset & 7));
4071 : else
4072 0 : pabyDstData[nDstOffset >> 3] &= ~(0x80 >> (nDstOffset & 7));
4073 :
4074 0 : nSrcOffset++;
4075 0 : nDstOffset++;
4076 : }
4077 :
4078 0 : nSrcOffset += (nSrcStep - nBitCount);
4079 0 : nDstOffset += (nDstStep - nBitCount);
4080 : }
4081 : }
4082 :
4083 : /************************************************************************/
4084 : /* GDALGetBestOverviewLevel() */
4085 : /* */
4086 : /* Returns the best overview level to satisfy the query or -1 if none */
4087 : /* Also updates nXOff, nYOff, nXSize, nYSize and psExtraArg when */
4088 : /* returning a valid overview level */
4089 : /************************************************************************/
4090 :
4091 0 : int GDALBandGetBestOverviewLevel(GDALRasterBand *poBand, int &nXOff, int &nYOff,
4092 : int &nXSize, int &nYSize, int nBufXSize,
4093 : int nBufYSize)
4094 : {
4095 0 : return GDALBandGetBestOverviewLevel2(poBand, nXOff, nYOff, nXSize, nYSize,
4096 0 : nBufXSize, nBufYSize, nullptr);
4097 : }
4098 :
4099 523846 : int GDALBandGetBestOverviewLevel2(GDALRasterBand *poBand, int &nXOff,
4100 : int &nYOff, int &nXSize, int &nYSize,
4101 : int nBufXSize, int nBufYSize,
4102 : GDALRasterIOExtraArg *psExtraArg)
4103 : {
4104 523846 : if (psExtraArg != nullptr && psExtraArg->nVersion > 1 &&
4105 523846 : psExtraArg->bUseOnlyThisScale)
4106 109 : return -1;
4107 : /* -------------------------------------------------------------------- */
4108 : /* Compute the desired downsampling factor. It is */
4109 : /* based on the least reduced axis, and represents the number */
4110 : /* of source pixels to one destination pixel. */
4111 : /* -------------------------------------------------------------------- */
4112 523737 : const double dfDesiredDownsamplingFactor =
4113 523737 : ((nXSize / static_cast<double>(nBufXSize)) <
4114 361399 : (nYSize / static_cast<double>(nBufYSize)) ||
4115 : nBufYSize == 1)
4116 752122 : ? nXSize / static_cast<double>(nBufXSize)
4117 133014 : : nYSize / static_cast<double>(nBufYSize);
4118 :
4119 : /* -------------------------------------------------------------------- */
4120 : /* Find the overview level that largest downsampling factor (most */
4121 : /* downsampled) that is still less than (or only a little more) */
4122 : /* downsampled than the request. */
4123 : /* -------------------------------------------------------------------- */
4124 523737 : const int nOverviewCount = poBand->GetOverviewCount();
4125 523737 : GDALRasterBand *poBestOverview = nullptr;
4126 523737 : double dfBestDownsamplingFactor = 0;
4127 523737 : int nBestOverviewLevel = -1;
4128 :
4129 : const char *pszOversampligThreshold =
4130 523737 : CPLGetConfigOption("GDAL_OVERVIEW_OVERSAMPLING_THRESHOLD", nullptr);
4131 :
4132 : // Note: keep this logic for overview selection in sync between
4133 : // gdalwarp_lib.cpp and rasterio.cpp
4134 : // Cf https://github.com/OSGeo/gdal/pull/9040#issuecomment-1898524693
4135 : const double dfOversamplingThreshold =
4136 1047460 : pszOversampligThreshold ? CPLAtof(pszOversampligThreshold)
4137 523728 : : psExtraArg && psExtraArg->eResampleAlg != GRIORA_NearestNeighbour
4138 1047460 : ? 1.0
4139 523737 : : 1.2;
4140 526432 : for (int iOverview = 0; iOverview < nOverviewCount; iOverview++)
4141 : {
4142 5547 : GDALRasterBand *poOverview = poBand->GetOverview(iOverview);
4143 11094 : if (poOverview == nullptr ||
4144 11093 : poOverview->GetXSize() > poBand->GetXSize() ||
4145 5546 : poOverview->GetYSize() > poBand->GetYSize())
4146 : {
4147 1 : continue;
4148 : }
4149 :
4150 : // Compute downsampling factor of this overview
4151 : const double dfDownsamplingFactor = std::min(
4152 5546 : poBand->GetXSize() / static_cast<double>(poOverview->GetXSize()),
4153 11092 : poBand->GetYSize() / static_cast<double>(poOverview->GetYSize()));
4154 :
4155 : // Is it nearly the requested factor and better (lower) than
4156 : // the current best factor?
4157 : // Use an epsilon because of numerical instability.
4158 5546 : constexpr double EPSILON = 1e-1;
4159 5654 : if (dfDownsamplingFactor >=
4160 5546 : dfDesiredDownsamplingFactor * dfOversamplingThreshold +
4161 5438 : EPSILON ||
4162 : dfDownsamplingFactor <= dfBestDownsamplingFactor)
4163 : {
4164 108 : continue;
4165 : }
4166 :
4167 : // Ignore AVERAGE_BIT2GRAYSCALE overviews for RasterIO purposes.
4168 5438 : const char *pszResampling = poOverview->GetMetadataItem("RESAMPLING");
4169 :
4170 5438 : if (pszResampling != nullptr &&
4171 71 : STARTS_WITH_CI(pszResampling, "AVERAGE_BIT2"))
4172 16 : continue;
4173 :
4174 : // OK, this is our new best overview.
4175 5422 : poBestOverview = poOverview;
4176 5422 : nBestOverviewLevel = iOverview;
4177 5422 : dfBestDownsamplingFactor = dfDownsamplingFactor;
4178 :
4179 5422 : if (std::abs(dfDesiredDownsamplingFactor - dfDownsamplingFactor) <
4180 : EPSILON)
4181 : {
4182 2852 : break;
4183 : }
4184 : }
4185 :
4186 : /* -------------------------------------------------------------------- */
4187 : /* If we didn't find an overview that helps us, just return */
4188 : /* indicating failure and the full resolution image will be used. */
4189 : /* -------------------------------------------------------------------- */
4190 523737 : if (nBestOverviewLevel < 0)
4191 520813 : return -1;
4192 :
4193 : /* -------------------------------------------------------------------- */
4194 : /* Recompute the source window in terms of the selected */
4195 : /* overview. */
4196 : /* -------------------------------------------------------------------- */
4197 : const double dfXFactor =
4198 2924 : poBand->GetXSize() / static_cast<double>(poBestOverview->GetXSize());
4199 : const double dfYFactor =
4200 2924 : poBand->GetYSize() / static_cast<double>(poBestOverview->GetYSize());
4201 2924 : CPLDebug("GDAL", "Selecting overview %d x %d", poBestOverview->GetXSize(),
4202 : poBestOverview->GetYSize());
4203 :
4204 8772 : const int nOXOff = std::min(poBestOverview->GetXSize() - 1,
4205 2924 : static_cast<int>(nXOff / dfXFactor + 0.5));
4206 8772 : const int nOYOff = std::min(poBestOverview->GetYSize() - 1,
4207 2924 : static_cast<int>(nYOff / dfYFactor + 0.5));
4208 2924 : int nOXSize = std::max(1, static_cast<int>(nXSize / dfXFactor + 0.5));
4209 2924 : int nOYSize = std::max(1, static_cast<int>(nYSize / dfYFactor + 0.5));
4210 2924 : if (nOXOff + nOXSize > poBestOverview->GetXSize())
4211 0 : nOXSize = poBestOverview->GetXSize() - nOXOff;
4212 2924 : if (nOYOff + nOYSize > poBestOverview->GetYSize())
4213 2 : nOYSize = poBestOverview->GetYSize() - nOYOff;
4214 :
4215 2924 : if (psExtraArg)
4216 : {
4217 2924 : if (psExtraArg->bFloatingPointWindowValidity)
4218 : {
4219 50 : psExtraArg->dfXOff /= dfXFactor;
4220 50 : psExtraArg->dfXSize /= dfXFactor;
4221 50 : psExtraArg->dfYOff /= dfYFactor;
4222 50 : psExtraArg->dfYSize /= dfYFactor;
4223 : }
4224 2874 : else if (psExtraArg->eResampleAlg != GRIORA_NearestNeighbour)
4225 : {
4226 16 : psExtraArg->bFloatingPointWindowValidity = true;
4227 16 : psExtraArg->dfXOff = nXOff / dfXFactor;
4228 16 : psExtraArg->dfXSize = nXSize / dfXFactor;
4229 16 : psExtraArg->dfYOff = nYOff / dfYFactor;
4230 16 : psExtraArg->dfYSize = nYSize / dfYFactor;
4231 : }
4232 : }
4233 :
4234 2924 : nXOff = nOXOff;
4235 2924 : nYOff = nOYOff;
4236 2924 : nXSize = nOXSize;
4237 2924 : nYSize = nOYSize;
4238 :
4239 2924 : return nBestOverviewLevel;
4240 : }
4241 :
4242 : /************************************************************************/
4243 : /* OverviewRasterIO() */
4244 : /* */
4245 : /* Special work function to utilize available overviews to */
4246 : /* more efficiently satisfy downsampled requests. It will */
4247 : /* return CE_Failure if there are no appropriate overviews */
4248 : /* available but it doesn't emit any error messages. */
4249 : /************************************************************************/
4250 :
4251 : //! @cond Doxygen_Suppress
4252 2 : CPLErr GDALRasterBand::OverviewRasterIO(
4253 : GDALRWFlag eRWFlag, int nXOff, int nYOff, int nXSize, int nYSize,
4254 : void *pData, int nBufXSize, int nBufYSize, GDALDataType eBufType,
4255 : GSpacing nPixelSpace, GSpacing nLineSpace, GDALRasterIOExtraArg *psExtraArg)
4256 :
4257 : {
4258 : GDALRasterIOExtraArg sExtraArg;
4259 2 : GDALCopyRasterIOExtraArg(&sExtraArg, psExtraArg);
4260 :
4261 2 : const int nOverview = GDALBandGetBestOverviewLevel2(
4262 : this, nXOff, nYOff, nXSize, nYSize, nBufXSize, nBufYSize, &sExtraArg);
4263 2 : if (nOverview < 0)
4264 1 : return CE_Failure;
4265 :
4266 : /* -------------------------------------------------------------------- */
4267 : /* Recast the call in terms of the new raster layer. */
4268 : /* -------------------------------------------------------------------- */
4269 1 : GDALRasterBand *poOverviewBand = GetOverview(nOverview);
4270 1 : if (poOverviewBand == nullptr)
4271 0 : return CE_Failure;
4272 :
4273 1 : return poOverviewBand->RasterIO(eRWFlag, nXOff, nYOff, nXSize, nYSize,
4274 : pData, nBufXSize, nBufYSize, eBufType,
4275 1 : nPixelSpace, nLineSpace, &sExtraArg);
4276 : }
4277 :
4278 : /************************************************************************/
4279 : /* TryOverviewRasterIO() */
4280 : /************************************************************************/
4281 :
4282 362416 : CPLErr GDALRasterBand::TryOverviewRasterIO(
4283 : GDALRWFlag eRWFlag, int nXOff, int nYOff, int nXSize, int nYSize,
4284 : void *pData, int nBufXSize, int nBufYSize, GDALDataType eBufType,
4285 : GSpacing nPixelSpace, GSpacing nLineSpace, GDALRasterIOExtraArg *psExtraArg,
4286 : int *pbTried)
4287 : {
4288 362416 : int nXOffMod = nXOff;
4289 362416 : int nYOffMod = nYOff;
4290 362416 : int nXSizeMod = nXSize;
4291 362416 : int nYSizeMod = nYSize;
4292 : GDALRasterIOExtraArg sExtraArg;
4293 :
4294 362416 : GDALCopyRasterIOExtraArg(&sExtraArg, psExtraArg);
4295 :
4296 362416 : int iOvrLevel = GDALBandGetBestOverviewLevel2(
4297 : this, nXOffMod, nYOffMod, nXSizeMod, nYSizeMod, nBufXSize, nBufYSize,
4298 : &sExtraArg);
4299 :
4300 362416 : if (iOvrLevel >= 0)
4301 : {
4302 50 : GDALRasterBand *poOverviewBand = GetOverview(iOvrLevel);
4303 50 : if (poOverviewBand)
4304 : {
4305 50 : *pbTried = TRUE;
4306 50 : return poOverviewBand->RasterIO(
4307 : eRWFlag, nXOffMod, nYOffMod, nXSizeMod, nYSizeMod, pData,
4308 : nBufXSize, nBufYSize, eBufType, nPixelSpace, nLineSpace,
4309 50 : &sExtraArg);
4310 : }
4311 : }
4312 :
4313 362366 : *pbTried = FALSE;
4314 362366 : return CE_None;
4315 : }
4316 :
4317 : /************************************************************************/
4318 : /* TryOverviewRasterIO() */
4319 : /************************************************************************/
4320 :
4321 158519 : CPLErr GDALDataset::TryOverviewRasterIO(
4322 : GDALRWFlag eRWFlag, int nXOff, int nYOff, int nXSize, int nYSize,
4323 : void *pData, int nBufXSize, int nBufYSize, GDALDataType eBufType,
4324 : int nBandCount, const int *panBandMap, GSpacing nPixelSpace,
4325 : GSpacing nLineSpace, GSpacing nBandSpace, GDALRasterIOExtraArg *psExtraArg,
4326 : int *pbTried)
4327 : {
4328 158519 : int nXOffMod = nXOff;
4329 158519 : int nYOffMod = nYOff;
4330 158519 : int nXSizeMod = nXSize;
4331 158519 : int nYSizeMod = nYSize;
4332 : GDALRasterIOExtraArg sExtraArg;
4333 158519 : GDALCopyRasterIOExtraArg(&sExtraArg, psExtraArg);
4334 :
4335 317038 : int iOvrLevel = GDALBandGetBestOverviewLevel2(
4336 158519 : papoBands[0], nXOffMod, nYOffMod, nXSizeMod, nYSizeMod, nBufXSize,
4337 : nBufYSize, &sExtraArg);
4338 :
4339 158560 : if (iOvrLevel >= 0 && papoBands[0]->GetOverview(iOvrLevel) != nullptr &&
4340 41 : papoBands[0]->GetOverview(iOvrLevel)->GetDataset() != nullptr)
4341 : {
4342 41 : *pbTried = TRUE;
4343 41 : return papoBands[0]->GetOverview(iOvrLevel)->GetDataset()->RasterIO(
4344 : eRWFlag, nXOffMod, nYOffMod, nXSizeMod, nYSizeMod, pData, nBufXSize,
4345 : nBufYSize, eBufType, nBandCount, panBandMap, nPixelSpace,
4346 41 : nLineSpace, nBandSpace, &sExtraArg);
4347 : }
4348 : else
4349 : {
4350 158478 : *pbTried = FALSE;
4351 158478 : return CE_None;
4352 : }
4353 : }
4354 :
4355 : /************************************************************************/
4356 : /* GetBestOverviewLevel() */
4357 : /* */
4358 : /* Returns the best overview level to satisfy the query or -1 if none */
4359 : /* Also updates nXOff, nYOff, nXSize, nYSize when returning a valid */
4360 : /* overview level */
4361 : /************************************************************************/
4362 :
4363 4 : static int GDALDatasetGetBestOverviewLevel(GDALDataset *poDS, int &nXOff,
4364 : int &nYOff, int &nXSize, int &nYSize,
4365 : int nBufXSize, int nBufYSize,
4366 : int nBandCount,
4367 : const int *panBandMap,
4368 : GDALRasterIOExtraArg *psExtraArg)
4369 : {
4370 4 : int nOverviewCount = 0;
4371 4 : GDALRasterBand *poFirstBand = nullptr;
4372 :
4373 : /* -------------------------------------------------------------------- */
4374 : /* Check that all bands have the same number of overviews and */
4375 : /* that they have all the same size and block dimensions */
4376 : /* -------------------------------------------------------------------- */
4377 12 : for (int iBand = 0; iBand < nBandCount; iBand++)
4378 : {
4379 8 : GDALRasterBand *poBand = poDS->GetRasterBand(panBandMap[iBand]);
4380 8 : if (poBand == nullptr)
4381 0 : return -1;
4382 8 : if (iBand == 0)
4383 : {
4384 4 : poFirstBand = poBand;
4385 4 : nOverviewCount = poBand->GetOverviewCount();
4386 : }
4387 4 : else if (nOverviewCount != poBand->GetOverviewCount())
4388 : {
4389 0 : CPLDebug("GDAL", "GDALDataset::GetBestOverviewLevel() ... "
4390 : "mismatched overview count, use std method.");
4391 0 : return -1;
4392 : }
4393 : else
4394 : {
4395 4 : for (int iOverview = 0; iOverview < nOverviewCount; iOverview++)
4396 : {
4397 0 : GDALRasterBand *poOvrBand = poBand->GetOverview(iOverview);
4398 : GDALRasterBand *poOvrFirstBand =
4399 0 : poFirstBand->GetOverview(iOverview);
4400 0 : if (poOvrBand == nullptr || poOvrFirstBand == nullptr)
4401 0 : continue;
4402 :
4403 0 : if (poOvrFirstBand->GetXSize() != poOvrBand->GetXSize() ||
4404 0 : poOvrFirstBand->GetYSize() != poOvrBand->GetYSize())
4405 : {
4406 0 : CPLDebug("GDAL",
4407 : "GDALDataset::GetBestOverviewLevel() ... "
4408 : "mismatched overview sizes, use std method.");
4409 0 : return -1;
4410 : }
4411 0 : int nBlockXSizeFirst = 0;
4412 0 : int nBlockYSizeFirst = 0;
4413 0 : poOvrFirstBand->GetBlockSize(&nBlockXSizeFirst,
4414 : &nBlockYSizeFirst);
4415 :
4416 0 : int nBlockXSizeCurrent = 0;
4417 0 : int nBlockYSizeCurrent = 0;
4418 0 : poOvrBand->GetBlockSize(&nBlockXSizeCurrent,
4419 : &nBlockYSizeCurrent);
4420 :
4421 0 : if (nBlockXSizeFirst != nBlockXSizeCurrent ||
4422 0 : nBlockYSizeFirst != nBlockYSizeCurrent)
4423 : {
4424 0 : CPLDebug("GDAL", "GDALDataset::GetBestOverviewLevel() ... "
4425 : "mismatched block sizes, use std method.");
4426 0 : return -1;
4427 : }
4428 : }
4429 : }
4430 : }
4431 4 : if (poFirstBand == nullptr)
4432 0 : return -1;
4433 :
4434 4 : return GDALBandGetBestOverviewLevel2(poFirstBand, nXOff, nYOff, nXSize,
4435 : nYSize, nBufXSize, nBufYSize,
4436 4 : psExtraArg);
4437 : }
4438 :
4439 : /************************************************************************/
4440 : /* BlockBasedRasterIO() */
4441 : /* */
4442 : /* This convenience function implements a dataset level */
4443 : /* RasterIO() interface based on calling down to fetch blocks, */
4444 : /* much like the GDALRasterBand::IRasterIO(), but it handles */
4445 : /* all bands at once, so that a format driver that handles a */
4446 : /* request for different bands of the same block efficiently */
4447 : /* (i.e. without re-reading interleaved data) will efficiently. */
4448 : /* */
4449 : /* This method is intended to be called by an overridden */
4450 : /* IRasterIO() method in the driver specific GDALDataset */
4451 : /* derived class. */
4452 : /* */
4453 : /* Default internal implementation of RasterIO() ... utilizes */
4454 : /* the Block access methods to satisfy the request. This would */
4455 : /* normally only be overridden by formats with overviews. */
4456 : /* */
4457 : /* To keep things relatively simple, this method does not */
4458 : /* currently take advantage of some special cases addressed in */
4459 : /* GDALRasterBand::IRasterIO(), so it is likely best to only */
4460 : /* call it when you know it will help. That is in cases where */
4461 : /* data is at 1:1 to the buffer, and you know the driver is */
4462 : /* implementing interleaved IO efficiently on a block by block */
4463 : /* basis. Overviews will be used when possible. */
4464 : /************************************************************************/
4465 :
4466 63955 : CPLErr GDALDataset::BlockBasedRasterIO(
4467 : GDALRWFlag eRWFlag, int nXOff, int nYOff, int nXSize, int nYSize,
4468 : void *pData, int nBufXSize, int nBufYSize, GDALDataType eBufType,
4469 : int nBandCount, const int *panBandMap, GSpacing nPixelSpace,
4470 : GSpacing nLineSpace, GSpacing nBandSpace, GDALRasterIOExtraArg *psExtraArg)
4471 :
4472 : {
4473 63955 : CPLAssert(nullptr != pData);
4474 :
4475 63955 : GByte **papabySrcBlock = nullptr;
4476 63955 : GDALRasterBlock *poBlock = nullptr;
4477 63955 : GDALRasterBlock **papoBlocks = nullptr;
4478 63955 : int nLBlockX = -1;
4479 63955 : int nLBlockY = -1;
4480 : int iBufYOff;
4481 : int iBufXOff;
4482 63955 : int nBlockXSize = 1;
4483 63955 : int nBlockYSize = 1;
4484 63955 : CPLErr eErr = CE_None;
4485 63955 : GDALDataType eDataType = GDT_Byte;
4486 :
4487 63955 : const bool bUseIntegerRequestCoords =
4488 63985 : (!psExtraArg->bFloatingPointWindowValidity ||
4489 30 : (nXOff == psExtraArg->dfXOff && nYOff == psExtraArg->dfYOff &&
4490 28 : nXSize == psExtraArg->dfXSize && nYSize == psExtraArg->dfYSize));
4491 :
4492 : /* -------------------------------------------------------------------- */
4493 : /* Ensure that all bands share a common block size and data type. */
4494 : /* -------------------------------------------------------------------- */
4495 303182 : for (int iBand = 0; iBand < nBandCount; iBand++)
4496 : {
4497 239227 : GDALRasterBand *poBand = GetRasterBand(panBandMap[iBand]);
4498 :
4499 239227 : if (iBand == 0)
4500 : {
4501 63955 : poBand->GetBlockSize(&nBlockXSize, &nBlockYSize);
4502 63955 : eDataType = poBand->GetRasterDataType();
4503 : }
4504 : else
4505 : {
4506 175272 : int nThisBlockXSize = 0;
4507 175272 : int nThisBlockYSize = 0;
4508 175272 : poBand->GetBlockSize(&nThisBlockXSize, &nThisBlockYSize);
4509 175272 : if (nThisBlockXSize != nBlockXSize ||
4510 175272 : nThisBlockYSize != nBlockYSize)
4511 : {
4512 0 : CPLDebug("GDAL", "GDALDataset::BlockBasedRasterIO() ... "
4513 : "mismatched block sizes, use std method.");
4514 0 : return BandBasedRasterIO(eRWFlag, nXOff, nYOff, nXSize, nYSize,
4515 : pData, nBufXSize, nBufYSize, eBufType,
4516 : nBandCount, panBandMap, nPixelSpace,
4517 0 : nLineSpace, nBandSpace, psExtraArg);
4518 : }
4519 :
4520 175272 : if (eDataType != poBand->GetRasterDataType() &&
4521 0 : (nXSize != nBufXSize || nYSize != nBufYSize))
4522 : {
4523 0 : CPLDebug("GDAL", "GDALDataset::BlockBasedRasterIO() ... "
4524 : "mismatched band data types, use std method.");
4525 0 : return BandBasedRasterIO(eRWFlag, nXOff, nYOff, nXSize, nYSize,
4526 : pData, nBufXSize, nBufYSize, eBufType,
4527 : nBandCount, panBandMap, nPixelSpace,
4528 0 : nLineSpace, nBandSpace, psExtraArg);
4529 : }
4530 : }
4531 : }
4532 :
4533 : /* ==================================================================== */
4534 : /* In this special case at full resolution we step through in */
4535 : /* blocks, turning the request over to the per-band */
4536 : /* IRasterIO(), but ensuring that all bands of one block are */
4537 : /* called before proceeding to the next. */
4538 : /* ==================================================================== */
4539 :
4540 63955 : if (nXSize == nBufXSize && nYSize == nBufYSize && bUseIntegerRequestCoords)
4541 : {
4542 : GDALRasterIOExtraArg sDummyExtraArg;
4543 63951 : INIT_RASTERIO_EXTRA_ARG(sDummyExtraArg);
4544 :
4545 63951 : int nChunkYSize = 0;
4546 63951 : int nChunkXSize = 0;
4547 :
4548 210228 : for (iBufYOff = 0; iBufYOff < nBufYSize; iBufYOff += nChunkYSize)
4549 : {
4550 147292 : const int nChunkYOff = iBufYOff + nYOff;
4551 147292 : nChunkYSize = nBlockYSize - (nChunkYOff % nBlockYSize);
4552 147292 : if (nChunkYOff + nChunkYSize > nYOff + nYSize)
4553 59131 : nChunkYSize = (nYOff + nYSize) - nChunkYOff;
4554 :
4555 817191 : for (iBufXOff = 0; iBufXOff < nBufXSize; iBufXOff += nChunkXSize)
4556 : {
4557 670913 : const int nChunkXOff = iBufXOff + nXOff;
4558 670913 : nChunkXSize = nBlockXSize - (nChunkXOff % nBlockXSize);
4559 670913 : if (nChunkXOff + nChunkXSize > nXOff + nXSize)
4560 70247 : nChunkXSize = (nXOff + nXSize) - nChunkXOff;
4561 :
4562 670913 : GByte *pabyChunkData =
4563 670913 : static_cast<GByte *>(pData) + iBufXOff * nPixelSpace +
4564 670913 : static_cast<GPtrDiff_t>(iBufYOff) * nLineSpace;
4565 :
4566 3267300 : for (int iBand = 0; iBand < nBandCount; iBand++)
4567 : {
4568 2597400 : GDALRasterBand *poBand = GetRasterBand(panBandMap[iBand]);
4569 :
4570 5194790 : eErr = poBand->IRasterIO(
4571 : eRWFlag, nChunkXOff, nChunkYOff, nChunkXSize,
4572 : nChunkYSize,
4573 2597400 : pabyChunkData +
4574 2597400 : static_cast<GPtrDiff_t>(iBand) * nBandSpace,
4575 : nChunkXSize, nChunkYSize, eBufType, nPixelSpace,
4576 2597400 : nLineSpace, &sDummyExtraArg);
4577 2597400 : if (eErr != CE_None)
4578 1013 : return eErr;
4579 : }
4580 : }
4581 :
4582 165068 : if (psExtraArg->pfnProgress != nullptr &&
4583 18790 : !psExtraArg->pfnProgress(
4584 165068 : 1.0 * std::min(nBufYSize, iBufYOff + nChunkYSize) /
4585 : nBufYSize,
4586 : "", psExtraArg->pProgressData))
4587 : {
4588 1 : return CE_Failure;
4589 : }
4590 : }
4591 :
4592 62936 : return CE_None;
4593 : }
4594 :
4595 : /* Below code is not compatible with that case. It would need a complete */
4596 : /* separate code like done in GDALRasterBand::IRasterIO. */
4597 4 : if (eRWFlag == GF_Write && (nBufXSize < nXSize || nBufYSize < nYSize))
4598 : {
4599 0 : return BandBasedRasterIO(eRWFlag, nXOff, nYOff, nXSize, nYSize, pData,
4600 : nBufXSize, nBufYSize, eBufType, nBandCount,
4601 : panBandMap, nPixelSpace, nLineSpace,
4602 0 : nBandSpace, psExtraArg);
4603 : }
4604 :
4605 : /* We could have a smarter implementation, but that will do for now */
4606 4 : if (psExtraArg->eResampleAlg != GRIORA_NearestNeighbour &&
4607 0 : (nBufXSize != nXSize || nBufYSize != nYSize))
4608 : {
4609 0 : return BandBasedRasterIO(eRWFlag, nXOff, nYOff, nXSize, nYSize, pData,
4610 : nBufXSize, nBufYSize, eBufType, nBandCount,
4611 : panBandMap, nPixelSpace, nLineSpace,
4612 0 : nBandSpace, psExtraArg);
4613 : }
4614 :
4615 : /* ==================================================================== */
4616 : /* Loop reading required source blocks to satisfy output */
4617 : /* request. This is the most general implementation. */
4618 : /* ==================================================================== */
4619 :
4620 4 : const int nBandDataSize = GDALGetDataTypeSizeBytes(eDataType);
4621 :
4622 : papabySrcBlock =
4623 4 : static_cast<GByte **>(CPLCalloc(sizeof(GByte *), nBandCount));
4624 : papoBlocks =
4625 4 : static_cast<GDALRasterBlock **>(CPLCalloc(sizeof(void *), nBandCount));
4626 :
4627 : /* -------------------------------------------------------------------- */
4628 : /* Select an overview level if appropriate. */
4629 : /* -------------------------------------------------------------------- */
4630 :
4631 : GDALRasterIOExtraArg sExtraArg;
4632 4 : GDALCopyRasterIOExtraArg(&sExtraArg, psExtraArg);
4633 4 : const int nOverviewLevel = GDALDatasetGetBestOverviewLevel(
4634 : this, nXOff, nYOff, nXSize, nYSize, nBufXSize, nBufYSize, nBandCount,
4635 : panBandMap, &sExtraArg);
4636 4 : if (nOverviewLevel >= 0)
4637 : {
4638 2 : GetRasterBand(panBandMap[0])
4639 2 : ->GetOverview(nOverviewLevel)
4640 2 : ->GetBlockSize(&nBlockXSize, &nBlockYSize);
4641 : }
4642 :
4643 4 : double dfXOff = nXOff;
4644 4 : double dfYOff = nYOff;
4645 4 : double dfXSize = nXSize;
4646 4 : double dfYSize = nYSize;
4647 4 : if (sExtraArg.bFloatingPointWindowValidity)
4648 : {
4649 2 : dfXOff = sExtraArg.dfXOff;
4650 2 : dfYOff = sExtraArg.dfYOff;
4651 2 : dfXSize = sExtraArg.dfXSize;
4652 2 : dfYSize = sExtraArg.dfYSize;
4653 : }
4654 :
4655 : /* -------------------------------------------------------------------- */
4656 : /* Compute stepping increment. */
4657 : /* -------------------------------------------------------------------- */
4658 4 : const double dfSrcXInc = dfXSize / static_cast<double>(nBufXSize);
4659 4 : const double dfSrcYInc = dfYSize / static_cast<double>(nBufYSize);
4660 :
4661 4 : constexpr double EPS = 1e-10;
4662 : /* -------------------------------------------------------------------- */
4663 : /* Loop over buffer computing source locations. */
4664 : /* -------------------------------------------------------------------- */
4665 36 : for (iBufYOff = 0; iBufYOff < nBufYSize; iBufYOff++)
4666 : {
4667 : GPtrDiff_t iSrcOffset;
4668 :
4669 : // Add small epsilon to avoid some numeric precision issues.
4670 32 : const double dfSrcY = (iBufYOff + 0.5) * dfSrcYInc + dfYOff + EPS;
4671 32 : const int iSrcY = static_cast<int>(std::min(
4672 32 : std::max(0.0, dfSrcY), static_cast<double>(nRasterYSize - 1)));
4673 :
4674 32 : GPtrDiff_t iBufOffset = static_cast<GPtrDiff_t>(iBufYOff) *
4675 : static_cast<GPtrDiff_t>(nLineSpace);
4676 :
4677 302 : for (iBufXOff = 0; iBufXOff < nBufXSize; iBufXOff++)
4678 : {
4679 270 : const double dfSrcX = (iBufXOff + 0.5) * dfSrcXInc + dfXOff + EPS;
4680 270 : const int iSrcX = static_cast<int>(std::min(
4681 270 : std::max(0.0, dfSrcX), static_cast<double>(nRasterXSize - 1)));
4682 :
4683 : // FIXME: this code likely doesn't work if the dirty block gets
4684 : // flushed to disk before being completely written. In the meantime,
4685 : // bJustInitialize should probably be set to FALSE even if it is not
4686 : // ideal performance wise, and for lossy compression
4687 :
4688 : /* --------------------------------------------------------------------
4689 : */
4690 : /* Ensure we have the appropriate block loaded. */
4691 : /* --------------------------------------------------------------------
4692 : */
4693 270 : if (iSrcX < nLBlockX * nBlockXSize ||
4694 270 : iSrcX - nBlockXSize >= nLBlockX * nBlockXSize ||
4695 266 : iSrcY < nLBlockY * nBlockYSize ||
4696 266 : iSrcY - nBlockYSize >= nLBlockY * nBlockYSize)
4697 : {
4698 4 : nLBlockX = iSrcX / nBlockXSize;
4699 4 : nLBlockY = iSrcY / nBlockYSize;
4700 :
4701 4 : const bool bJustInitialize =
4702 0 : eRWFlag == GF_Write && nYOff <= nLBlockY * nBlockYSize &&
4703 0 : nYOff + nYSize - nBlockYSize >= nLBlockY * nBlockYSize &&
4704 4 : nXOff <= nLBlockX * nBlockXSize &&
4705 0 : nXOff + nXSize - nBlockXSize >= nLBlockX * nBlockXSize;
4706 : /*bool bMemZeroBuffer = FALSE;
4707 : if( eRWFlag == GF_Write && !bJustInitialize &&
4708 : nXOff <= nLBlockX * nBlockXSize &&
4709 : nYOff <= nLBlockY * nBlockYSize &&
4710 : (nXOff + nXSize >= (nLBlockX+1) * nBlockXSize ||
4711 : (nXOff + nXSize == GetRasterXSize() &&
4712 : (nLBlockX+1) * nBlockXSize > GetRasterXSize())) &&
4713 : (nYOff + nYSize >= (nLBlockY+1) * nBlockYSize ||
4714 : (nYOff + nYSize == GetRasterYSize() &&
4715 : (nLBlockY+1) * nBlockYSize > GetRasterYSize())) )
4716 : {
4717 : bJustInitialize = TRUE;
4718 : bMemZeroBuffer = TRUE;
4719 : }*/
4720 12 : for (int iBand = 0; iBand < nBandCount; iBand++)
4721 : {
4722 8 : GDALRasterBand *poBand = GetRasterBand(panBandMap[iBand]);
4723 8 : if (nOverviewLevel >= 0)
4724 2 : poBand = poBand->GetOverview(nOverviewLevel);
4725 16 : poBlock = poBand->GetLockedBlockRef(nLBlockX, nLBlockY,
4726 8 : bJustInitialize);
4727 8 : if (poBlock == nullptr)
4728 : {
4729 0 : eErr = CE_Failure;
4730 0 : goto CleanupAndReturn;
4731 : }
4732 :
4733 8 : if (eRWFlag == GF_Write)
4734 0 : poBlock->MarkDirty();
4735 :
4736 8 : if (papoBlocks[iBand] != nullptr)
4737 0 : papoBlocks[iBand]->DropLock();
4738 :
4739 8 : papoBlocks[iBand] = poBlock;
4740 :
4741 8 : papabySrcBlock[iBand] =
4742 8 : static_cast<GByte *>(poBlock->GetDataRef());
4743 : /*if( bMemZeroBuffer )
4744 : {
4745 : memset(papabySrcBlock[iBand], 0,
4746 : static_cast<GPtrDiff_t>(nBandDataSize) * nBlockXSize
4747 : * nBlockYSize);
4748 : }*/
4749 : }
4750 : }
4751 :
4752 : /* --------------------------------------------------------------------
4753 : */
4754 : /* Copy over this pixel of data. */
4755 : /* --------------------------------------------------------------------
4756 : */
4757 270 : iSrcOffset = (static_cast<GPtrDiff_t>(iSrcX) -
4758 270 : static_cast<GPtrDiff_t>(nLBlockX) * nBlockXSize +
4759 270 : (static_cast<GPtrDiff_t>(iSrcY) -
4760 270 : static_cast<GPtrDiff_t>(nLBlockY) * nBlockYSize) *
4761 270 : nBlockXSize) *
4762 270 : nBandDataSize;
4763 :
4764 980 : for (int iBand = 0; iBand < nBandCount; iBand++)
4765 : {
4766 710 : GByte *pabySrcBlock = papabySrcBlock[iBand];
4767 710 : GPtrDiff_t iBandBufOffset =
4768 710 : iBufOffset + static_cast<GPtrDiff_t>(iBand) *
4769 : static_cast<GPtrDiff_t>(nBandSpace);
4770 :
4771 710 : if (eDataType == eBufType)
4772 : {
4773 710 : if (eRWFlag == GF_Read)
4774 710 : memcpy(static_cast<GByte *>(pData) + iBandBufOffset,
4775 710 : pabySrcBlock + iSrcOffset, nBandDataSize);
4776 : else
4777 0 : memcpy(pabySrcBlock + iSrcOffset,
4778 : static_cast<const GByte *>(pData) +
4779 0 : iBandBufOffset,
4780 : nBandDataSize);
4781 : }
4782 : else
4783 : {
4784 : /* type to type conversion ... ouch, this is expensive way
4785 : of handling single words */
4786 :
4787 0 : if (eRWFlag == GF_Read)
4788 0 : GDALCopyWords64(pabySrcBlock + iSrcOffset, eDataType, 0,
4789 : static_cast<GByte *>(pData) +
4790 0 : iBandBufOffset,
4791 : eBufType, 0, 1);
4792 : else
4793 0 : GDALCopyWords64(static_cast<const GByte *>(pData) +
4794 0 : iBandBufOffset,
4795 0 : eBufType, 0, pabySrcBlock + iSrcOffset,
4796 : eDataType, 0, 1);
4797 : }
4798 : }
4799 :
4800 270 : iBufOffset += static_cast<int>(nPixelSpace);
4801 : }
4802 : }
4803 :
4804 : /* -------------------------------------------------------------------- */
4805 : /* CleanupAndReturn. */
4806 : /* -------------------------------------------------------------------- */
4807 4 : CleanupAndReturn:
4808 4 : CPLFree(papabySrcBlock);
4809 4 : if (papoBlocks != nullptr)
4810 : {
4811 12 : for (int iBand = 0; iBand < nBandCount; iBand++)
4812 : {
4813 8 : if (papoBlocks[iBand] != nullptr)
4814 8 : papoBlocks[iBand]->DropLock();
4815 : }
4816 4 : CPLFree(papoBlocks);
4817 : }
4818 :
4819 4 : return eErr;
4820 : }
4821 :
4822 : //! @endcond
4823 :
4824 : /************************************************************************/
4825 : /* GDALCopyWholeRasterGetSwathSize() */
4826 : /************************************************************************/
4827 :
4828 3162 : static void GDALCopyWholeRasterGetSwathSize(GDALRasterBand *poSrcPrototypeBand,
4829 : GDALRasterBand *poDstPrototypeBand,
4830 : int nBandCount,
4831 : int bDstIsCompressed,
4832 : int bInterleave, int *pnSwathCols,
4833 : int *pnSwathLines)
4834 : {
4835 3162 : GDALDataType eDT = poDstPrototypeBand->GetRasterDataType();
4836 3162 : int nSrcBlockXSize = 0;
4837 3162 : int nSrcBlockYSize = 0;
4838 3162 : int nBlockXSize = 0;
4839 3162 : int nBlockYSize = 0;
4840 :
4841 3162 : int nXSize = poSrcPrototypeBand->GetXSize();
4842 3162 : int nYSize = poSrcPrototypeBand->GetYSize();
4843 :
4844 3162 : poSrcPrototypeBand->GetBlockSize(&nSrcBlockXSize, &nSrcBlockYSize);
4845 3162 : poDstPrototypeBand->GetBlockSize(&nBlockXSize, &nBlockYSize);
4846 :
4847 3162 : const int nMaxBlockXSize = std::max(nBlockXSize, nSrcBlockXSize);
4848 3162 : const int nMaxBlockYSize = std::max(nBlockYSize, nSrcBlockYSize);
4849 :
4850 3162 : int nPixelSize = GDALGetDataTypeSizeBytes(eDT);
4851 3162 : if (bInterleave)
4852 550 : nPixelSize *= nBandCount;
4853 :
4854 : // aim for one row of blocks. Do not settle for less.
4855 3162 : int nSwathCols = nXSize;
4856 3162 : int nSwathLines = nMaxBlockYSize;
4857 :
4858 : const char *pszSrcCompression =
4859 3162 : poSrcPrototypeBand->GetMetadataItem("COMPRESSION", "IMAGE_STRUCTURE");
4860 3162 : if (pszSrcCompression == nullptr)
4861 : {
4862 3136 : auto poSrcDS = poSrcPrototypeBand->GetDataset();
4863 3136 : if (poSrcDS)
4864 : pszSrcCompression =
4865 3130 : poSrcDS->GetMetadataItem("COMPRESSION", "IMAGE_STRUCTURE");
4866 : }
4867 :
4868 : /* -------------------------------------------------------------------- */
4869 : /* What will our swath size be? */
4870 : /* -------------------------------------------------------------------- */
4871 : // When writing interleaved data in a compressed format, we want to be sure
4872 : // that each block will only be written once, so the swath size must not be
4873 : // greater than the block cache.
4874 3162 : const char *pszSwathSize = CPLGetConfigOption("GDAL_SWATH_SIZE", nullptr);
4875 : int nTargetSwathSize;
4876 3162 : if (pszSwathSize != nullptr)
4877 0 : nTargetSwathSize = static_cast<int>(
4878 0 : std::min(GIntBig(INT_MAX), CPLAtoGIntBig(pszSwathSize)));
4879 : else
4880 : {
4881 : // As a default, take one 1/4 of the cache size.
4882 3162 : nTargetSwathSize = static_cast<int>(
4883 3162 : std::min(GIntBig(INT_MAX), GDALGetCacheMax64() / 4));
4884 :
4885 : // but if the minimum idal swath buf size is less, then go for it to
4886 : // avoid unnecessarily abusing RAM usage.
4887 : // but try to use 10 MB at least.
4888 3162 : GIntBig nIdealSwathBufSize =
4889 3162 : static_cast<GIntBig>(nSwathCols) * nSwathLines * nPixelSize;
4890 3162 : int nMinTargetSwathSize = 10 * 1000 * 1000;
4891 :
4892 3162 : if ((poSrcPrototypeBand->GetSuggestedBlockAccessPattern() &
4893 3162 : GSBAP_LARGEST_CHUNK_POSSIBLE) != 0)
4894 : {
4895 2 : nMinTargetSwathSize = nTargetSwathSize;
4896 : }
4897 :
4898 3162 : if (nIdealSwathBufSize < nTargetSwathSize &&
4899 3152 : nIdealSwathBufSize < nMinTargetSwathSize)
4900 : {
4901 3149 : nIdealSwathBufSize = nMinTargetSwathSize;
4902 : }
4903 :
4904 3162 : if (pszSrcCompression != nullptr &&
4905 180 : EQUAL(pszSrcCompression, "JPEG2000") &&
4906 0 : (!bDstIsCompressed || ((nSrcBlockXSize % nBlockXSize) == 0 &&
4907 0 : (nSrcBlockYSize % nBlockYSize) == 0)))
4908 : {
4909 2 : nIdealSwathBufSize =
4910 4 : std::max(nIdealSwathBufSize, static_cast<GIntBig>(nSwathCols) *
4911 2 : nSrcBlockYSize * nPixelSize);
4912 : }
4913 3162 : if (nTargetSwathSize > nIdealSwathBufSize)
4914 3148 : nTargetSwathSize = static_cast<int>(
4915 3148 : std::min(GIntBig(INT_MAX), nIdealSwathBufSize));
4916 : }
4917 :
4918 3162 : if (nTargetSwathSize < 1000000)
4919 8 : nTargetSwathSize = 1000000;
4920 :
4921 : /* But let's check that */
4922 3380 : if (bDstIsCompressed && bInterleave &&
4923 218 : nTargetSwathSize > GDALGetCacheMax64())
4924 : {
4925 0 : CPLError(CE_Warning, CPLE_AppDefined,
4926 : "When translating into a compressed interleave format, "
4927 : "the block cache size (" CPL_FRMT_GIB ") "
4928 : "should be at least the size of the swath (%d) "
4929 : "(GDAL_SWATH_SIZE config. option)",
4930 : GDALGetCacheMax64(), nTargetSwathSize);
4931 : }
4932 :
4933 : #define IS_DIVIDER_OF(x, y) ((y) % (x) == 0)
4934 : #define ROUND_TO(x, y) (((x) / (y)) * (y))
4935 :
4936 : // if both input and output datasets are tiled, that the tile dimensions
4937 : // are "compatible", try to stick to a swath dimension that is a multiple
4938 : // of input and output block dimensions.
4939 3162 : if (nBlockXSize != nXSize && nSrcBlockXSize != nXSize &&
4940 38 : IS_DIVIDER_OF(nBlockXSize, nMaxBlockXSize) &&
4941 38 : IS_DIVIDER_OF(nSrcBlockXSize, nMaxBlockXSize) &&
4942 38 : IS_DIVIDER_OF(nBlockYSize, nMaxBlockYSize) &&
4943 38 : IS_DIVIDER_OF(nSrcBlockYSize, nMaxBlockYSize))
4944 : {
4945 38 : if (static_cast<GIntBig>(nMaxBlockXSize) * nMaxBlockYSize *
4946 38 : nPixelSize <=
4947 38 : static_cast<GIntBig>(nTargetSwathSize))
4948 : {
4949 38 : nSwathCols = nTargetSwathSize / (nMaxBlockYSize * nPixelSize);
4950 38 : nSwathCols = ROUND_TO(nSwathCols, nMaxBlockXSize);
4951 38 : if (nSwathCols == 0)
4952 0 : nSwathCols = nMaxBlockXSize;
4953 38 : if (nSwathCols > nXSize)
4954 36 : nSwathCols = nXSize;
4955 38 : nSwathLines = nMaxBlockYSize;
4956 :
4957 38 : if (static_cast<GIntBig>(nSwathCols) * nSwathLines * nPixelSize >
4958 38 : static_cast<GIntBig>(nTargetSwathSize))
4959 : {
4960 0 : nSwathCols = nXSize;
4961 0 : nSwathLines = nBlockYSize;
4962 : }
4963 : }
4964 : }
4965 :
4966 3162 : const GIntBig nMemoryPerCol = static_cast<GIntBig>(nSwathCols) * nPixelSize;
4967 3162 : const GIntBig nSwathBufSize = nMemoryPerCol * nSwathLines;
4968 3162 : if (nSwathBufSize > static_cast<GIntBig>(nTargetSwathSize))
4969 : {
4970 1 : nSwathLines = static_cast<int>(nTargetSwathSize / nMemoryPerCol);
4971 1 : if (nSwathLines == 0)
4972 1 : nSwathLines = 1;
4973 :
4974 1 : CPLDebug(
4975 : "GDAL",
4976 : "GDALCopyWholeRasterGetSwathSize(): adjusting to %d line swath "
4977 : "since requirement (" CPL_FRMT_GIB " bytes) exceed target swath "
4978 : "size (%d bytes) (GDAL_SWATH_SIZE config. option)",
4979 1 : nSwathLines, nBlockYSize * nMemoryPerCol, nTargetSwathSize);
4980 : }
4981 : // If we are processing single scans, try to handle several at once.
4982 : // If we are handling swaths already, only grow the swath if a row
4983 : // of blocks is substantially less than our target buffer size.
4984 3161 : else if (nSwathLines == 1 ||
4985 2620 : nMemoryPerCol * nSwathLines <
4986 2620 : static_cast<GIntBig>(nTargetSwathSize) / 10)
4987 : {
4988 3133 : nSwathLines = std::min(
4989 : nYSize,
4990 3133 : std::max(1, static_cast<int>(nTargetSwathSize / nMemoryPerCol)));
4991 :
4992 : /* If possible try to align to source and target block height */
4993 3133 : if ((nSwathLines % nMaxBlockYSize) != 0 &&
4994 251 : nSwathLines > nMaxBlockYSize &&
4995 251 : IS_DIVIDER_OF(nBlockYSize, nMaxBlockYSize) &&
4996 222 : IS_DIVIDER_OF(nSrcBlockYSize, nMaxBlockYSize))
4997 202 : nSwathLines = ROUND_TO(nSwathLines, nMaxBlockYSize);
4998 : }
4999 :
5000 3162 : if (pszSrcCompression != nullptr && EQUAL(pszSrcCompression, "JPEG2000") &&
5001 0 : (!bDstIsCompressed || (IS_DIVIDER_OF(nBlockXSize, nSrcBlockXSize) &&
5002 0 : IS_DIVIDER_OF(nBlockYSize, nSrcBlockYSize))))
5003 : {
5004 : // Typical use case: converting from Pleaiades that is 2048x2048 tiled.
5005 2 : if (nSwathLines < nSrcBlockYSize)
5006 : {
5007 0 : nSwathLines = nSrcBlockYSize;
5008 :
5009 : // Number of pixels that can be read/write simultaneously.
5010 0 : nSwathCols = nTargetSwathSize / (nSrcBlockXSize * nPixelSize);
5011 0 : nSwathCols = ROUND_TO(nSwathCols, nSrcBlockXSize);
5012 0 : if (nSwathCols == 0)
5013 0 : nSwathCols = nSrcBlockXSize;
5014 0 : if (nSwathCols > nXSize)
5015 0 : nSwathCols = nXSize;
5016 :
5017 0 : CPLDebug(
5018 : "GDAL",
5019 : "GDALCopyWholeRasterGetSwathSize(): because of compression and "
5020 : "too high block, "
5021 : "use partial width at one time");
5022 : }
5023 2 : else if ((nSwathLines % nSrcBlockYSize) != 0)
5024 : {
5025 : /* Round on a multiple of nSrcBlockYSize */
5026 0 : nSwathLines = ROUND_TO(nSwathLines, nSrcBlockYSize);
5027 0 : CPLDebug(
5028 : "GDAL",
5029 : "GDALCopyWholeRasterGetSwathSize(): because of compression, "
5030 : "round nSwathLines to block height : %d",
5031 : nSwathLines);
5032 : }
5033 : }
5034 3160 : else if (bDstIsCompressed)
5035 : {
5036 408 : if (nSwathLines < nBlockYSize)
5037 : {
5038 146 : nSwathLines = nBlockYSize;
5039 :
5040 : // Number of pixels that can be read/write simultaneously.
5041 146 : nSwathCols = nTargetSwathSize / (nSwathLines * nPixelSize);
5042 146 : nSwathCols = ROUND_TO(nSwathCols, nBlockXSize);
5043 146 : if (nSwathCols == 0)
5044 0 : nSwathCols = nBlockXSize;
5045 146 : if (nSwathCols > nXSize)
5046 146 : nSwathCols = nXSize;
5047 :
5048 146 : CPLDebug(
5049 : "GDAL",
5050 : "GDALCopyWholeRasterGetSwathSize(): because of compression and "
5051 : "too high block, "
5052 : "use partial width at one time");
5053 : }
5054 262 : else if ((nSwathLines % nBlockYSize) != 0)
5055 : {
5056 : // Round on a multiple of nBlockYSize.
5057 9 : nSwathLines = ROUND_TO(nSwathLines, nBlockYSize);
5058 9 : CPLDebug(
5059 : "GDAL",
5060 : "GDALCopyWholeRasterGetSwathSize(): because of compression, "
5061 : "round nSwathLines to block height : %d",
5062 : nSwathLines);
5063 : }
5064 : }
5065 :
5066 3162 : *pnSwathCols = nSwathCols;
5067 3162 : *pnSwathLines = nSwathLines;
5068 3162 : }
5069 :
5070 : /************************************************************************/
5071 : /* GDALDatasetCopyWholeRaster() */
5072 : /************************************************************************/
5073 :
5074 : /**
5075 : * \brief Copy all dataset raster data.
5076 : *
5077 : * This function copies the complete raster contents of one dataset to
5078 : * another similarly configured dataset. The source and destination
5079 : * dataset must have the same number of bands, and the same width
5080 : * and height. The bands do not have to have the same data type.
5081 : *
5082 : * This function is primarily intended to support implementation of
5083 : * driver specific CreateCopy() functions. It implements efficient copying,
5084 : * in particular "chunking" the copy in substantial blocks and, if appropriate,
5085 : * performing the transfer in a pixel interleaved fashion.
5086 : *
5087 : * Currently the only papszOptions value supported are :
5088 : * <ul>
5089 : * <li>"INTERLEAVE=PIXEL/BAND" to force pixel (resp. band) interleaved read and
5090 : * write access pattern (this does not modify the layout of the destination
5091 : * data)</li> <li>"COMPRESSED=YES" to force alignment on target dataset block
5092 : * sizes to achieve best compression.</li> <li>"SKIP_HOLES=YES" to skip chunks
5093 : * for which GDALGetDataCoverageStatus() returns GDAL_DATA_COVERAGE_STATUS_EMPTY
5094 : * (GDAL >= 2.2)</li>
5095 : * </ul>
5096 : * More options may be supported in the future.
5097 : *
5098 : * @param hSrcDS the source dataset
5099 : * @param hDstDS the destination dataset
5100 : * @param papszOptions transfer hints in "StringList" Name=Value format.
5101 : * @param pfnProgress progress reporting function.
5102 : * @param pProgressData callback data for progress function.
5103 : *
5104 : * @return CE_None on success, or CE_Failure on failure.
5105 : */
5106 :
5107 3135 : CPLErr CPL_STDCALL GDALDatasetCopyWholeRaster(GDALDatasetH hSrcDS,
5108 : GDALDatasetH hDstDS,
5109 : CSLConstList papszOptions,
5110 : GDALProgressFunc pfnProgress,
5111 : void *pProgressData)
5112 :
5113 : {
5114 3135 : VALIDATE_POINTER1(hSrcDS, "GDALDatasetCopyWholeRaster", CE_Failure);
5115 3135 : VALIDATE_POINTER1(hDstDS, "GDALDatasetCopyWholeRaster", CE_Failure);
5116 :
5117 3135 : GDALDataset *poSrcDS = GDALDataset::FromHandle(hSrcDS);
5118 3135 : GDALDataset *poDstDS = GDALDataset::FromHandle(hDstDS);
5119 :
5120 3135 : if (pfnProgress == nullptr)
5121 0 : pfnProgress = GDALDummyProgress;
5122 :
5123 : /* -------------------------------------------------------------------- */
5124 : /* Confirm the datasets match in size and band counts. */
5125 : /* -------------------------------------------------------------------- */
5126 3135 : const int nXSize = poDstDS->GetRasterXSize();
5127 3135 : const int nYSize = poDstDS->GetRasterYSize();
5128 3135 : const int nBandCount = poDstDS->GetRasterCount();
5129 :
5130 3135 : if (poSrcDS->GetRasterXSize() != nXSize ||
5131 6270 : poSrcDS->GetRasterYSize() != nYSize ||
5132 3135 : poSrcDS->GetRasterCount() != nBandCount)
5133 : {
5134 0 : CPLError(CE_Failure, CPLE_AppDefined,
5135 : "Input and output dataset sizes or band counts do not\n"
5136 : "match in GDALDatasetCopyWholeRaster()");
5137 0 : return CE_Failure;
5138 : }
5139 :
5140 : /* -------------------------------------------------------------------- */
5141 : /* Report preliminary (0) progress. */
5142 : /* -------------------------------------------------------------------- */
5143 3135 : if (!pfnProgress(0.0, nullptr, pProgressData))
5144 : {
5145 1 : CPLError(CE_Failure, CPLE_UserInterrupt,
5146 : "User terminated CreateCopy()");
5147 1 : return CE_Failure;
5148 : }
5149 :
5150 : /* -------------------------------------------------------------------- */
5151 : /* Get our prototype band, and assume the others are similarly */
5152 : /* configured. */
5153 : /* -------------------------------------------------------------------- */
5154 3134 : if (nBandCount == 0)
5155 0 : return CE_None;
5156 :
5157 3134 : GDALRasterBand *poSrcPrototypeBand = poSrcDS->GetRasterBand(1);
5158 3134 : GDALRasterBand *poDstPrototypeBand = poDstDS->GetRasterBand(1);
5159 3134 : GDALDataType eDT = poDstPrototypeBand->GetRasterDataType();
5160 :
5161 : /* -------------------------------------------------------------------- */
5162 : /* Do we want to try and do the operation in a pixel */
5163 : /* interleaved fashion? */
5164 : /* -------------------------------------------------------------------- */
5165 3134 : bool bInterleave = false;
5166 : const char *pszInterleave =
5167 3134 : poSrcDS->GetMetadataItem("INTERLEAVE", "IMAGE_STRUCTURE");
5168 3134 : if (pszInterleave != nullptr &&
5169 2779 : (EQUAL(pszInterleave, "PIXEL") || EQUAL(pszInterleave, "LINE")))
5170 184 : bInterleave = true;
5171 :
5172 3134 : pszInterleave = poDstDS->GetMetadataItem("INTERLEAVE", "IMAGE_STRUCTURE");
5173 3134 : if (pszInterleave != nullptr &&
5174 2679 : (EQUAL(pszInterleave, "PIXEL") || EQUAL(pszInterleave, "LINE")))
5175 497 : bInterleave = true;
5176 :
5177 3134 : pszInterleave = CSLFetchNameValue(papszOptions, "INTERLEAVE");
5178 3134 : if (pszInterleave != nullptr && EQUAL(pszInterleave, "PIXEL"))
5179 5 : bInterleave = true;
5180 3129 : else if (pszInterleave != nullptr && EQUAL(pszInterleave, "BAND"))
5181 13 : bInterleave = false;
5182 : // attributes is specific to the TileDB driver
5183 3116 : else if (pszInterleave != nullptr && EQUAL(pszInterleave, "ATTRIBUTES"))
5184 4 : bInterleave = true;
5185 3112 : else if (pszInterleave != nullptr)
5186 : {
5187 0 : CPLError(CE_Warning, CPLE_NotSupported,
5188 : "Unsupported value for option INTERLEAVE");
5189 : }
5190 :
5191 : // If the destination is compressed, we must try to write blocks just once,
5192 : // to save disk space (GTiff case for example), and to avoid data loss
5193 : // (JPEG compression for example).
5194 3134 : bool bDstIsCompressed = false;
5195 : const char *pszDstCompressed =
5196 3134 : CSLFetchNameValue(papszOptions, "COMPRESSED");
5197 3134 : if (pszDstCompressed != nullptr && CPLTestBool(pszDstCompressed))
5198 383 : bDstIsCompressed = true;
5199 :
5200 : /* -------------------------------------------------------------------- */
5201 : /* What will our swath size be? */
5202 : /* -------------------------------------------------------------------- */
5203 :
5204 3134 : int nSwathCols = 0;
5205 3134 : int nSwathLines = 0;
5206 3134 : GDALCopyWholeRasterGetSwathSize(poSrcPrototypeBand, poDstPrototypeBand,
5207 : nBandCount, bDstIsCompressed, bInterleave,
5208 : &nSwathCols, &nSwathLines);
5209 :
5210 3134 : int nPixelSize = GDALGetDataTypeSizeBytes(eDT);
5211 3134 : if (bInterleave)
5212 550 : nPixelSize *= nBandCount;
5213 :
5214 3134 : void *pSwathBuf = VSI_MALLOC3_VERBOSE(nSwathCols, nSwathLines, nPixelSize);
5215 3134 : if (pSwathBuf == nullptr)
5216 : {
5217 0 : return CE_Failure;
5218 : }
5219 :
5220 3134 : CPLDebug("GDAL",
5221 : "GDALDatasetCopyWholeRaster(): %d*%d swaths, bInterleave=%d",
5222 : nSwathCols, nSwathLines, static_cast<int>(bInterleave));
5223 :
5224 : // Advise the source raster that we are going to read it completely
5225 : // Note: this might already have been done by GDALCreateCopy() in the
5226 : // likely case this function is indirectly called by it
5227 3134 : poSrcDS->AdviseRead(0, 0, nXSize, nYSize, nXSize, nYSize, eDT, nBandCount,
5228 3134 : nullptr, nullptr);
5229 :
5230 : /* ==================================================================== */
5231 : /* Band oriented (uninterleaved) case. */
5232 : /* ==================================================================== */
5233 3134 : CPLErr eErr = CE_None;
5234 : const bool bCheckHoles =
5235 3134 : CPLTestBool(CSLFetchNameValueDef(papszOptions, "SKIP_HOLES", "NO"));
5236 :
5237 3134 : if (!bInterleave)
5238 : {
5239 : GDALRasterIOExtraArg sExtraArg;
5240 2584 : INIT_RASTERIO_EXTRA_ARG(sExtraArg);
5241 2584 : CPL_IGNORE_RET_VAL(sExtraArg.pfnProgress); // to make cppcheck happy
5242 :
5243 7752 : const GIntBig nTotalBlocks = static_cast<GIntBig>(nBandCount) *
5244 2584 : DIV_ROUND_UP(nYSize, nSwathLines) *
5245 2584 : DIV_ROUND_UP(nXSize, nSwathCols);
5246 2584 : GIntBig nBlocksDone = 0;
5247 :
5248 7561 : for (int iBand = 0; iBand < nBandCount && eErr == CE_None; iBand++)
5249 : {
5250 4977 : int nBand = iBand + 1;
5251 :
5252 10269 : for (int iY = 0; iY < nYSize && eErr == CE_None; iY += nSwathLines)
5253 : {
5254 5292 : int nThisLines = nSwathLines;
5255 :
5256 5292 : if (iY + nThisLines > nYSize)
5257 375 : nThisLines = nYSize - iY;
5258 :
5259 10584 : for (int iX = 0; iX < nXSize && eErr == CE_None;
5260 5292 : iX += nSwathCols)
5261 : {
5262 5292 : int nThisCols = nSwathCols;
5263 :
5264 5292 : if (iX + nThisCols > nXSize)
5265 0 : nThisCols = nXSize - iX;
5266 :
5267 5292 : int nStatus = GDAL_DATA_COVERAGE_STATUS_DATA;
5268 5292 : if (bCheckHoles)
5269 : {
5270 : nStatus = poSrcDS->GetRasterBand(nBand)
5271 3640 : ->GetDataCoverageStatus(
5272 : iX, iY, nThisCols, nThisLines,
5273 : GDAL_DATA_COVERAGE_STATUS_DATA);
5274 : }
5275 5292 : if (nStatus & GDAL_DATA_COVERAGE_STATUS_DATA)
5276 : {
5277 5288 : sExtraArg.pfnProgress = GDALScaledProgress;
5278 10576 : sExtraArg.pProgressData = GDALCreateScaledProgress(
5279 5288 : nBlocksDone / static_cast<double>(nTotalBlocks),
5280 5288 : (nBlocksDone + 0.5) /
5281 5288 : static_cast<double>(nTotalBlocks),
5282 : pfnProgress, pProgressData);
5283 5288 : if (sExtraArg.pProgressData == nullptr)
5284 1622 : sExtraArg.pfnProgress = nullptr;
5285 :
5286 5288 : eErr = poSrcDS->RasterIO(GF_Read, iX, iY, nThisCols,
5287 : nThisLines, pSwathBuf,
5288 : nThisCols, nThisLines, eDT, 1,
5289 : &nBand, 0, 0, 0, &sExtraArg);
5290 :
5291 5288 : GDALDestroyScaledProgress(sExtraArg.pProgressData);
5292 :
5293 5288 : if (eErr == CE_None)
5294 5281 : eErr = poDstDS->RasterIO(
5295 : GF_Write, iX, iY, nThisCols, nThisLines,
5296 : pSwathBuf, nThisCols, nThisLines, eDT, 1,
5297 : &nBand, 0, 0, 0, nullptr);
5298 : }
5299 :
5300 5292 : nBlocksDone++;
5301 10542 : if (eErr == CE_None &&
5302 5250 : !pfnProgress(nBlocksDone /
5303 5250 : static_cast<double>(nTotalBlocks),
5304 : nullptr, pProgressData))
5305 : {
5306 2 : eErr = CE_Failure;
5307 2 : CPLError(CE_Failure, CPLE_UserInterrupt,
5308 : "User terminated CreateCopy()");
5309 : }
5310 : }
5311 : }
5312 : }
5313 : }
5314 :
5315 : /* ==================================================================== */
5316 : /* Pixel interleaved case. */
5317 : /* ==================================================================== */
5318 : else /* if( bInterleave ) */
5319 : {
5320 : GDALRasterIOExtraArg sExtraArg;
5321 550 : INIT_RASTERIO_EXTRA_ARG(sExtraArg);
5322 550 : CPL_IGNORE_RET_VAL(sExtraArg.pfnProgress); // to make cppcheck happy
5323 :
5324 550 : const GIntBig nTotalBlocks =
5325 550 : static_cast<GIntBig>(DIV_ROUND_UP(nYSize, nSwathLines)) *
5326 550 : DIV_ROUND_UP(nXSize, nSwathCols);
5327 550 : GIntBig nBlocksDone = 0;
5328 :
5329 1315 : for (int iY = 0; iY < nYSize && eErr == CE_None; iY += nSwathLines)
5330 : {
5331 765 : int nThisLines = nSwathLines;
5332 :
5333 765 : if (iY + nThisLines > nYSize)
5334 191 : nThisLines = nYSize - iY;
5335 :
5336 1535 : for (int iX = 0; iX < nXSize && eErr == CE_None; iX += nSwathCols)
5337 : {
5338 770 : int nThisCols = nSwathCols;
5339 :
5340 770 : if (iX + nThisCols > nXSize)
5341 3 : nThisCols = nXSize - iX;
5342 :
5343 770 : int nStatus = GDAL_DATA_COVERAGE_STATUS_DATA;
5344 770 : if (bCheckHoles)
5345 : {
5346 540 : nStatus = 0;
5347 593 : for (int iBand = 0; iBand < nBandCount; iBand++)
5348 : {
5349 574 : nStatus |= poSrcDS->GetRasterBand(iBand + 1)
5350 574 : ->GetDataCoverageStatus(
5351 : iX, iY, nThisCols, nThisLines,
5352 : GDAL_DATA_COVERAGE_STATUS_DATA);
5353 574 : if (nStatus & GDAL_DATA_COVERAGE_STATUS_DATA)
5354 521 : break;
5355 : }
5356 : }
5357 770 : if (nStatus & GDAL_DATA_COVERAGE_STATUS_DATA)
5358 : {
5359 751 : sExtraArg.pfnProgress = GDALScaledProgress;
5360 1502 : sExtraArg.pProgressData = GDALCreateScaledProgress(
5361 751 : nBlocksDone / static_cast<double>(nTotalBlocks),
5362 751 : (nBlocksDone + 0.5) / static_cast<double>(nTotalBlocks),
5363 : pfnProgress, pProgressData);
5364 751 : if (sExtraArg.pProgressData == nullptr)
5365 343 : sExtraArg.pfnProgress = nullptr;
5366 :
5367 751 : eErr = poSrcDS->RasterIO(GF_Read, iX, iY, nThisCols,
5368 : nThisLines, pSwathBuf, nThisCols,
5369 : nThisLines, eDT, nBandCount,
5370 : nullptr, 0, 0, 0, &sExtraArg);
5371 :
5372 751 : GDALDestroyScaledProgress(sExtraArg.pProgressData);
5373 :
5374 751 : if (eErr == CE_None)
5375 750 : eErr = poDstDS->RasterIO(
5376 : GF_Write, iX, iY, nThisCols, nThisLines, pSwathBuf,
5377 : nThisCols, nThisLines, eDT, nBandCount, nullptr, 0,
5378 : 0, 0, nullptr);
5379 : }
5380 :
5381 770 : nBlocksDone++;
5382 1536 : if (eErr == CE_None &&
5383 766 : !pfnProgress(nBlocksDone /
5384 766 : static_cast<double>(nTotalBlocks),
5385 : nullptr, pProgressData))
5386 : {
5387 1 : eErr = CE_Failure;
5388 1 : CPLError(CE_Failure, CPLE_UserInterrupt,
5389 : "User terminated CreateCopy()");
5390 : }
5391 : }
5392 : }
5393 : }
5394 :
5395 : /* -------------------------------------------------------------------- */
5396 : /* Cleanup */
5397 : /* -------------------------------------------------------------------- */
5398 3134 : CPLFree(pSwathBuf);
5399 :
5400 3134 : return eErr;
5401 : }
5402 :
5403 : /************************************************************************/
5404 : /* GDALRasterBandCopyWholeRaster() */
5405 : /************************************************************************/
5406 :
5407 : /**
5408 : * \brief Copy a whole raster band
5409 : *
5410 : * This function copies the complete raster contents of one band to
5411 : * another similarly configured band. The source and destination
5412 : * bands must have the same width and height. The bands do not have
5413 : * to have the same data type.
5414 : *
5415 : * It implements efficient copying, in particular "chunking" the copy in
5416 : * substantial blocks.
5417 : *
5418 : * Currently the only papszOptions value supported are :
5419 : * <ul>
5420 : * <li>"COMPRESSED=YES" to force alignment on target dataset block sizes to
5421 : * achieve best compression.</li>
5422 : * <li>"SKIP_HOLES=YES" to skip chunks for which GDALGetDataCoverageStatus()
5423 : * returns GDAL_DATA_COVERAGE_STATUS_EMPTY (GDAL >= 2.2)</li>
5424 : * </ul>
5425 : *
5426 : * @param hSrcBand the source band
5427 : * @param hDstBand the destination band
5428 : * @param papszOptions transfer hints in "StringList" Name=Value format.
5429 : * @param pfnProgress progress reporting function.
5430 : * @param pProgressData callback data for progress function.
5431 : *
5432 : * @return CE_None on success, or CE_Failure on failure.
5433 : */
5434 :
5435 28 : CPLErr CPL_STDCALL GDALRasterBandCopyWholeRaster(
5436 : GDALRasterBandH hSrcBand, GDALRasterBandH hDstBand,
5437 : const char *const *const papszOptions, GDALProgressFunc pfnProgress,
5438 : void *pProgressData)
5439 :
5440 : {
5441 28 : VALIDATE_POINTER1(hSrcBand, "GDALRasterBandCopyWholeRaster", CE_Failure);
5442 28 : VALIDATE_POINTER1(hDstBand, "GDALRasterBandCopyWholeRaster", CE_Failure);
5443 :
5444 28 : GDALRasterBand *poSrcBand = GDALRasterBand::FromHandle(hSrcBand);
5445 28 : GDALRasterBand *poDstBand = GDALRasterBand::FromHandle(hDstBand);
5446 28 : CPLErr eErr = CE_None;
5447 :
5448 28 : if (pfnProgress == nullptr)
5449 2 : pfnProgress = GDALDummyProgress;
5450 :
5451 : /* -------------------------------------------------------------------- */
5452 : /* Confirm the datasets match in size and band counts. */
5453 : /* -------------------------------------------------------------------- */
5454 28 : int nXSize = poSrcBand->GetXSize();
5455 28 : int nYSize = poSrcBand->GetYSize();
5456 :
5457 28 : if (poDstBand->GetXSize() != nXSize || poDstBand->GetYSize() != nYSize)
5458 : {
5459 0 : CPLError(CE_Failure, CPLE_AppDefined,
5460 : "Input and output band sizes do not\n"
5461 : "match in GDALRasterBandCopyWholeRaster()");
5462 0 : return CE_Failure;
5463 : }
5464 :
5465 : /* -------------------------------------------------------------------- */
5466 : /* Report preliminary (0) progress. */
5467 : /* -------------------------------------------------------------------- */
5468 28 : if (!pfnProgress(0.0, nullptr, pProgressData))
5469 : {
5470 0 : CPLError(CE_Failure, CPLE_UserInterrupt,
5471 : "User terminated CreateCopy()");
5472 0 : return CE_Failure;
5473 : }
5474 :
5475 28 : GDALDataType eDT = poDstBand->GetRasterDataType();
5476 :
5477 : // If the destination is compressed, we must try to write blocks just once,
5478 : // to save disk space (GTiff case for example), and to avoid data loss
5479 : // (JPEG compression for example).
5480 28 : bool bDstIsCompressed = false;
5481 : const char *pszDstCompressed =
5482 28 : CSLFetchNameValue(const_cast<char **>(papszOptions), "COMPRESSED");
5483 28 : if (pszDstCompressed != nullptr && CPLTestBool(pszDstCompressed))
5484 25 : bDstIsCompressed = true;
5485 :
5486 : /* -------------------------------------------------------------------- */
5487 : /* What will our swath size be? */
5488 : /* -------------------------------------------------------------------- */
5489 :
5490 28 : int nSwathCols = 0;
5491 28 : int nSwathLines = 0;
5492 28 : GDALCopyWholeRasterGetSwathSize(poSrcBand, poDstBand, 1, bDstIsCompressed,
5493 : FALSE, &nSwathCols, &nSwathLines);
5494 :
5495 28 : const int nPixelSize = GDALGetDataTypeSizeBytes(eDT);
5496 :
5497 28 : void *pSwathBuf = VSI_MALLOC3_VERBOSE(nSwathCols, nSwathLines, nPixelSize);
5498 28 : if (pSwathBuf == nullptr)
5499 : {
5500 0 : return CE_Failure;
5501 : }
5502 :
5503 28 : CPLDebug("GDAL", "GDALRasterBandCopyWholeRaster(): %d*%d swaths",
5504 : nSwathCols, nSwathLines);
5505 :
5506 : const bool bCheckHoles =
5507 28 : CPLTestBool(CSLFetchNameValueDef(papszOptions, "SKIP_HOLES", "NO"));
5508 :
5509 : // Advise the source raster that we are going to read it completely
5510 28 : poSrcBand->AdviseRead(0, 0, nXSize, nYSize, nXSize, nYSize, eDT, nullptr);
5511 :
5512 : /* ==================================================================== */
5513 : /* Band oriented (uninterleaved) case. */
5514 : /* ==================================================================== */
5515 :
5516 70 : for (int iY = 0; iY < nYSize && eErr == CE_None; iY += nSwathLines)
5517 : {
5518 42 : int nThisLines = nSwathLines;
5519 :
5520 42 : if (iY + nThisLines > nYSize)
5521 8 : nThisLines = nYSize - iY;
5522 :
5523 84 : for (int iX = 0; iX < nXSize && eErr == CE_None; iX += nSwathCols)
5524 : {
5525 42 : int nThisCols = nSwathCols;
5526 :
5527 42 : if (iX + nThisCols > nXSize)
5528 0 : nThisCols = nXSize - iX;
5529 :
5530 42 : int nStatus = GDAL_DATA_COVERAGE_STATUS_DATA;
5531 42 : if (bCheckHoles)
5532 : {
5533 0 : nStatus = poSrcBand->GetDataCoverageStatus(
5534 : iX, iY, nThisCols, nThisLines,
5535 : GDAL_DATA_COVERAGE_STATUS_DATA);
5536 : }
5537 42 : if (nStatus & GDAL_DATA_COVERAGE_STATUS_DATA)
5538 : {
5539 42 : eErr = poSrcBand->RasterIO(GF_Read, iX, iY, nThisCols,
5540 : nThisLines, pSwathBuf, nThisCols,
5541 : nThisLines, eDT, 0, 0, nullptr);
5542 :
5543 42 : if (eErr == CE_None)
5544 42 : eErr = poDstBand->RasterIO(GF_Write, iX, iY, nThisCols,
5545 : nThisLines, pSwathBuf, nThisCols,
5546 : nThisLines, eDT, 0, 0, nullptr);
5547 : }
5548 :
5549 84 : if (eErr == CE_None &&
5550 42 : !pfnProgress((iY + nThisLines) / static_cast<float>(nYSize),
5551 : nullptr, pProgressData))
5552 : {
5553 0 : eErr = CE_Failure;
5554 0 : CPLError(CE_Failure, CPLE_UserInterrupt,
5555 : "User terminated CreateCopy()");
5556 : }
5557 : }
5558 : }
5559 :
5560 : /* -------------------------------------------------------------------- */
5561 : /* Cleanup */
5562 : /* -------------------------------------------------------------------- */
5563 28 : CPLFree(pSwathBuf);
5564 :
5565 28 : return eErr;
5566 : }
5567 :
5568 : /************************************************************************/
5569 : /* GDALCopyRasterIOExtraArg () */
5570 : /************************************************************************/
5571 :
5572 526974 : void GDALCopyRasterIOExtraArg(GDALRasterIOExtraArg *psDestArg,
5573 : GDALRasterIOExtraArg *psSrcArg)
5574 : {
5575 526974 : INIT_RASTERIO_EXTRA_ARG(*psDestArg);
5576 526974 : if (psSrcArg)
5577 : {
5578 526974 : psDestArg->eResampleAlg = psSrcArg->eResampleAlg;
5579 526974 : psDestArg->pfnProgress = psSrcArg->pfnProgress;
5580 526974 : psDestArg->pProgressData = psSrcArg->pProgressData;
5581 526974 : psDestArg->bFloatingPointWindowValidity =
5582 526974 : psSrcArg->bFloatingPointWindowValidity;
5583 526974 : if (psSrcArg->bFloatingPointWindowValidity)
5584 : {
5585 204182 : psDestArg->dfXOff = psSrcArg->dfXOff;
5586 204182 : psDestArg->dfYOff = psSrcArg->dfYOff;
5587 204182 : psDestArg->dfXSize = psSrcArg->dfXSize;
5588 204182 : psDestArg->dfYSize = psSrcArg->dfYSize;
5589 : }
5590 526974 : if (psSrcArg->nVersion >= 2)
5591 : {
5592 526974 : psDestArg->bUseOnlyThisScale = psSrcArg->bUseOnlyThisScale;
5593 : }
5594 : }
5595 526974 : }
5596 :
5597 : /************************************************************************/
5598 : /* HasOnlyNoData() */
5599 : /************************************************************************/
5600 :
5601 24858136 : template <class T> static inline bool IsEqualToNoData(T value, T noDataValue)
5602 : {
5603 24858136 : return value == noDataValue;
5604 : }
5605 :
5606 0 : template <> bool IsEqualToNoData<GFloat16>(GFloat16 value, GFloat16 noDataValue)
5607 : {
5608 : using std::isnan;
5609 0 : return isnan(noDataValue) ? isnan(value) : value == noDataValue;
5610 : }
5611 :
5612 560462 : template <> bool IsEqualToNoData<float>(float value, float noDataValue)
5613 : {
5614 560462 : return std::isnan(noDataValue) ? std::isnan(value) : value == noDataValue;
5615 : }
5616 :
5617 13481900 : template <> bool IsEqualToNoData<double>(double value, double noDataValue)
5618 : {
5619 13481900 : return std::isnan(noDataValue) ? std::isnan(value) : value == noDataValue;
5620 : }
5621 :
5622 : template <class T>
5623 15885 : static bool HasOnlyNoDataT(const T *pBuffer, T noDataValue, size_t nWidth,
5624 : size_t nHeight, size_t nLineStride,
5625 : size_t nComponents)
5626 : {
5627 : // Fast test: check the 4 corners and the middle pixel.
5628 30871 : for (size_t iBand = 0; iBand < nComponents; iBand++)
5629 : {
5630 32519 : if (!(IsEqualToNoData(pBuffer[iBand], noDataValue) &&
5631 16172 : IsEqualToNoData(pBuffer[(nWidth - 1) * nComponents + iBand],
5632 15948 : noDataValue) &&
5633 15948 : IsEqualToNoData(
5634 15948 : pBuffer[((nHeight - 1) / 2 * nLineStride + (nWidth - 1) / 2) *
5635 15948 : nComponents +
5636 : iBand],
5637 14999 : noDataValue) &&
5638 14999 : IsEqualToNoData(
5639 14999 : pBuffer[(nHeight - 1) * nLineStride * nComponents + iBand],
5640 : noDataValue) &&
5641 14991 : IsEqualToNoData(
5642 14991 : pBuffer[((nHeight - 1) * nLineStride + nWidth - 1) *
5643 14991 : nComponents +
5644 : iBand],
5645 : noDataValue)))
5646 : {
5647 1361 : return false;
5648 : }
5649 : }
5650 :
5651 : // Test all pixels.
5652 45036 : for (size_t iY = 0; iY < nHeight; iY++)
5653 : {
5654 30566 : const T *pBufferLine = pBuffer + iY * nLineStride * nComponents;
5655 38852446 : for (size_t iX = 0; iX < nWidth * nComponents; iX++)
5656 : {
5657 38821963 : if (!IsEqualToNoData(pBufferLine[iX], noDataValue))
5658 : {
5659 54 : return false;
5660 : }
5661 : }
5662 : }
5663 14470 : return true;
5664 : }
5665 :
5666 : /************************************************************************/
5667 : /* GDALBufferHasOnlyNoData() */
5668 : /************************************************************************/
5669 :
5670 42607 : bool GDALBufferHasOnlyNoData(const void *pBuffer, double dfNoDataValue,
5671 : size_t nWidth, size_t nHeight, size_t nLineStride,
5672 : size_t nComponents, int nBitsPerSample,
5673 : GDALBufferSampleFormat nSampleFormat)
5674 : {
5675 : // In the case where the nodata is 0, we can compare several bytes at
5676 : // once. Select the largest natural integer type for the architecture.
5677 : #if SIZEOF_VOIDP >= 8 || defined(__x86_64__)
5678 : // We test __x86_64__ for x32 arch where SIZEOF_VOIDP == 4
5679 : typedef std::uint64_t WordType;
5680 : #else
5681 : typedef std::uint32_t WordType;
5682 : #endif
5683 42607 : if (dfNoDataValue == 0.0 && nWidth == nLineStride &&
5684 : // Do not use this optimized code path for floating point numbers,
5685 : // as it can't detect negative zero.
5686 : nSampleFormat != GSF_FLOATING_POINT)
5687 : {
5688 26716 : const GByte *pabyBuffer = static_cast<const GByte *>(pBuffer);
5689 26716 : const size_t nSize =
5690 26716 : (nWidth * nHeight * nComponents * nBitsPerSample + 7) / 8;
5691 26716 : size_t i = 0;
5692 : const size_t nInitialIters =
5693 53432 : std::min(sizeof(WordType) -
5694 26716 : static_cast<size_t>(
5695 : reinterpret_cast<std::uintptr_t>(pabyBuffer) %
5696 : sizeof(WordType)),
5697 26716 : nSize);
5698 220408 : for (; i < nInitialIters; i++)
5699 : {
5700 198066 : if (pabyBuffer[i])
5701 4374 : return false;
5702 : }
5703 16522900 : for (; i + sizeof(WordType) - 1 < nSize; i += sizeof(WordType))
5704 : {
5705 16507800 : if (*(reinterpret_cast<const WordType *>(pabyBuffer + i)))
5706 7198 : return false;
5707 : }
5708 52533 : for (; i < nSize; i++)
5709 : {
5710 37394 : if (pabyBuffer[i])
5711 5 : return false;
5712 : }
5713 15139 : return true;
5714 : }
5715 :
5716 15891 : if (nBitsPerSample == 8 && nSampleFormat == GSF_UNSIGNED_INT)
5717 : {
5718 22272 : return GDALIsValueInRange<uint8_t>(dfNoDataValue) &&
5719 11136 : HasOnlyNoDataT(static_cast<const uint8_t *>(pBuffer),
5720 11136 : static_cast<uint8_t>(dfNoDataValue), nWidth,
5721 11136 : nHeight, nLineStride, nComponents);
5722 : }
5723 4755 : if (nBitsPerSample == 8 && nSampleFormat == GSF_SIGNED_INT)
5724 : {
5725 : // Use unsigned implementation by converting the nodatavalue to
5726 : // unsigned
5727 63 : return GDALIsValueInRange<int8_t>(dfNoDataValue) &&
5728 31 : HasOnlyNoDataT(
5729 : static_cast<const uint8_t *>(pBuffer),
5730 31 : static_cast<uint8_t>(static_cast<int8_t>(dfNoDataValue)),
5731 32 : nWidth, nHeight, nLineStride, nComponents);
5732 : }
5733 4723 : if (nBitsPerSample == 16 && nSampleFormat == GSF_UNSIGNED_INT)
5734 : {
5735 21 : return GDALIsValueInRange<uint16_t>(dfNoDataValue) &&
5736 10 : HasOnlyNoDataT(static_cast<const uint16_t *>(pBuffer),
5737 10 : static_cast<uint16_t>(dfNoDataValue), nWidth,
5738 11 : nHeight, nLineStride, nComponents);
5739 : }
5740 4712 : if (nBitsPerSample == 16 && nSampleFormat == GSF_SIGNED_INT)
5741 : {
5742 : // Use unsigned implementation by converting the nodatavalue to
5743 : // unsigned
5744 99 : return GDALIsValueInRange<int16_t>(dfNoDataValue) &&
5745 49 : HasOnlyNoDataT(
5746 : static_cast<const uint16_t *>(pBuffer),
5747 49 : static_cast<uint16_t>(static_cast<int16_t>(dfNoDataValue)),
5748 50 : nWidth, nHeight, nLineStride, nComponents);
5749 : }
5750 4662 : if (nBitsPerSample == 32 && nSampleFormat == GSF_UNSIGNED_INT)
5751 : {
5752 73 : return GDALIsValueInRange<uint32_t>(dfNoDataValue) &&
5753 36 : HasOnlyNoDataT(static_cast<const uint32_t *>(pBuffer),
5754 : static_cast<uint32_t>(dfNoDataValue), nWidth,
5755 37 : nHeight, nLineStride, nComponents);
5756 : }
5757 4625 : if (nBitsPerSample == 32 && nSampleFormat == GSF_SIGNED_INT)
5758 : {
5759 : // Use unsigned implementation by converting the nodatavalue to
5760 : // unsigned
5761 23 : return GDALIsValueInRange<int32_t>(dfNoDataValue) &&
5762 11 : HasOnlyNoDataT(
5763 : static_cast<const uint32_t *>(pBuffer),
5764 11 : static_cast<uint32_t>(static_cast<int32_t>(dfNoDataValue)),
5765 12 : nWidth, nHeight, nLineStride, nComponents);
5766 : }
5767 4613 : if (nBitsPerSample == 64 && nSampleFormat == GSF_UNSIGNED_INT)
5768 : {
5769 56 : return GDALIsValueInRange<uint64_t>(dfNoDataValue) &&
5770 28 : HasOnlyNoDataT(static_cast<const uint64_t *>(pBuffer),
5771 : static_cast<uint64_t>(dfNoDataValue), nWidth,
5772 28 : nHeight, nLineStride, nComponents);
5773 : }
5774 4585 : if (nBitsPerSample == 64 && nSampleFormat == GSF_SIGNED_INT)
5775 : {
5776 : // Use unsigned implementation by converting the nodatavalue to
5777 : // unsigned
5778 0 : return GDALIsValueInRange<int64_t>(dfNoDataValue) &&
5779 0 : HasOnlyNoDataT(
5780 : static_cast<const uint64_t *>(pBuffer),
5781 0 : static_cast<uint64_t>(static_cast<int64_t>(dfNoDataValue)),
5782 0 : nWidth, nHeight, nLineStride, nComponents);
5783 : }
5784 4585 : if (nBitsPerSample == 16 && nSampleFormat == GSF_FLOATING_POINT)
5785 : {
5786 0 : return (std::isnan(dfNoDataValue) ||
5787 0 : GDALIsValueInRange<GFloat16>(dfNoDataValue)) &&
5788 0 : HasOnlyNoDataT(static_cast<const GFloat16 *>(pBuffer),
5789 : static_cast<GFloat16>(dfNoDataValue), nWidth,
5790 0 : nHeight, nLineStride, nComponents);
5791 : }
5792 4585 : if (nBitsPerSample == 32 && nSampleFormat == GSF_FLOATING_POINT)
5793 : {
5794 754 : return (std::isnan(dfNoDataValue) ||
5795 1507 : GDALIsValueInRange<float>(dfNoDataValue)) &&
5796 753 : HasOnlyNoDataT(static_cast<const float *>(pBuffer),
5797 : static_cast<float>(dfNoDataValue), nWidth,
5798 754 : nHeight, nLineStride, nComponents);
5799 : }
5800 3831 : if (nBitsPerSample == 64 && nSampleFormat == GSF_FLOATING_POINT)
5801 : {
5802 3831 : return HasOnlyNoDataT(static_cast<const double *>(pBuffer),
5803 : dfNoDataValue, nWidth, nHeight, nLineStride,
5804 3831 : nComponents);
5805 : }
5806 0 : return false;
5807 : }
5808 :
5809 : #ifdef HAVE_SSE2
5810 :
5811 : /************************************************************************/
5812 : /* GDALDeinterleave3Byte() */
5813 : /************************************************************************/
5814 :
5815 : #if defined(__GNUC__) && !defined(__clang__)
5816 : __attribute__((optimize("no-tree-vectorize")))
5817 : #endif
5818 : static void
5819 202472 : GDALDeinterleave3Byte(const GByte *CPL_RESTRICT pabySrc,
5820 : GByte *CPL_RESTRICT pabyDest0,
5821 : GByte *CPL_RESTRICT pabyDest1,
5822 : GByte *CPL_RESTRICT pabyDest2, size_t nIters)
5823 : #ifdef USE_NEON_OPTIMIZATIONS
5824 : {
5825 : return GDALDeinterleave3Byte_SSSE3(pabySrc, pabyDest0, pabyDest1, pabyDest2,
5826 : nIters);
5827 : }
5828 : #else
5829 : {
5830 : #ifdef HAVE_SSSE3_AT_COMPILE_TIME
5831 202472 : if (CPLHaveRuntimeSSSE3())
5832 : {
5833 202488 : return GDALDeinterleave3Byte_SSSE3(pabySrc, pabyDest0, pabyDest1,
5834 202485 : pabyDest2, nIters);
5835 : }
5836 : #endif
5837 :
5838 1 : size_t i = 0;
5839 1 : if (((reinterpret_cast<uintptr_t>(pabySrc) |
5840 1 : reinterpret_cast<uintptr_t>(pabyDest0) |
5841 1 : reinterpret_cast<uintptr_t>(pabyDest1) |
5842 1 : reinterpret_cast<uintptr_t>(pabyDest2)) %
5843 : sizeof(unsigned int)) == 0)
5844 : {
5845 : // Slightly better than GCC autovectorizer
5846 17 : for (size_t j = 0; i + 3 < nIters; i += 4, ++j)
5847 : {
5848 15 : unsigned int word0 =
5849 15 : *reinterpret_cast<const unsigned int *>(pabySrc + 3 * i);
5850 15 : unsigned int word1 =
5851 15 : *reinterpret_cast<const unsigned int *>(pabySrc + 3 * i + 4);
5852 15 : unsigned int word2 =
5853 15 : *reinterpret_cast<const unsigned int *>(pabySrc + 3 * i + 8);
5854 15 : reinterpret_cast<unsigned int *>(pabyDest0)[j] =
5855 15 : (word0 & 0xff) | ((word0 >> 24) << 8) | (word1 & 0x00ff0000) |
5856 15 : ((word2 >> 8) << 24);
5857 15 : reinterpret_cast<unsigned int *>(pabyDest1)[j] =
5858 15 : ((word0 >> 8) & 0xff) | ((word1 & 0xff) << 8) |
5859 15 : (((word1 >> 24)) << 16) | ((word2 >> 16) << 24);
5860 15 : pabyDest2[j * 4] = static_cast<GByte>(word0 >> 16);
5861 15 : pabyDest2[j * 4 + 1] = static_cast<GByte>(word1 >> 8);
5862 15 : pabyDest2[j * 4 + 2] = static_cast<GByte>(word2);
5863 15 : pabyDest2[j * 4 + 3] = static_cast<GByte>(word2 >> 24);
5864 : }
5865 : }
5866 : #if defined(__clang__)
5867 : #pragma clang loop vectorize(disable)
5868 : #endif
5869 2 : for (; i < nIters; ++i)
5870 : {
5871 1 : pabyDest0[i] = pabySrc[3 * i + 0];
5872 1 : pabyDest1[i] = pabySrc[3 * i + 1];
5873 1 : pabyDest2[i] = pabySrc[3 * i + 2];
5874 : }
5875 : }
5876 : #endif
5877 :
5878 : /************************************************************************/
5879 : /* GDALDeinterleave4Byte() */
5880 : /************************************************************************/
5881 :
5882 : #if !defined(__GNUC__) || defined(__clang__)
5883 :
5884 : /************************************************************************/
5885 : /* deinterleave() */
5886 : /************************************************************************/
5887 :
5888 : template <bool SHIFT, bool MASK>
5889 : inline __m128i deinterleave(__m128i &xmm0_ori, __m128i &xmm1_ori,
5890 : __m128i &xmm2_ori, __m128i &xmm3_ori)
5891 : {
5892 : // Set higher 24bit of each int32 packed word to 0
5893 : if (SHIFT)
5894 : {
5895 : xmm0_ori = _mm_srli_epi32(xmm0_ori, 8);
5896 : xmm1_ori = _mm_srli_epi32(xmm1_ori, 8);
5897 : xmm2_ori = _mm_srli_epi32(xmm2_ori, 8);
5898 : xmm3_ori = _mm_srli_epi32(xmm3_ori, 8);
5899 : }
5900 : __m128i xmm0;
5901 : __m128i xmm1;
5902 : __m128i xmm2;
5903 : __m128i xmm3;
5904 : if (MASK)
5905 : {
5906 : const __m128i xmm_mask = _mm_set1_epi32(0xff);
5907 : xmm0 = _mm_and_si128(xmm0_ori, xmm_mask);
5908 : xmm1 = _mm_and_si128(xmm1_ori, xmm_mask);
5909 : xmm2 = _mm_and_si128(xmm2_ori, xmm_mask);
5910 : xmm3 = _mm_and_si128(xmm3_ori, xmm_mask);
5911 : }
5912 : else
5913 : {
5914 : xmm0 = xmm0_ori;
5915 : xmm1 = xmm1_ori;
5916 : xmm2 = xmm2_ori;
5917 : xmm3 = xmm3_ori;
5918 : }
5919 : // Pack int32 to int16
5920 : xmm0 = _mm_packs_epi32(xmm0, xmm1);
5921 : xmm2 = _mm_packs_epi32(xmm2, xmm3);
5922 : // Pack int16 to uint8
5923 : xmm0 = _mm_packus_epi16(xmm0, xmm2);
5924 : return xmm0;
5925 : }
5926 :
5927 : static void GDALDeinterleave4Byte(const GByte *CPL_RESTRICT pabySrc,
5928 : GByte *CPL_RESTRICT pabyDest0,
5929 : GByte *CPL_RESTRICT pabyDest1,
5930 : GByte *CPL_RESTRICT pabyDest2,
5931 : GByte *CPL_RESTRICT pabyDest3, size_t nIters)
5932 : #ifdef USE_NEON_OPTIMIZATIONS
5933 : {
5934 : return GDALDeinterleave4Byte_SSSE3(pabySrc, pabyDest0, pabyDest1, pabyDest2,
5935 : pabyDest3, nIters);
5936 : }
5937 : #else
5938 : {
5939 : #ifdef HAVE_SSSE3_AT_COMPILE_TIME
5940 : if (CPLHaveRuntimeSSSE3())
5941 : {
5942 : return GDALDeinterleave4Byte_SSSE3(pabySrc, pabyDest0, pabyDest1,
5943 : pabyDest2, pabyDest3, nIters);
5944 : }
5945 : #endif
5946 :
5947 : // Not the optimal SSE2-only code, as gcc auto-vectorizer manages to
5948 : // do something slightly better.
5949 : size_t i = 0;
5950 : for (; i + 15 < nIters; i += 16)
5951 : {
5952 : __m128i xmm0_ori = _mm_loadu_si128(
5953 : reinterpret_cast<__m128i const *>(pabySrc + 4 * i + 0));
5954 : __m128i xmm1_ori = _mm_loadu_si128(
5955 : reinterpret_cast<__m128i const *>(pabySrc + 4 * i + 16));
5956 : __m128i xmm2_ori = _mm_loadu_si128(
5957 : reinterpret_cast<__m128i const *>(pabySrc + 4 * i + 32));
5958 : __m128i xmm3_ori = _mm_loadu_si128(
5959 : reinterpret_cast<__m128i const *>(pabySrc + 4 * i + 48));
5960 :
5961 : _mm_storeu_si128(
5962 : reinterpret_cast<__m128i *>(pabyDest0 + i),
5963 : deinterleave<false, true>(xmm0_ori, xmm1_ori, xmm2_ori, xmm3_ori));
5964 : _mm_storeu_si128(
5965 : reinterpret_cast<__m128i *>(pabyDest1 + i),
5966 : deinterleave<true, true>(xmm0_ori, xmm1_ori, xmm2_ori, xmm3_ori));
5967 : _mm_storeu_si128(
5968 : reinterpret_cast<__m128i *>(pabyDest2 + i),
5969 : deinterleave<true, true>(xmm0_ori, xmm1_ori, xmm2_ori, xmm3_ori));
5970 : _mm_storeu_si128(
5971 : reinterpret_cast<__m128i *>(pabyDest3 + i),
5972 : deinterleave<true, false>(xmm0_ori, xmm1_ori, xmm2_ori, xmm3_ori));
5973 : }
5974 :
5975 : #if defined(__clang__)
5976 : #pragma clang loop vectorize(disable)
5977 : #endif
5978 : for (; i < nIters; ++i)
5979 : {
5980 : pabyDest0[i] = pabySrc[4 * i + 0];
5981 : pabyDest1[i] = pabySrc[4 * i + 1];
5982 : pabyDest2[i] = pabySrc[4 * i + 2];
5983 : pabyDest3[i] = pabySrc[4 * i + 3];
5984 : }
5985 : }
5986 : #endif
5987 : #else
5988 : // GCC autovectorizer does an excellent job
5989 61553 : __attribute__((optimize("tree-vectorize"))) static void GDALDeinterleave4Byte(
5990 : const GByte *CPL_RESTRICT pabySrc, GByte *CPL_RESTRICT pabyDest0,
5991 : GByte *CPL_RESTRICT pabyDest1, GByte *CPL_RESTRICT pabyDest2,
5992 : GByte *CPL_RESTRICT pabyDest3, size_t nIters)
5993 : {
5994 526862000 : for (size_t i = 0; i < nIters; ++i)
5995 : {
5996 526801000 : pabyDest0[i] = pabySrc[4 * i + 0];
5997 526801000 : pabyDest1[i] = pabySrc[4 * i + 1];
5998 526801000 : pabyDest2[i] = pabySrc[4 * i + 2];
5999 526801000 : pabyDest3[i] = pabySrc[4 * i + 3];
6000 : }
6001 61553 : }
6002 : #endif
6003 :
6004 : #else
6005 :
6006 : /************************************************************************/
6007 : /* GDALDeinterleave3Byte() */
6008 : /************************************************************************/
6009 :
6010 : // TODO: Enabling below could help on non-Intel architectures where GCC knows
6011 : // how to auto-vectorize
6012 : // #if defined(__GNUC__)
6013 : //__attribute__((optimize("tree-vectorize")))
6014 : // #endif
6015 : static void GDALDeinterleave3Byte(const GByte *CPL_RESTRICT pabySrc,
6016 : GByte *CPL_RESTRICT pabyDest0,
6017 : GByte *CPL_RESTRICT pabyDest1,
6018 : GByte *CPL_RESTRICT pabyDest2, size_t nIters)
6019 : {
6020 : for (size_t i = 0; i < nIters; ++i)
6021 : {
6022 : pabyDest0[i] = pabySrc[3 * i + 0];
6023 : pabyDest1[i] = pabySrc[3 * i + 1];
6024 : pabyDest2[i] = pabySrc[3 * i + 2];
6025 : }
6026 : }
6027 :
6028 : /************************************************************************/
6029 : /* GDALDeinterleave4Byte() */
6030 : /************************************************************************/
6031 :
6032 : // TODO: Enabling below could help on non-Intel architectures where gcc knows
6033 : // how to auto-vectorize
6034 : // #if defined(__GNUC__)
6035 : //__attribute__((optimize("tree-vectorize")))
6036 : // #endif
6037 : static void GDALDeinterleave4Byte(const GByte *CPL_RESTRICT pabySrc,
6038 : GByte *CPL_RESTRICT pabyDest0,
6039 : GByte *CPL_RESTRICT pabyDest1,
6040 : GByte *CPL_RESTRICT pabyDest2,
6041 : GByte *CPL_RESTRICT pabyDest3, size_t nIters)
6042 : {
6043 : for (size_t i = 0; i < nIters; ++i)
6044 : {
6045 : pabyDest0[i] = pabySrc[4 * i + 0];
6046 : pabyDest1[i] = pabySrc[4 * i + 1];
6047 : pabyDest2[i] = pabySrc[4 * i + 2];
6048 : pabyDest3[i] = pabySrc[4 * i + 3];
6049 : }
6050 : }
6051 :
6052 : #endif
6053 :
6054 : /************************************************************************/
6055 : /* GDALDeinterleave() */
6056 : /************************************************************************/
6057 :
6058 : /*! Copy values from a pixel-interleave buffer to multiple per-component
6059 : buffers.
6060 :
6061 : In pseudo-code
6062 : \verbatim
6063 : for(size_t i = 0; i < nIters; ++i)
6064 : for(int iComp = 0; iComp < nComponents; iComp++ )
6065 : ppDestBuffer[iComp][i] = pSourceBuffer[nComponents * i + iComp]
6066 : \endverbatim
6067 :
6068 : The implementation is optimized for a few cases, like de-interleaving
6069 : of 3 or 4-components Byte buffers.
6070 :
6071 : \since GDAL 3.6
6072 : */
6073 264370 : void GDALDeinterleave(const void *pSourceBuffer, GDALDataType eSourceDT,
6074 : int nComponents, void **ppDestBuffer,
6075 : GDALDataType eDestDT, size_t nIters)
6076 : {
6077 264370 : if (eSourceDT == eDestDT)
6078 : {
6079 264352 : if (eSourceDT == GDT_Byte || eSourceDT == GDT_Int8)
6080 : {
6081 264031 : if (nComponents == 3)
6082 : {
6083 202476 : const GByte *CPL_RESTRICT pabySrc =
6084 : static_cast<const GByte *>(pSourceBuffer);
6085 202476 : GByte *CPL_RESTRICT pabyDest0 =
6086 : static_cast<GByte *>(ppDestBuffer[0]);
6087 202476 : GByte *CPL_RESTRICT pabyDest1 =
6088 : static_cast<GByte *>(ppDestBuffer[1]);
6089 202476 : GByte *CPL_RESTRICT pabyDest2 =
6090 : static_cast<GByte *>(ppDestBuffer[2]);
6091 202476 : GDALDeinterleave3Byte(pabySrc, pabyDest0, pabyDest1, pabyDest2,
6092 : nIters);
6093 202488 : return;
6094 : }
6095 61555 : else if (nComponents == 4)
6096 : {
6097 61553 : const GByte *CPL_RESTRICT pabySrc =
6098 : static_cast<const GByte *>(pSourceBuffer);
6099 61553 : GByte *CPL_RESTRICT pabyDest0 =
6100 : static_cast<GByte *>(ppDestBuffer[0]);
6101 61553 : GByte *CPL_RESTRICT pabyDest1 =
6102 : static_cast<GByte *>(ppDestBuffer[1]);
6103 61553 : GByte *CPL_RESTRICT pabyDest2 =
6104 : static_cast<GByte *>(ppDestBuffer[2]);
6105 61553 : GByte *CPL_RESTRICT pabyDest3 =
6106 : static_cast<GByte *>(ppDestBuffer[3]);
6107 61553 : GDALDeinterleave4Byte(pabySrc, pabyDest0, pabyDest1, pabyDest2,
6108 : pabyDest3, nIters);
6109 61553 : return;
6110 2 : }
6111 : }
6112 : #if ((defined(__GNUC__) && !defined(__clang__)) || \
6113 : defined(__INTEL_CLANG_COMPILER)) && \
6114 : defined(HAVE_SSE2) && defined(HAVE_SSSE3_AT_COMPILE_TIME)
6115 642 : else if ((eSourceDT == GDT_Int16 || eSourceDT == GDT_UInt16) &&
6116 321 : CPLHaveRuntimeSSSE3())
6117 : {
6118 321 : if (nComponents == 3)
6119 : {
6120 126 : const GUInt16 *CPL_RESTRICT panSrc =
6121 : static_cast<const GUInt16 *>(pSourceBuffer);
6122 126 : GUInt16 *CPL_RESTRICT panDest0 =
6123 : static_cast<GUInt16 *>(ppDestBuffer[0]);
6124 126 : GUInt16 *CPL_RESTRICT panDest1 =
6125 : static_cast<GUInt16 *>(ppDestBuffer[1]);
6126 126 : GUInt16 *CPL_RESTRICT panDest2 =
6127 : static_cast<GUInt16 *>(ppDestBuffer[2]);
6128 126 : GDALDeinterleave3UInt16_SSSE3(panSrc, panDest0, panDest1,
6129 : panDest2, nIters);
6130 126 : return;
6131 : }
6132 : #if !defined(__INTEL_CLANG_COMPILER)
6133 : // ICC autovectorizer doesn't do a good job, at least with icx
6134 : // 2022.1.0.20220316
6135 195 : else if (nComponents == 4)
6136 : {
6137 195 : const GUInt16 *CPL_RESTRICT panSrc =
6138 : static_cast<const GUInt16 *>(pSourceBuffer);
6139 195 : GUInt16 *CPL_RESTRICT panDest0 =
6140 : static_cast<GUInt16 *>(ppDestBuffer[0]);
6141 195 : GUInt16 *CPL_RESTRICT panDest1 =
6142 : static_cast<GUInt16 *>(ppDestBuffer[1]);
6143 195 : GUInt16 *CPL_RESTRICT panDest2 =
6144 : static_cast<GUInt16 *>(ppDestBuffer[2]);
6145 195 : GUInt16 *CPL_RESTRICT panDest3 =
6146 : static_cast<GUInt16 *>(ppDestBuffer[3]);
6147 195 : GDALDeinterleave4UInt16_SSSE3(panSrc, panDest0, panDest1,
6148 : panDest2, panDest3, nIters);
6149 195 : return;
6150 : }
6151 : #endif
6152 : }
6153 : #endif
6154 : }
6155 :
6156 20 : const int nSourceDTSize = GDALGetDataTypeSizeBytes(eSourceDT);
6157 29 : const int nDestDTSize = GDALGetDataTypeSizeBytes(eDestDT);
6158 108 : for (int iComp = 0; iComp < nComponents; iComp++)
6159 : {
6160 79 : GDALCopyWords64(static_cast<const GByte *>(pSourceBuffer) +
6161 79 : iComp * nSourceDTSize,
6162 : eSourceDT, nComponents * nSourceDTSize,
6163 79 : ppDestBuffer[iComp], eDestDT, nDestDTSize, nIters);
6164 : }
6165 : }
6166 :
6167 : /************************************************************************/
6168 : /* GDALTranspose2DSingleToSingle() */
6169 : /************************************************************************/
6170 : /**
6171 : * Transpose a 2D array of non-complex values, in a efficient (cache-oblivious) way.
6172 : *
6173 : * @param pSrc Source array of height = nSrcHeight and width = nSrcWidth.
6174 : * @param pDst Destination transposed array of height = nSrcWidth and width = nSrcHeight.
6175 : * @param nSrcWidth Width of pSrc array.
6176 : * @param nSrcHeight Height of pSrc array.
6177 : */
6178 :
6179 : template <class DST, class SRC>
6180 145 : void GDALTranspose2DSingleToSingle(const SRC *CPL_RESTRICT pSrc,
6181 : DST *CPL_RESTRICT pDst, size_t nSrcWidth,
6182 : size_t nSrcHeight)
6183 : {
6184 145 : constexpr size_t blocksize = 32;
6185 315 : for (size_t i = 0; i < nSrcHeight; i += blocksize)
6186 : {
6187 170 : const size_t max_k = std::min(i + blocksize, nSrcHeight);
6188 390 : for (size_t j = 0; j < nSrcWidth; j += blocksize)
6189 : {
6190 : // transpose the block beginning at [i,j]
6191 220 : const size_t max_l = std::min(j + blocksize, nSrcWidth);
6192 2509 : for (size_t k = i; k < max_k; ++k)
6193 : {
6194 41017 : for (size_t l = j; l < max_l; ++l)
6195 : {
6196 38728 : GDALCopyWord(pSrc[l + k * nSrcWidth],
6197 38728 : pDst[k + l * nSrcHeight]);
6198 : }
6199 : }
6200 : }
6201 : }
6202 145 : }
6203 :
6204 : /************************************************************************/
6205 : /* GDALTranspose2DComplexToComplex() */
6206 : /************************************************************************/
6207 : /**
6208 : * Transpose a 2D array of complex values into an array of complex values,
6209 : * in a efficient (cache-oblivious) way.
6210 : *
6211 : * @param pSrc Source array of height = nSrcHeight and width = nSrcWidth.
6212 : * @param pDst Destination transposed array of height = nSrcWidth and width = nSrcHeight.
6213 : * @param nSrcWidth Width of pSrc array.
6214 : * @param nSrcHeight Height of pSrc array.
6215 : */
6216 : template <class DST, class SRC>
6217 25 : void GDALTranspose2DComplexToComplex(const SRC *CPL_RESTRICT pSrc,
6218 : DST *CPL_RESTRICT pDst, size_t nSrcWidth,
6219 : size_t nSrcHeight)
6220 : {
6221 25 : constexpr size_t blocksize = 32;
6222 50 : for (size_t i = 0; i < nSrcHeight; i += blocksize)
6223 : {
6224 25 : const size_t max_k = std::min(i + blocksize, nSrcHeight);
6225 50 : for (size_t j = 0; j < nSrcWidth; j += blocksize)
6226 : {
6227 : // transpose the block beginning at [i,j]
6228 25 : const size_t max_l = std::min(j + blocksize, nSrcWidth);
6229 75 : for (size_t k = i; k < max_k; ++k)
6230 : {
6231 200 : for (size_t l = j; l < max_l; ++l)
6232 : {
6233 150 : GDALCopyWord(pSrc[2 * (l + k * nSrcWidth) + 0],
6234 150 : pDst[2 * (k + l * nSrcHeight) + 0]);
6235 150 : GDALCopyWord(pSrc[2 * (l + k * nSrcWidth) + 1],
6236 150 : pDst[2 * (k + l * nSrcHeight) + 1]);
6237 : }
6238 : }
6239 : }
6240 : }
6241 25 : }
6242 :
6243 : /************************************************************************/
6244 : /* GDALTranspose2DComplexToSingle() */
6245 : /************************************************************************/
6246 : /**
6247 : * Transpose a 2D array of complex values into an array of non-complex values,
6248 : * in a efficient (cache-oblivious) way.
6249 : *
6250 : * @param pSrc Source array of height = nSrcHeight and width = nSrcWidth.
6251 : * @param pDst Destination transposed array of height = nSrcWidth and width = nSrcHeight.
6252 : * @param nSrcWidth Width of pSrc array.
6253 : * @param nSrcHeight Height of pSrc array.
6254 : */
6255 : template <class DST, class SRC>
6256 55 : void GDALTranspose2DComplexToSingle(const SRC *CPL_RESTRICT pSrc,
6257 : DST *CPL_RESTRICT pDst, size_t nSrcWidth,
6258 : size_t nSrcHeight)
6259 : {
6260 55 : constexpr size_t blocksize = 32;
6261 110 : for (size_t i = 0; i < nSrcHeight; i += blocksize)
6262 : {
6263 55 : const size_t max_k = std::min(i + blocksize, nSrcHeight);
6264 110 : for (size_t j = 0; j < nSrcWidth; j += blocksize)
6265 : {
6266 : // transpose the block beginning at [i,j]
6267 55 : const size_t max_l = std::min(j + blocksize, nSrcWidth);
6268 165 : for (size_t k = i; k < max_k; ++k)
6269 : {
6270 440 : for (size_t l = j; l < max_l; ++l)
6271 : {
6272 330 : GDALCopyWord(pSrc[2 * (l + k * nSrcWidth) + 0],
6273 330 : pDst[k + l * nSrcHeight]);
6274 : }
6275 : }
6276 : }
6277 : }
6278 55 : }
6279 :
6280 : /************************************************************************/
6281 : /* GDALTranspose2DSingleToComplex() */
6282 : /************************************************************************/
6283 : /**
6284 : * Transpose a 2D array of non-complex values into an array of complex values,
6285 : * in a efficient (cache-oblivious) way.
6286 : *
6287 : * @param pSrc Source array of height = nSrcHeight and width = nSrcWidth.
6288 : * @param pDst Destination transposed array of height = nSrcWidth and width = nSrcHeight.
6289 : * @param nSrcWidth Width of pSrc array.
6290 : * @param nSrcHeight Height of pSrc array.
6291 : */
6292 : template <class DST, class SRC>
6293 55 : void GDALTranspose2DSingleToComplex(const SRC *CPL_RESTRICT pSrc,
6294 : DST *CPL_RESTRICT pDst, size_t nSrcWidth,
6295 : size_t nSrcHeight)
6296 : {
6297 55 : constexpr size_t blocksize = 32;
6298 110 : for (size_t i = 0; i < nSrcHeight; i += blocksize)
6299 : {
6300 55 : const size_t max_k = std::min(i + blocksize, nSrcHeight);
6301 110 : for (size_t j = 0; j < nSrcWidth; j += blocksize)
6302 : {
6303 : // transpose the block beginning at [i,j]
6304 55 : const size_t max_l = std::min(j + blocksize, nSrcWidth);
6305 165 : for (size_t k = i; k < max_k; ++k)
6306 : {
6307 440 : for (size_t l = j; l < max_l; ++l)
6308 : {
6309 330 : GDALCopyWord(pSrc[l + k * nSrcWidth],
6310 330 : pDst[2 * (k + l * nSrcHeight) + 0]);
6311 330 : pDst[2 * (k + l * nSrcHeight) + 1] = 0;
6312 : }
6313 : }
6314 : }
6315 : }
6316 55 : }
6317 :
6318 : /************************************************************************/
6319 : /* GDALTranspose2D() */
6320 : /************************************************************************/
6321 :
6322 : template <class DST, bool DST_IS_COMPLEX>
6323 280 : static void GDALTranspose2D(const void *pSrc, GDALDataType eSrcType, DST *pDst,
6324 : size_t nSrcWidth, size_t nSrcHeight)
6325 : {
6326 : #define CALL_GDALTranspose2D_internal(SRC_TYPE) \
6327 : do \
6328 : { \
6329 : if constexpr (DST_IS_COMPLEX) \
6330 : { \
6331 : GDALTranspose2DSingleToComplex( \
6332 : static_cast<const SRC_TYPE *>(pSrc), pDst, nSrcWidth, \
6333 : nSrcHeight); \
6334 : } \
6335 : else \
6336 : { \
6337 : GDALTranspose2DSingleToSingle(static_cast<const SRC_TYPE *>(pSrc), \
6338 : pDst, nSrcWidth, nSrcHeight); \
6339 : } \
6340 : } while (0)
6341 :
6342 : #define CALL_GDALTranspose2DComplex_internal(SRC_TYPE) \
6343 : do \
6344 : { \
6345 : if constexpr (DST_IS_COMPLEX) \
6346 : { \
6347 : GDALTranspose2DComplexToComplex( \
6348 : static_cast<const SRC_TYPE *>(pSrc), pDst, nSrcWidth, \
6349 : nSrcHeight); \
6350 : } \
6351 : else \
6352 : { \
6353 : GDALTranspose2DComplexToSingle( \
6354 : static_cast<const SRC_TYPE *>(pSrc), pDst, nSrcWidth, \
6355 : nSrcHeight); \
6356 : } \
6357 : } while (0)
6358 :
6359 : // clang-format off
6360 280 : switch (eSrcType)
6361 : {
6362 16 : case GDT_Byte: CALL_GDALTranspose2D_internal(uint8_t); break;
6363 15 : case GDT_Int8: CALL_GDALTranspose2D_internal(int8_t); break;
6364 24 : case GDT_UInt16: CALL_GDALTranspose2D_internal(uint16_t); break;
6365 16 : case GDT_Int16: CALL_GDALTranspose2D_internal(int16_t); break;
6366 24 : case GDT_UInt32: CALL_GDALTranspose2D_internal(uint32_t); break;
6367 16 : case GDT_Int32: CALL_GDALTranspose2D_internal(int32_t); break;
6368 16 : case GDT_UInt64: CALL_GDALTranspose2D_internal(uint64_t); break;
6369 16 : case GDT_Int64: CALL_GDALTranspose2D_internal(int64_t); break;
6370 16 : case GDT_Float16: CALL_GDALTranspose2D_internal(GFloat16); break;
6371 17 : case GDT_Float32: CALL_GDALTranspose2D_internal(float); break;
6372 24 : case GDT_Float64: CALL_GDALTranspose2D_internal(double); break;
6373 16 : case GDT_CInt16: CALL_GDALTranspose2DComplex_internal(int16_t); break;
6374 16 : case GDT_CInt32: CALL_GDALTranspose2DComplex_internal(int32_t); break;
6375 16 : case GDT_CFloat16: CALL_GDALTranspose2DComplex_internal(GFloat16); break;
6376 16 : case GDT_CFloat32: CALL_GDALTranspose2DComplex_internal(float); break;
6377 16 : case GDT_CFloat64: CALL_GDALTranspose2DComplex_internal(double); break;
6378 0 : case GDT_Unknown:
6379 : case GDT_TypeCount:
6380 0 : break;
6381 : }
6382 : // clang-format on
6383 :
6384 : #undef CALL_GDALTranspose2D_internal
6385 : #undef CALL_GDALTranspose2DComplex_internal
6386 280 : }
6387 :
6388 : /************************************************************************/
6389 : /* GDALInterleave2Byte() */
6390 : /************************************************************************/
6391 :
6392 : #if defined(HAVE_SSE2) && \
6393 : (!defined(__GNUC__) || defined(__INTEL_CLANG_COMPILER))
6394 :
6395 : // ICC autovectorizer doesn't do a good job at generating good SSE code,
6396 : // at least with icx 2024.0.2.20231213, but it nicely unrolls the below loop.
6397 : #if defined(__GNUC__)
6398 : __attribute__((noinline))
6399 : #endif
6400 : static void
6401 : GDALInterleave2Byte(const uint8_t *CPL_RESTRICT pSrc,
6402 : uint8_t *CPL_RESTRICT pDst, size_t nIters)
6403 : {
6404 : size_t i = 0;
6405 : constexpr size_t VALS_PER_ITER = 16;
6406 : for (i = 0; i + VALS_PER_ITER <= nIters; i += VALS_PER_ITER)
6407 : {
6408 : __m128i xmm0 =
6409 : _mm_loadu_si128(reinterpret_cast<__m128i const *>(pSrc + i));
6410 : __m128i xmm1 = _mm_loadu_si128(
6411 : reinterpret_cast<__m128i const *>(pSrc + i + nIters));
6412 : _mm_storeu_si128(reinterpret_cast<__m128i *>(pDst + 2 * i),
6413 : _mm_unpacklo_epi8(xmm0, xmm1));
6414 : _mm_storeu_si128(
6415 : reinterpret_cast<__m128i *>(pDst + 2 * i + VALS_PER_ITER),
6416 : _mm_unpackhi_epi8(xmm0, xmm1));
6417 : }
6418 : #if defined(__clang__)
6419 : #pragma clang loop vectorize(disable)
6420 : #endif
6421 : for (; i < nIters; ++i)
6422 : {
6423 : pDst[2 * i + 0] = pSrc[i + 0 * nIters];
6424 : pDst[2 * i + 1] = pSrc[i + 1 * nIters];
6425 : }
6426 : }
6427 :
6428 : #else
6429 :
6430 : #if defined(__GNUC__) && !defined(__clang__)
6431 : __attribute__((optimize("tree-vectorize")))
6432 : #endif
6433 : #if defined(__GNUC__)
6434 : __attribute__((noinline))
6435 : #endif
6436 : #if defined(__clang__) && !defined(__INTEL_CLANG_COMPILER)
6437 : // clang++ -O2 -fsanitize=undefined fails to vectorize, ignore that warning
6438 : #pragma clang diagnostic push
6439 : #pragma clang diagnostic ignored "-Wpass-failed"
6440 : #endif
6441 : static void
6442 4 : GDALInterleave2Byte(const uint8_t *CPL_RESTRICT pSrc,
6443 : uint8_t *CPL_RESTRICT pDst, size_t nIters)
6444 : {
6445 : #if defined(__clang__) && !defined(__INTEL_CLANG_COMPILER)
6446 : #pragma clang loop vectorize(enable)
6447 : #endif
6448 44 : for (size_t i = 0; i < nIters; ++i)
6449 : {
6450 40 : pDst[2 * i + 0] = pSrc[i + 0 * nIters];
6451 40 : pDst[2 * i + 1] = pSrc[i + 1 * nIters];
6452 : }
6453 4 : }
6454 : #if defined(__clang__) && !defined(__INTEL_CLANG_COMPILER)
6455 : #pragma clang diagnostic pop
6456 : #endif
6457 :
6458 : #endif
6459 :
6460 : /************************************************************************/
6461 : /* GDALInterleave4Byte() */
6462 : /************************************************************************/
6463 :
6464 : #if defined(HAVE_SSE2) && \
6465 : (!defined(__GNUC__) || defined(__INTEL_CLANG_COMPILER))
6466 :
6467 : // ICC autovectorizer doesn't do a good job at generating good SSE code,
6468 : // at least with icx 2024.0.2.20231213, but it nicely unrolls the below loop.
6469 : #if defined(__GNUC__)
6470 : __attribute__((noinline))
6471 : #endif
6472 : static void
6473 : GDALInterleave4Byte(const uint8_t *CPL_RESTRICT pSrc,
6474 : uint8_t *CPL_RESTRICT pDst, size_t nIters)
6475 : {
6476 : size_t i = 0;
6477 : constexpr size_t VALS_PER_ITER = 16;
6478 : for (i = 0; i + VALS_PER_ITER <= nIters; i += VALS_PER_ITER)
6479 : {
6480 : __m128i xmm0 = _mm_loadu_si128(
6481 : reinterpret_cast<__m128i const *>(pSrc + i + 0 * nIters));
6482 : __m128i xmm1 = _mm_loadu_si128(
6483 : reinterpret_cast<__m128i const *>(pSrc + i + 1 * nIters));
6484 : __m128i xmm2 = _mm_loadu_si128(
6485 : reinterpret_cast<__m128i const *>(pSrc + i + 2 * nIters));
6486 : __m128i xmm3 = _mm_loadu_si128(
6487 : reinterpret_cast<__m128i const *>(pSrc + i + 3 * nIters));
6488 : auto tmp0 = _mm_unpacklo_epi8(
6489 : xmm0,
6490 : xmm1); // (xmm0_0, xmm1_0, xmm0_1, xmm1_1, xmm0_2, xmm1_2, ...)
6491 : auto tmp1 = _mm_unpackhi_epi8(
6492 : xmm0,
6493 : xmm1); // (xmm0_8, xmm1_8, xmm0_9, xmm1_9, xmm0_10, xmm1_10, ...)
6494 : auto tmp2 = _mm_unpacklo_epi8(
6495 : xmm2,
6496 : xmm3); // (xmm2_0, xmm3_0, xmm2_1, xmm3_1, xmm2_2, xmm3_2, ...)
6497 : auto tmp3 = _mm_unpackhi_epi8(
6498 : xmm2,
6499 : xmm3); // (xmm2_8, xmm3_8, xmm2_9, xmm3_9, xmm2_10, xmm3_10, ...)
6500 : auto tmp2_0 = _mm_unpacklo_epi16(
6501 : tmp0,
6502 : tmp2); // (xmm0_0, xmm1_0, xmm2_0, xmm3_0, xmm0_1, xmm1_1, xmm2_1, xmm3_1, ...)
6503 : auto tmp2_1 = _mm_unpackhi_epi16(tmp0, tmp2);
6504 : auto tmp2_2 = _mm_unpacklo_epi16(tmp1, tmp3);
6505 : auto tmp2_3 = _mm_unpackhi_epi16(tmp1, tmp3);
6506 : _mm_storeu_si128(
6507 : reinterpret_cast<__m128i *>(pDst + 4 * i + 0 * VALS_PER_ITER),
6508 : tmp2_0);
6509 : _mm_storeu_si128(
6510 : reinterpret_cast<__m128i *>(pDst + 4 * i + 1 * VALS_PER_ITER),
6511 : tmp2_1);
6512 : _mm_storeu_si128(
6513 : reinterpret_cast<__m128i *>(pDst + 4 * i + 2 * VALS_PER_ITER),
6514 : tmp2_2);
6515 : _mm_storeu_si128(
6516 : reinterpret_cast<__m128i *>(pDst + 4 * i + 3 * VALS_PER_ITER),
6517 : tmp2_3);
6518 : }
6519 : #if defined(__clang__)
6520 : #pragma clang loop vectorize(disable)
6521 : #endif
6522 : for (; i < nIters; ++i)
6523 : {
6524 : pDst[4 * i + 0] = pSrc[i + 0 * nIters];
6525 : pDst[4 * i + 1] = pSrc[i + 1 * nIters];
6526 : pDst[4 * i + 2] = pSrc[i + 2 * nIters];
6527 : pDst[4 * i + 3] = pSrc[i + 3 * nIters];
6528 : }
6529 : }
6530 :
6531 : #else
6532 :
6533 : #if defined(__GNUC__) && !defined(__clang__)
6534 : __attribute__((optimize("tree-vectorize")))
6535 : #endif
6536 : #if defined(__GNUC__)
6537 : __attribute__((noinline))
6538 : #endif
6539 : #if defined(__clang__) && !defined(__INTEL_CLANG_COMPILER)
6540 : // clang++ -O2 -fsanitize=undefined fails to vectorize, ignore that warning
6541 : #pragma clang diagnostic push
6542 : #pragma clang diagnostic ignored "-Wpass-failed"
6543 : #endif
6544 : static void
6545 2 : GDALInterleave4Byte(const uint8_t *CPL_RESTRICT pSrc,
6546 : uint8_t *CPL_RESTRICT pDst, size_t nIters)
6547 : {
6548 : #if defined(__clang__) && !defined(__INTEL_CLANG_COMPILER)
6549 : #pragma clang loop vectorize(enable)
6550 : #endif
6551 36 : for (size_t i = 0; i < nIters; ++i)
6552 : {
6553 34 : pDst[4 * i + 0] = pSrc[i + 0 * nIters];
6554 34 : pDst[4 * i + 1] = pSrc[i + 1 * nIters];
6555 34 : pDst[4 * i + 2] = pSrc[i + 2 * nIters];
6556 34 : pDst[4 * i + 3] = pSrc[i + 3 * nIters];
6557 : }
6558 2 : }
6559 : #if defined(__clang__) && !defined(__INTEL_CLANG_COMPILER)
6560 : #pragma clang diagnostic pop
6561 : #endif
6562 :
6563 : #endif
6564 :
6565 : /************************************************************************/
6566 : /* GDALTranspose2D() */
6567 : /************************************************************************/
6568 :
6569 : /**
6570 : * Transpose a 2D array in a efficient (cache-oblivious) way.
6571 : *
6572 : * @param pSrc Source array of width = nSrcWidth and height = nSrcHeight.
6573 : * @param eSrcType Data type of pSrc.
6574 : * @param pDst Destination transposed array of width = nSrcHeight and height = nSrcWidth.
6575 : * @param eDstType Data type of pDst.
6576 : * @param nSrcWidth Width of pSrc array.
6577 : * @param nSrcHeight Height of pSrc array.
6578 : * @since GDAL 3.11
6579 : */
6580 :
6581 305 : void GDALTranspose2D(const void *pSrc, GDALDataType eSrcType, void *pDst,
6582 : GDALDataType eDstType, size_t nSrcWidth, size_t nSrcHeight)
6583 : {
6584 305 : if (eSrcType == eDstType && (eSrcType == GDT_Byte || eSrcType == GDT_Int8))
6585 : {
6586 25 : if (nSrcHeight == 2)
6587 : {
6588 4 : GDALInterleave2Byte(static_cast<const uint8_t *>(pSrc),
6589 : static_cast<uint8_t *>(pDst), nSrcWidth);
6590 4 : return;
6591 : }
6592 21 : if (nSrcHeight == 4)
6593 : {
6594 2 : GDALInterleave4Byte(static_cast<const uint8_t *>(pSrc),
6595 : static_cast<uint8_t *>(pDst), nSrcWidth);
6596 2 : return;
6597 : }
6598 : #if (defined(HAVE_SSSE3_AT_COMPILE_TIME) && \
6599 : (defined(__x86_64) || defined(_M_X64)))
6600 19 : if (CPLHaveRuntimeSSSE3())
6601 : {
6602 19 : GDALTranspose2D_Byte_SSSE3(static_cast<const uint8_t *>(pSrc),
6603 : static_cast<uint8_t *>(pDst), nSrcWidth,
6604 : nSrcHeight);
6605 19 : return;
6606 : }
6607 : #elif defined(USE_NEON_OPTIMIZATIONS)
6608 : {
6609 : GDALTranspose2D_Byte_SSSE3(static_cast<const uint8_t *>(pSrc),
6610 : static_cast<uint8_t *>(pDst), nSrcWidth,
6611 : nSrcHeight);
6612 : return;
6613 : }
6614 : #endif
6615 : }
6616 :
6617 : #define CALL_GDALTranspose2D_internal(DST_TYPE, DST_IS_COMPLEX) \
6618 : GDALTranspose2D<DST_TYPE, DST_IS_COMPLEX>( \
6619 : pSrc, eSrcType, static_cast<DST_TYPE *>(pDst), nSrcWidth, nSrcHeight)
6620 :
6621 : // clang-format off
6622 280 : switch (eDstType)
6623 : {
6624 15 : case GDT_Byte: CALL_GDALTranspose2D_internal(uint8_t, false); break;
6625 15 : case GDT_Int8: CALL_GDALTranspose2D_internal(int8_t, false); break;
6626 24 : case GDT_UInt16: CALL_GDALTranspose2D_internal(uint16_t, false); break;
6627 16 : case GDT_Int16: CALL_GDALTranspose2D_internal(int16_t, false); break;
6628 24 : case GDT_UInt32: CALL_GDALTranspose2D_internal(uint32_t, false); break;
6629 16 : case GDT_Int32: CALL_GDALTranspose2D_internal(int32_t, false); break;
6630 16 : case GDT_UInt64: CALL_GDALTranspose2D_internal(uint64_t, false); break;
6631 16 : case GDT_Int64: CALL_GDALTranspose2D_internal(int64_t, false); break;
6632 16 : case GDT_Float16: CALL_GDALTranspose2D_internal(GFloat16, false); break;
6633 17 : case GDT_Float32: CALL_GDALTranspose2D_internal(float, false); break;
6634 25 : case GDT_Float64: CALL_GDALTranspose2D_internal(double, false); break;
6635 16 : case GDT_CInt16: CALL_GDALTranspose2D_internal(int16_t, true); break;
6636 16 : case GDT_CInt32: CALL_GDALTranspose2D_internal(int32_t, true); break;
6637 16 : case GDT_CFloat16: CALL_GDALTranspose2D_internal(GFloat16, true); break;
6638 16 : case GDT_CFloat32: CALL_GDALTranspose2D_internal(float, true); break;
6639 16 : case GDT_CFloat64: CALL_GDALTranspose2D_internal(double, true); break;
6640 0 : case GDT_Unknown:
6641 : case GDT_TypeCount:
6642 0 : break;
6643 : }
6644 : // clang-format on
6645 :
6646 : #undef CALL_GDALTranspose2D_internal
6647 : }
6648 :
6649 : /************************************************************************/
6650 : /* ExtractBitAndConvertTo255() */
6651 : /************************************************************************/
6652 :
6653 : #if defined(__GNUC__) || defined(_MSC_VER)
6654 : // Signedness of char implementation dependent, so be explicit.
6655 : // Assumes 2-complement integer types and sign extension of right shifting
6656 : // GCC guarantees such:
6657 : // https://gcc.gnu.org/onlinedocs/gcc/Integers-implementation.html#Integers-implementation
6658 157290 : static inline GByte ExtractBitAndConvertTo255(GByte byVal, int nBit)
6659 : {
6660 157290 : return static_cast<GByte>(static_cast<signed char>(byVal << (7 - nBit)) >>
6661 157290 : 7);
6662 : }
6663 : #else
6664 : // Portable way
6665 : static inline GByte ExtractBitAndConvertTo255(GByte byVal, int nBit)
6666 : {
6667 : return (byVal & (1 << nBit)) ? 255 : 0;
6668 : }
6669 : #endif
6670 :
6671 : /************************************************************************/
6672 : /* ExpandEightPackedBitsToByteAt255() */
6673 : /************************************************************************/
6674 :
6675 19457 : static inline void ExpandEightPackedBitsToByteAt255(GByte byVal,
6676 : GByte abyOutput[8])
6677 : {
6678 19457 : abyOutput[0] = ExtractBitAndConvertTo255(byVal, 7);
6679 19457 : abyOutput[1] = ExtractBitAndConvertTo255(byVal, 6);
6680 19457 : abyOutput[2] = ExtractBitAndConvertTo255(byVal, 5);
6681 19457 : abyOutput[3] = ExtractBitAndConvertTo255(byVal, 4);
6682 19457 : abyOutput[4] = ExtractBitAndConvertTo255(byVal, 3);
6683 19457 : abyOutput[5] = ExtractBitAndConvertTo255(byVal, 2);
6684 19457 : abyOutput[6] = ExtractBitAndConvertTo255(byVal, 1);
6685 19457 : abyOutput[7] = ExtractBitAndConvertTo255(byVal, 0);
6686 19457 : }
6687 :
6688 : /************************************************************************/
6689 : /* GDALExpandPackedBitsToByteAt0Or255() */
6690 : /************************************************************************/
6691 :
6692 : /** Expand packed-bits (ordered from most-significant bit to least one)
6693 : into a byte each, where a bit at 0 is expanded to a byte at 0, and a bit
6694 : at 1 to a byte at 255.
6695 :
6696 : The function does (in a possibly more optimized way) the following:
6697 : \code{.cpp}
6698 : for (size_t i = 0; i < nInputBits; ++i )
6699 : {
6700 : pabyOutput[i] = (pabyInput[i / 8] & (1 << (7 - (i % 8)))) ? 255 : 0;
6701 : }
6702 : \endcode
6703 :
6704 : @param pabyInput Input array of (nInputBits + 7) / 8 bytes.
6705 : @param pabyOutput Output array of nInputBits bytes.
6706 : @param nInputBits Number of valid bits in pabyInput.
6707 :
6708 : @since 3.11
6709 : */
6710 :
6711 44445 : void GDALExpandPackedBitsToByteAt0Or255(const GByte *CPL_RESTRICT pabyInput,
6712 : GByte *CPL_RESTRICT pabyOutput,
6713 : size_t nInputBits)
6714 : {
6715 44445 : const size_t nInputWholeBytes = nInputBits / 8;
6716 44445 : size_t iByte = 0;
6717 :
6718 : #ifdef HAVE_SSE2
6719 : // Mask to isolate each bit
6720 44445 : const __m128i bit_mask = _mm_set_epi8(1, 2, 4, 8, 16, 32, 64, -128, 1, 2, 4,
6721 : 8, 16, 32, 64, -128);
6722 44445 : const __m128i zero = _mm_setzero_si128();
6723 44445 : const __m128i all_ones = _mm_set1_epi8(-1);
6724 : #ifdef __SSSE3__
6725 : const __m128i dispatch_two_bytes =
6726 : _mm_set_epi8(1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0);
6727 : #endif
6728 44445 : constexpr size_t SSE_REG_SIZE = sizeof(bit_mask);
6729 132394 : for (; iByte + SSE_REG_SIZE <= nInputWholeBytes; iByte += SSE_REG_SIZE)
6730 : {
6731 87949 : __m128i reg_ori = _mm_loadu_si128(
6732 87949 : reinterpret_cast<const __m128i *>(pabyInput + iByte));
6733 :
6734 87949 : constexpr int NUM_PROCESSED_BYTES_PER_REG = 2;
6735 791541 : for (size_t k = 0; k < SSE_REG_SIZE / NUM_PROCESSED_BYTES_PER_REG; ++k)
6736 : {
6737 : // Given reg_ori = (A, B, ... 14 other bytes ...),
6738 : // expand to (A, A, A, A, A, A, A, A, B, B, B, B, B, B, B, B)
6739 : #ifdef __SSSE3__
6740 : __m128i reg = _mm_shuffle_epi8(reg_ori, dispatch_two_bytes);
6741 : #else
6742 703592 : __m128i reg = _mm_unpacklo_epi8(reg_ori, reg_ori);
6743 703592 : reg = _mm_unpacklo_epi16(reg, reg);
6744 703592 : reg = _mm_unpacklo_epi32(reg, reg);
6745 : #endif
6746 :
6747 : // Test if bits of interest are set
6748 703592 : reg = _mm_and_si128(reg, bit_mask);
6749 :
6750 : // Now test if those bits are set, by comparing to zero. So the
6751 : // result will be that bytes where bits are set will be at 0, and
6752 : // ones where they are cleared will be at 0xFF. So the inverse of
6753 : // the end result we want!
6754 703592 : reg = _mm_cmpeq_epi8(reg, zero);
6755 :
6756 : // Invert the result
6757 703592 : reg = _mm_andnot_si128(reg, all_ones);
6758 :
6759 : _mm_storeu_si128(reinterpret_cast<__m128i *>(pabyOutput), reg);
6760 :
6761 703592 : pabyOutput += SSE_REG_SIZE;
6762 :
6763 : // Right-shift of 2 bytes
6764 703592 : reg_ori = _mm_bsrli_si128(reg_ori, NUM_PROCESSED_BYTES_PER_REG);
6765 : }
6766 : }
6767 :
6768 : #endif // HAVE_SSE2
6769 :
6770 63902 : for (; iByte < nInputWholeBytes; ++iByte)
6771 : {
6772 19457 : ExpandEightPackedBitsToByteAt255(pabyInput[iByte], pabyOutput);
6773 19457 : pabyOutput += 8;
6774 : }
6775 46079 : for (int iBit = 0; iBit < static_cast<int>(nInputBits % 8); ++iBit)
6776 : {
6777 1634 : *pabyOutput = ExtractBitAndConvertTo255(pabyInput[iByte], 7 - iBit);
6778 1634 : ++pabyOutput;
6779 : }
6780 44445 : }
6781 :
6782 : /************************************************************************/
6783 : /* ExpandEightPackedBitsToByteAt1() */
6784 : /************************************************************************/
6785 :
6786 136113 : static inline void ExpandEightPackedBitsToByteAt1(GByte byVal,
6787 : GByte abyOutput[8])
6788 : {
6789 136113 : abyOutput[0] = (byVal >> 7) & 0x1;
6790 136113 : abyOutput[1] = (byVal >> 6) & 0x1;
6791 136113 : abyOutput[2] = (byVal >> 5) & 0x1;
6792 136113 : abyOutput[3] = (byVal >> 4) & 0x1;
6793 136113 : abyOutput[4] = (byVal >> 3) & 0x1;
6794 136113 : abyOutput[5] = (byVal >> 2) & 0x1;
6795 136113 : abyOutput[6] = (byVal >> 1) & 0x1;
6796 136113 : abyOutput[7] = (byVal >> 0) & 0x1;
6797 136113 : }
6798 :
6799 : /************************************************************************/
6800 : /* GDALExpandPackedBitsToByteAt0Or1() */
6801 : /************************************************************************/
6802 :
6803 : /** Expand packed-bits (ordered from most-significant bit to least one)
6804 : into a byte each, where a bit at 0 is expanded to a byte at 0, and a bit
6805 : at 1 to a byte at 1.
6806 :
6807 : The function does (in a possibly more optimized way) the following:
6808 : \code{.cpp}
6809 : for (size_t i = 0; i < nInputBits; ++i )
6810 : {
6811 : pabyOutput[i] = (pabyInput[i / 8] & (1 << (7 - (i % 8)))) ? 1 : 0;
6812 : }
6813 : \endcode
6814 :
6815 : @param pabyInput Input array of (nInputBits + 7) / 8 bytes.
6816 : @param pabyOutput Output array of nInputBits bytes.
6817 : @param nInputBits Number of valid bits in pabyInput.
6818 :
6819 : @since 3.11
6820 : */
6821 :
6822 7041 : void GDALExpandPackedBitsToByteAt0Or1(const GByte *CPL_RESTRICT pabyInput,
6823 : GByte *CPL_RESTRICT pabyOutput,
6824 : size_t nInputBits)
6825 : {
6826 7041 : const size_t nInputWholeBytes = nInputBits / 8;
6827 7041 : size_t iByte = 0;
6828 143154 : for (; iByte < nInputWholeBytes; ++iByte)
6829 : {
6830 136113 : ExpandEightPackedBitsToByteAt1(pabyInput[iByte], pabyOutput);
6831 136113 : pabyOutput += 8;
6832 : }
6833 18902 : for (int iBit = 0; iBit < static_cast<int>(nInputBits % 8); ++iBit)
6834 : {
6835 11861 : *pabyOutput = (pabyInput[iByte] >> (7 - iBit)) & 0x1;
6836 11861 : ++pabyOutput;
6837 : }
6838 7041 : }
|
