Line data Source code
1 : /******************************************************************************
2 : *
3 : * Project: GDAL Core
4 : * Purpose: Contains default implementation of GDALRasterBand::IRasterIO()
5 : * and supporting functions of broader utility.
6 : * Author: Frank Warmerdam, warmerdam@pobox.com
7 : *
8 : ******************************************************************************
9 : * Copyright (c) 1998, Frank Warmerdam
10 : * Copyright (c) 2007-2014, Even Rouault <even dot rouault at spatialys.com>
11 : *
12 : * SPDX-License-Identifier: MIT
13 : ****************************************************************************/
14 :
15 : #include "cpl_port.h"
16 : #include "gdal.h"
17 : #include "gdal_priv.h"
18 :
19 : #include <cassert>
20 : #include <climits>
21 : #include <cmath>
22 : #include <cstddef>
23 : #include <cstdio>
24 : #include <cstdlib>
25 : #include <cstring>
26 :
27 : #include <algorithm>
28 : #include <limits>
29 : #include <stdexcept>
30 : #include <type_traits>
31 :
32 : #include "cpl_conv.h"
33 : #include "cpl_cpu_features.h"
34 : #include "cpl_error.h"
35 : #include "cpl_float.h"
36 : #include "cpl_progress.h"
37 : #include "cpl_string.h"
38 : #include "cpl_vsi.h"
39 : #include "gdal_priv_templates.hpp"
40 : #include "gdal_vrt.h"
41 : #include "gdalwarper.h"
42 : #include "memdataset.h"
43 : #include "vrtdataset.h"
44 :
45 : #if defined(__x86_64) || defined(_M_X64)
46 : #include <emmintrin.h>
47 : #include <immintrin.h>
48 : #define HAVE_SSE2
49 : // AVX2 dispatch: compile AVX2 code with target attribute, detect at runtime
50 : #if (defined(__GNUC__) || defined(__clang__)) && \
51 : defined(HAVE_AVX2_AT_COMPILE_TIME)
52 : #define HAVE_AVX2_DISPATCH
53 : #elif defined(_MSC_VER)
54 : #include <intrin.h>
55 : #define HAVE_AVX2_DISPATCH
56 : #endif
57 : #elif defined(USE_NEON_OPTIMIZATIONS)
58 : #include "include_sse2neon.h"
59 : #define HAVE_SSE2
60 : #endif
61 :
62 : #ifdef HAVE_SSSE3_AT_COMPILE_TIME
63 : #include "rasterio_ssse3.h"
64 : #ifdef __SSSE3__
65 : #include <tmmintrin.h>
66 : #endif
67 : #endif
68 :
69 : #ifdef __SSE4_1__
70 : #include <smmintrin.h>
71 : #endif
72 :
73 : #ifdef __GNUC__
74 : #define CPL_NOINLINE __attribute__((noinline))
75 : #else
76 : #define CPL_NOINLINE
77 : #endif
78 :
79 : static void GDALFastCopyByte(const GByte *CPL_RESTRICT pSrcData,
80 : int nSrcPixelStride, GByte *CPL_RESTRICT pDstData,
81 : int nDstPixelStride, GPtrDiff_t nWordCount);
82 :
83 : /************************************************************************/
84 : /* DownsamplingIntegerXFactor() */
85 : /************************************************************************/
86 :
87 : template <bool bSameDataType, int DATA_TYPE_SIZE>
88 695850 : static bool DownsamplingIntegerXFactor(
89 : GDALRasterBand *poBand, int iSrcX, int nSrcXInc, GPtrDiff_t iSrcOffsetCst,
90 : GByte *CPL_RESTRICT pabyDstData, int nPixelSpace, int nBufXSize,
91 : GDALDataType eDataType, GDALDataType eBufType, int &nStartBlockX,
92 : int nBlockXSize, GDALRasterBlock *&poBlock, int nLBlockY)
93 : {
94 695850 : const int nBandDataSize =
95 : bSameDataType ? DATA_TYPE_SIZE : GDALGetDataTypeSizeBytes(eDataType);
96 695850 : int nOuterLoopIters = nBufXSize - 1;
97 695850 : const int nIncSrcOffset = nSrcXInc * nBandDataSize;
98 : const GByte *CPL_RESTRICT pabySrcData;
99 695850 : int nEndBlockX = nBlockXSize + nStartBlockX;
100 :
101 695850 : if (iSrcX < nEndBlockX)
102 : {
103 295062 : CPLAssert(poBlock);
104 295062 : goto no_reload_block;
105 : }
106 400788 : goto reload_block;
107 :
108 : // Don't do the last iteration in the loop, as iSrcX might go beyond
109 : // nRasterXSize - 1
110 1265113 : while (--nOuterLoopIters >= 1)
111 : {
112 201834 : iSrcX += nSrcXInc;
113 201834 : pabySrcData += nIncSrcOffset;
114 201834 : pabyDstData += nPixelSpace;
115 :
116 : /* --------------------------------------------------------------------
117 : */
118 : /* Ensure we have the appropriate block loaded. */
119 : /* --------------------------------------------------------------------
120 : */
121 201834 : if (iSrcX >= nEndBlockX)
122 : {
123 201834 : reload_block:
124 : {
125 615212 : const int nLBlockX = iSrcX / nBlockXSize;
126 615212 : nStartBlockX = nLBlockX * nBlockXSize;
127 615212 : nEndBlockX = nStartBlockX + nBlockXSize;
128 :
129 615212 : if (poBlock != nullptr)
130 341376 : poBlock->DropLock();
131 :
132 615212 : poBlock = poBand->GetLockedBlockRef(nLBlockX, nLBlockY, FALSE);
133 615212 : if (poBlock == nullptr)
134 : {
135 1 : return false;
136 : }
137 : }
138 :
139 615211 : no_reload_block:
140 : const GByte *pabySrcBlock =
141 1265113 : static_cast<const GByte *>(poBlock->GetDataRef());
142 1265113 : GPtrDiff_t iSrcOffset =
143 1265113 : (iSrcX - nStartBlockX + iSrcOffsetCst) * nBandDataSize;
144 1265113 : pabySrcData = pabySrcBlock + iSrcOffset;
145 : }
146 :
147 : /* --------------------------------------------------------------------
148 : */
149 : /* Copy the maximum run of pixels. */
150 : /* --------------------------------------------------------------------
151 : */
152 :
153 1265113 : const int nIters = std::min(
154 1265113 : (nEndBlockX - iSrcX + (nSrcXInc - 1)) / nSrcXInc, nOuterLoopIters);
155 : if (bSameDataType)
156 : {
157 1264670 : memcpy(pabyDstData, pabySrcData, nBandDataSize);
158 1264670 : if (nIters > 1)
159 : {
160 : if (DATA_TYPE_SIZE == 1)
161 : {
162 326320 : pabySrcData += nIncSrcOffset;
163 326320 : pabyDstData += nPixelSpace;
164 326320 : GDALFastCopyByte(pabySrcData, nIncSrcOffset, pabyDstData,
165 326320 : nPixelSpace, nIters - 1);
166 326320 : pabySrcData +=
167 326320 : static_cast<GPtrDiff_t>(nIncSrcOffset) * (nIters - 2);
168 326320 : pabyDstData +=
169 326320 : static_cast<GPtrDiff_t>(nPixelSpace) * (nIters - 2);
170 : }
171 : else
172 : {
173 4395716 : for (int i = 0; i < nIters - 1; i++)
174 : {
175 4197550 : pabySrcData += nIncSrcOffset;
176 4197550 : pabyDstData += nPixelSpace;
177 4197550 : memcpy(pabyDstData, pabySrcData, nBandDataSize);
178 : }
179 : }
180 524490 : iSrcX += nSrcXInc * (nIters - 1);
181 524490 : nOuterLoopIters -= nIters - 1;
182 : }
183 : }
184 : else
185 : {
186 : // Type to type conversion ...
187 443 : GDALCopyWords64(pabySrcData, eDataType, nIncSrcOffset, pabyDstData,
188 443 : eBufType, nPixelSpace, std::max(1, nIters));
189 443 : if (nIters > 1)
190 : {
191 216 : pabySrcData +=
192 216 : static_cast<GPtrDiff_t>(nIncSrcOffset) * (nIters - 1);
193 216 : pabyDstData +=
194 216 : static_cast<GPtrDiff_t>(nPixelSpace) * (nIters - 1);
195 216 : iSrcX += nSrcXInc * (nIters - 1);
196 216 : nOuterLoopIters -= nIters - 1;
197 : }
198 : }
199 : }
200 :
201 : // Deal with last iteration to avoid iSrcX to go beyond nRasterXSize - 1
202 1063279 : if (nOuterLoopIters == 0)
203 : {
204 367430 : const int nRasterXSize = poBand->GetXSize();
205 367430 : iSrcX =
206 734860 : static_cast<int>(std::min(static_cast<GInt64>(iSrcX) + nSrcXInc,
207 367430 : static_cast<GInt64>(nRasterXSize - 1)));
208 367430 : pabyDstData += nPixelSpace;
209 367430 : if (iSrcX < nEndBlockX)
210 : {
211 354840 : goto no_reload_block;
212 : }
213 12590 : goto reload_block;
214 : }
215 695849 : return true;
216 : }
217 :
218 : template <class A, class B>
219 2832260 : CPL_NOSANITIZE_UNSIGNED_INT_OVERFLOW inline auto CPLUnsanitizedMul(A a, B b)
220 : {
221 2832260 : return a * b;
222 : }
223 :
224 : /************************************************************************/
225 : /* IRasterIO() */
226 : /* */
227 : /* Default internal implementation of RasterIO() ... utilizes */
228 : /* the Block access methods to satisfy the request. This would */
229 : /* normally only be overridden by formats with overviews. */
230 : /************************************************************************/
231 :
232 6193540 : CPLErr GDALRasterBand::IRasterIO(GDALRWFlag eRWFlag, int nXOff, int nYOff,
233 : int nXSize, int nYSize, void *pData,
234 : int nBufXSize, int nBufYSize,
235 : GDALDataType eBufType, GSpacing nPixelSpace,
236 : GSpacing nLineSpace,
237 : GDALRasterIOExtraArg *psExtraArg)
238 :
239 : {
240 6193540 : if (eRWFlag == GF_Write && eFlushBlockErr != CE_None)
241 : {
242 0 : CPLError(eFlushBlockErr, CPLE_AppDefined,
243 : "An error occurred while writing a dirty block "
244 : "from GDALRasterBand::IRasterIO");
245 0 : CPLErr eErr = eFlushBlockErr;
246 0 : eFlushBlockErr = CE_None;
247 0 : return eErr;
248 : }
249 6193540 : if (nBlockXSize <= 0 || nBlockYSize <= 0)
250 : {
251 0 : CPLError(CE_Failure, CPLE_AppDefined, "Invalid block size");
252 0 : return CE_Failure;
253 : }
254 :
255 6193540 : const int nBandDataSize = GDALGetDataTypeSizeBytes(eDataType);
256 6193540 : const int nBufDataSize = GDALGetDataTypeSizeBytes(eBufType);
257 6193540 : GByte dummyBlock[2] = {0, 0};
258 6193540 : GByte *pabySrcBlock =
259 : dummyBlock; /* to avoid Coverity warning about nullptr dereference */
260 6193540 : GDALRasterBlock *poBlock = nullptr;
261 6193540 : const bool bUseIntegerRequestCoords =
262 6560100 : (!psExtraArg->bFloatingPointWindowValidity ||
263 366561 : (nXOff == psExtraArg->dfXOff && nYOff == psExtraArg->dfYOff &&
264 341629 : nXSize == psExtraArg->dfXSize && nYSize == psExtraArg->dfYSize));
265 :
266 : /* ==================================================================== */
267 : /* A common case is the data requested with the destination */
268 : /* is packed, and the block width is the raster width. */
269 : /* ==================================================================== */
270 6100190 : if (nPixelSpace == nBufDataSize && nLineSpace == nPixelSpace * nXSize &&
271 3239250 : nBlockXSize == GetXSize() && nBufXSize == nXSize &&
272 12293700 : nBufYSize == nYSize && bUseIntegerRequestCoords)
273 : {
274 3098300 : CPLErr eErr = CE_None;
275 3098300 : int nLBlockY = -1;
276 :
277 9801160 : for (int iBufYOff = 0; iBufYOff < nBufYSize; iBufYOff++)
278 : {
279 6703950 : const int iSrcY = iBufYOff + nYOff;
280 :
281 6703950 : if (iSrcY < nLBlockY * nBlockYSize ||
282 6703950 : iSrcY - nBlockYSize >= nLBlockY * nBlockYSize)
283 : {
284 3367630 : nLBlockY = iSrcY / nBlockYSize;
285 3367630 : bool bJustInitialize =
286 298076 : eRWFlag == GF_Write && nXOff == 0 &&
287 3723670 : nXSize == nBlockXSize && nYOff <= nLBlockY * nBlockYSize &&
288 57965 : nYOff + nYSize - nBlockYSize >= nLBlockY * nBlockYSize;
289 :
290 : // Is this a partial tile at right and/or bottom edges of
291 : // the raster, and that is going to be completely written?
292 : // If so, do not load it from storage, but zero it so that
293 : // the content outsize of the validity area is initialized.
294 3367630 : bool bMemZeroBuffer = false;
295 298076 : if (eRWFlag == GF_Write && !bJustInitialize && nXOff == 0 &&
296 25682 : nXSize == nBlockXSize && nYOff <= nLBlockY * nBlockYSize &&
297 3665800 : nYOff + nYSize == GetYSize() &&
298 90 : nLBlockY * nBlockYSize > GetYSize() - nBlockYSize)
299 : {
300 90 : bJustInitialize = true;
301 90 : bMemZeroBuffer = true;
302 : }
303 :
304 3367630 : if (poBlock)
305 269338 : poBlock->DropLock();
306 :
307 3367630 : const GUInt32 nErrorCounter = CPLGetErrorCounter();
308 3367630 : poBlock = GetLockedBlockRef(0, nLBlockY, bJustInitialize);
309 3367630 : if (poBlock == nullptr)
310 : {
311 1079 : if (strstr(CPLGetLastErrorMsg(), "IReadBlock failed") ==
312 : nullptr)
313 : {
314 0 : CPLError(CE_Failure, CPLE_AppDefined,
315 : "GetBlockRef failed at X block offset %d, "
316 : "Y block offset %d%s",
317 : 0, nLBlockY,
318 0 : (nErrorCounter != CPLGetErrorCounter())
319 0 : ? CPLSPrintf(": %s", CPLGetLastErrorMsg())
320 : : "");
321 : }
322 1079 : eErr = CE_Failure;
323 1079 : break;
324 : }
325 :
326 3366550 : if (eRWFlag == GF_Write)
327 298076 : poBlock->MarkDirty();
328 :
329 3366550 : pabySrcBlock = static_cast<GByte *>(poBlock->GetDataRef());
330 3366550 : if (bMemZeroBuffer)
331 : {
332 90 : memset(pabySrcBlock, 0,
333 90 : static_cast<GPtrDiff_t>(nBandDataSize) *
334 90 : nBlockXSize * nBlockYSize);
335 : }
336 : }
337 :
338 6702870 : const auto nSrcByteOffset =
339 6702870 : (static_cast<GPtrDiff_t>(iSrcY - nLBlockY * nBlockYSize) *
340 6702870 : nBlockXSize +
341 6702870 : nXOff) *
342 6702870 : nBandDataSize;
343 :
344 6702870 : if (eDataType == eBufType)
345 : {
346 3038300 : if (eRWFlag == GF_Read)
347 2563580 : memcpy(static_cast<GByte *>(pData) +
348 2563580 : static_cast<GPtrDiff_t>(iBufYOff) * nLineSpace,
349 2563580 : pabySrcBlock + nSrcByteOffset,
350 : static_cast<size_t>(nLineSpace));
351 : else
352 474723 : memcpy(pabySrcBlock + nSrcByteOffset,
353 474723 : static_cast<GByte *>(pData) +
354 474723 : static_cast<GPtrDiff_t>(iBufYOff) * nLineSpace,
355 : static_cast<size_t>(nLineSpace));
356 : }
357 : else
358 : {
359 : // Type to type conversion.
360 3664570 : if (eRWFlag == GF_Read)
361 3642270 : GDALCopyWords64(
362 3642270 : pabySrcBlock + nSrcByteOffset, eDataType, nBandDataSize,
363 : static_cast<GByte *>(pData) +
364 3642270 : static_cast<GPtrDiff_t>(iBufYOff) * nLineSpace,
365 : eBufType, static_cast<int>(nPixelSpace), nBufXSize);
366 : else
367 22299 : GDALCopyWords64(static_cast<GByte *>(pData) +
368 22299 : static_cast<GPtrDiff_t>(iBufYOff) *
369 : nLineSpace,
370 : eBufType, static_cast<int>(nPixelSpace),
371 22299 : pabySrcBlock + nSrcByteOffset, eDataType,
372 : nBandDataSize, nBufXSize);
373 : }
374 :
375 6790780 : if (psExtraArg->pfnProgress != nullptr &&
376 87908 : !psExtraArg->pfnProgress(1.0 * (iBufYOff + 1) / nBufYSize, "",
377 : psExtraArg->pProgressData))
378 : {
379 5 : eErr = CE_Failure;
380 5 : break;
381 : }
382 : }
383 :
384 3098300 : if (poBlock)
385 3097220 : poBlock->DropLock();
386 :
387 3098300 : return eErr;
388 : }
389 :
390 : /* ==================================================================== */
391 : /* Do we have overviews that would be appropriate to satisfy */
392 : /* this request? */
393 : /* ==================================================================== */
394 3095240 : if ((nBufXSize < nXSize || nBufYSize < nYSize) && GetOverviewCount() > 0 &&
395 : eRWFlag == GF_Read)
396 : {
397 : GDALRasterIOExtraArg sExtraArg;
398 2967 : GDALCopyRasterIOExtraArg(&sExtraArg, psExtraArg);
399 :
400 : const int nOverview =
401 2967 : GDALBandGetBestOverviewLevel2(this, nXOff, nYOff, nXSize, nYSize,
402 : nBufXSize, nBufYSize, &sExtraArg);
403 2967 : if (nOverview >= 0)
404 : {
405 2892 : GDALRasterBand *poOverviewBand = GetOverview(nOverview);
406 2892 : if (poOverviewBand == nullptr)
407 2892 : return CE_Failure;
408 :
409 2892 : return poOverviewBand->RasterIO(
410 : eRWFlag, nXOff, nYOff, nXSize, nYSize, pData, nBufXSize,
411 2892 : nBufYSize, eBufType, nPixelSpace, nLineSpace, &sExtraArg);
412 : }
413 : }
414 :
415 902829 : if (eRWFlag == GF_Read && nBufXSize < nXSize / 100 &&
416 6 : nBufYSize < nYSize / 100 && nPixelSpace == nBufDataSize &&
417 3995180 : nLineSpace == nPixelSpace * nBufXSize &&
418 6 : CPLTestBool(CPLGetConfigOption("GDAL_NO_COSTLY_OVERVIEW", "NO")))
419 : {
420 0 : memset(pData, 0, static_cast<size_t>(nLineSpace * nBufYSize));
421 0 : return CE_None;
422 : }
423 :
424 : /* ==================================================================== */
425 : /* The second case when we don't need subsample data but likely */
426 : /* need data type conversion. */
427 : /* ==================================================================== */
428 3092350 : if ( // nPixelSpace == nBufDataSize &&
429 3092350 : nXSize == nBufXSize && nYSize == nBufYSize && bUseIntegerRequestCoords)
430 : {
431 : #if DEBUG_VERBOSE
432 : printf("IRasterIO(%d,%d,%d,%d) rw=%d case 2\n", /*ok*/
433 : nXOff, nYOff, nXSize, nYSize, static_cast<int>(eRWFlag));
434 : #endif
435 :
436 : /* --------------------------------------------------------------------
437 : */
438 : /* Loop over buffer computing source locations. */
439 : /* --------------------------------------------------------------------
440 : */
441 : // Calculate starting values out of loop
442 2512900 : const int nLBlockXStart = nXOff / nBlockXSize;
443 2512900 : const int nXSpanEnd = nBufXSize + nXOff;
444 :
445 2512900 : int iBufYOff = 0;
446 2512900 : int iSrcY = nYOff;
447 : while (true)
448 : {
449 2553840 : GPtrDiff_t iBufOffset = static_cast<GPtrDiff_t>(iBufYOff) *
450 : static_cast<GPtrDiff_t>(nLineSpace);
451 2553840 : int nLBlockY = iSrcY / nBlockYSize;
452 2553840 : int nLBlockX = nLBlockXStart;
453 2553840 : int iSrcX = nXOff;
454 5386020 : while (iSrcX < nXSpanEnd)
455 : {
456 2832260 : int nXSpan = nLBlockX * nBlockXSize;
457 2832260 : if (nXSpan < INT_MAX - nBlockXSize)
458 2832260 : nXSpan += nBlockXSize;
459 : else
460 0 : nXSpan = INT_MAX;
461 2832260 : const int nXRight = nXSpan;
462 2832260 : nXSpan = (nXSpan < nXSpanEnd ? nXSpan : nXSpanEnd) - iSrcX;
463 :
464 : const size_t nXSpanSize =
465 2832260 : CPLUnsanitizedMul(nXSpan, static_cast<size_t>(nPixelSpace));
466 :
467 2832260 : bool bJustInitialize =
468 2043060 : eRWFlag == GF_Write && nYOff <= nLBlockY * nBlockYSize &&
469 38122 : nYOff + nYSize - nBlockYSize >= nLBlockY * nBlockYSize &&
470 4901720 : nXOff <= nLBlockX * nBlockXSize &&
471 26406 : nXOff + nXSize >= nXRight;
472 :
473 : // Is this a partial tile at right and/or bottom edges of
474 : // the raster, and that is going to be completely written?
475 : // If so, do not load it from storage, but zero it so that
476 : // the content outsize of the validity area is initialized.
477 2832260 : bool bMemZeroBuffer = false;
478 2043060 : if (eRWFlag == GF_Write && !bJustInitialize &&
479 2017900 : nXOff <= nLBlockX * nBlockXSize &&
480 2016250 : nYOff <= nLBlockY * nBlockYSize &&
481 12202 : (nXOff + nXSize >= nXRight ||
482 : // cppcheck-suppress knownConditionTrueFalse
483 4878070 : (nXOff + nXSize == GetXSize() && nXRight > GetXSize())) &&
484 12022 : (nYOff + nYSize - nBlockYSize >= nLBlockY * nBlockYSize ||
485 10788 : (nYOff + nYSize == GetYSize() &&
486 1996 : nLBlockY * nBlockYSize > GetYSize() - nBlockYSize)))
487 : {
488 3230 : bJustInitialize = true;
489 3230 : bMemZeroBuffer = true;
490 : }
491 :
492 : /* --------------------------------------------------------------------
493 : */
494 : /* Ensure we have the appropriate block loaded. */
495 : /* --------------------------------------------------------------------
496 : */
497 2832260 : const GUInt32 nErrorCounter = CPLGetErrorCounter();
498 2832260 : poBlock =
499 2832260 : GetLockedBlockRef(nLBlockX, nLBlockY, bJustInitialize);
500 2832260 : if (!poBlock)
501 : {
502 73 : if (strstr(CPLGetLastErrorMsg(), "IReadBlock failed") ==
503 : nullptr)
504 : {
505 0 : CPLError(CE_Failure, CPLE_AppDefined,
506 : "GetBlockRef failed at X block offset %d, "
507 : "Y block offset %d%s",
508 : nLBlockX, nLBlockY,
509 0 : (nErrorCounter != CPLGetErrorCounter())
510 0 : ? CPLSPrintf(": %s", CPLGetLastErrorMsg())
511 : : "");
512 : }
513 73 : return (CE_Failure);
514 : }
515 :
516 2832180 : if (eRWFlag == GF_Write)
517 2043060 : poBlock->MarkDirty();
518 :
519 2832180 : pabySrcBlock = static_cast<GByte *>(poBlock->GetDataRef());
520 2832180 : if (bMemZeroBuffer)
521 : {
522 3230 : memset(pabySrcBlock, 0,
523 3230 : static_cast<GPtrDiff_t>(nBandDataSize) *
524 3230 : nBlockXSize * nBlockYSize);
525 : }
526 : /* --------------------------------------------------------------------
527 : */
528 : /* Copy over this chunk of data. */
529 : /* --------------------------------------------------------------------
530 : */
531 2832180 : GPtrDiff_t iSrcOffset =
532 2832180 : (static_cast<GPtrDiff_t>(iSrcX) -
533 2832180 : static_cast<GPtrDiff_t>(nLBlockX * nBlockXSize) +
534 2832180 : (static_cast<GPtrDiff_t>(iSrcY) -
535 2832180 : static_cast<GPtrDiff_t>(nLBlockY) * nBlockYSize) *
536 2832180 : nBlockXSize) *
537 2832180 : nBandDataSize;
538 : // Fill up as many rows as possible for the loaded block.
539 5664360 : const int kmax = std::min(nBlockYSize - (iSrcY % nBlockYSize),
540 2832180 : nBufYSize - iBufYOff);
541 61147400 : for (int k = 0; k < kmax; k++)
542 : {
543 58315200 : if (eDataType == eBufType && nPixelSpace == nBufDataSize)
544 : {
545 53913400 : if (eRWFlag == GF_Read)
546 49463600 : memcpy(static_cast<GByte *>(pData) + iBufOffset +
547 49463600 : static_cast<GPtrDiff_t>(k) * nLineSpace,
548 49463600 : pabySrcBlock + iSrcOffset, nXSpanSize);
549 : else
550 4449710 : memcpy(pabySrcBlock + iSrcOffset,
551 4449710 : static_cast<GByte *>(pData) + iBufOffset +
552 4449710 : static_cast<GPtrDiff_t>(k) * nLineSpace,
553 : nXSpanSize);
554 : }
555 : else
556 : {
557 : /* type to type conversion */
558 4401890 : if (eRWFlag == GF_Read)
559 4251680 : GDALCopyWords64(
560 4251680 : pabySrcBlock + iSrcOffset, eDataType,
561 : nBandDataSize,
562 4251680 : static_cast<GByte *>(pData) + iBufOffset +
563 4251680 : static_cast<GPtrDiff_t>(k) * nLineSpace,
564 : eBufType, static_cast<int>(nPixelSpace),
565 : nXSpan);
566 : else
567 150209 : GDALCopyWords64(
568 150209 : static_cast<GByte *>(pData) + iBufOffset +
569 150209 : static_cast<GPtrDiff_t>(k) * nLineSpace,
570 : eBufType, static_cast<int>(nPixelSpace),
571 150209 : pabySrcBlock + iSrcOffset, eDataType,
572 : nBandDataSize, nXSpan);
573 : }
574 :
575 58315200 : iSrcOffset +=
576 58315200 : static_cast<GPtrDiff_t>(nBlockXSize) * nBandDataSize;
577 : }
578 :
579 : iBufOffset =
580 2832180 : CPLUnsanitizedAdd<GPtrDiff_t>(iBufOffset, nXSpanSize);
581 2832180 : nLBlockX++;
582 2832180 : iSrcX += nXSpan;
583 :
584 2832180 : poBlock->DropLock();
585 2832180 : poBlock = nullptr;
586 : }
587 :
588 : /* Compute the increment to go on a block boundary */
589 2553770 : const int nYInc = nBlockYSize - (iSrcY % nBlockYSize);
590 :
591 2555650 : if (psExtraArg->pfnProgress != nullptr &&
592 1884 : !psExtraArg->pfnProgress(
593 2555650 : 1.0 * std::min(nBufYSize, iBufYOff + nYInc) / nBufYSize, "",
594 : psExtraArg->pProgressData))
595 : {
596 0 : return CE_Failure;
597 : }
598 :
599 2553770 : iBufYOff += nYInc;
600 2553770 : if (iBufYOff >= nBufYSize)
601 2512820 : break;
602 : // Only increment iSrcY after above loop end check, to avoid
603 : // potential int overflow.
604 40944 : iSrcY += nYInc;
605 40944 : }
606 :
607 2512820 : return CE_None;
608 : }
609 :
610 : /* ==================================================================== */
611 : /* Loop reading required source blocks to satisfy output */
612 : /* request. This is the most general implementation. */
613 : /* ==================================================================== */
614 :
615 579452 : double dfXOff = nXOff;
616 579452 : double dfYOff = nYOff;
617 579452 : double dfXSize = nXSize;
618 579452 : double dfYSize = nYSize;
619 579452 : if (psExtraArg->bFloatingPointWindowValidity)
620 : {
621 244495 : dfXOff = psExtraArg->dfXOff;
622 244495 : dfYOff = psExtraArg->dfYOff;
623 244495 : dfXSize = psExtraArg->dfXSize;
624 244495 : dfYSize = psExtraArg->dfYSize;
625 : }
626 :
627 : /* -------------------------------------------------------------------- */
628 : /* Compute stepping increment. */
629 : /* -------------------------------------------------------------------- */
630 579452 : const double dfSrcXInc = dfXSize / static_cast<double>(nBufXSize);
631 579452 : const double dfSrcYInc = dfYSize / static_cast<double>(nBufYSize);
632 579452 : CPLErr eErr = CE_None;
633 :
634 579452 : if (eRWFlag == GF_Write)
635 : {
636 : /* --------------------------------------------------------------------
637 : */
638 : /* Write case */
639 : /* Loop over raster window computing source locations in the buffer.
640 : */
641 : /* --------------------------------------------------------------------
642 : */
643 166655 : GByte *pabyDstBlock = nullptr;
644 166655 : int nLBlockX = -1;
645 166655 : int nLBlockY = -1;
646 :
647 1260010 : for (int iDstY = nYOff; iDstY < nYOff + nYSize; iDstY++)
648 : {
649 1093360 : const int iBufYOff = static_cast<int>((iDstY - nYOff) / dfSrcYInc);
650 :
651 12384200 : for (int iDstX = nXOff; iDstX < nXOff + nXSize; iDstX++)
652 : {
653 11290800 : const int iBufXOff =
654 11290800 : static_cast<int>((iDstX - nXOff) / dfSrcXInc);
655 11290800 : GPtrDiff_t iBufOffset =
656 11290800 : static_cast<GPtrDiff_t>(iBufYOff) *
657 : static_cast<GPtrDiff_t>(nLineSpace) +
658 11290800 : iBufXOff * static_cast<GPtrDiff_t>(nPixelSpace);
659 :
660 : // FIXME: this code likely doesn't work if the dirty block gets
661 : // flushed to disk before being completely written.
662 : // In the meantime, bJustInitialize should probably be set to
663 : // FALSE even if it is not ideal performance wise, and for
664 : // lossy compression.
665 :
666 : /* --------------------------------------------------------------------
667 : */
668 : /* Ensure we have the appropriate block loaded. */
669 : /* --------------------------------------------------------------------
670 : */
671 11290800 : if (iDstX < nLBlockX * nBlockXSize ||
672 11041500 : iDstX - nBlockXSize >= nLBlockX * nBlockXSize ||
673 10584800 : iDstY < nLBlockY * nBlockYSize ||
674 10584800 : iDstY - nBlockYSize >= nLBlockY * nBlockYSize)
675 : {
676 738702 : nLBlockX = iDstX / nBlockXSize;
677 738702 : nLBlockY = iDstY / nBlockYSize;
678 :
679 738702 : const bool bJustInitialize =
680 1065990 : nYOff <= nLBlockY * nBlockYSize &&
681 327291 : nYOff + nYSize - nBlockYSize >=
682 327291 : nLBlockY * nBlockYSize &&
683 1116320 : nXOff <= nLBlockX * nBlockXSize &&
684 50325 : nXOff + nXSize - nBlockXSize >= nLBlockX * nBlockXSize;
685 : /*bool bMemZeroBuffer = FALSE;
686 : if( !bJustInitialize &&
687 : nXOff <= nLBlockX * nBlockXSize &&
688 : nYOff <= nLBlockY * nBlockYSize &&
689 : (nXOff + nXSize >= (nLBlockX+1) * nBlockXSize ||
690 : (nXOff + nXSize == GetXSize() &&
691 : (nLBlockX+1) * nBlockXSize > GetXSize())) &&
692 : (nYOff + nYSize >= (nLBlockY+1) * nBlockYSize ||
693 : (nYOff + nYSize == GetYSize() &&
694 : (nLBlockY+1) * nBlockYSize > GetYSize())) )
695 : {
696 : bJustInitialize = TRUE;
697 : bMemZeroBuffer = TRUE;
698 : }*/
699 738702 : if (poBlock != nullptr)
700 572047 : poBlock->DropLock();
701 :
702 738702 : poBlock =
703 738702 : GetLockedBlockRef(nLBlockX, nLBlockY, bJustInitialize);
704 738702 : if (poBlock == nullptr)
705 : {
706 0 : return (CE_Failure);
707 : }
708 :
709 738702 : poBlock->MarkDirty();
710 :
711 738702 : pabyDstBlock = static_cast<GByte *>(poBlock->GetDataRef());
712 : /*if( bMemZeroBuffer )
713 : {
714 : memset(pabyDstBlock, 0,
715 : static_cast<GPtrDiff_t>(nBandDataSize) * nBlockXSize
716 : * nBlockYSize);
717 : }*/
718 : }
719 :
720 : // To make Coverity happy. Should not happen by design.
721 11290800 : if (pabyDstBlock == nullptr)
722 : {
723 0 : CPLAssert(false);
724 : eErr = CE_Failure;
725 : break;
726 : }
727 :
728 : /* --------------------------------------------------------------------
729 : */
730 : /* Copy over this pixel of data. */
731 : /* --------------------------------------------------------------------
732 : */
733 11290800 : GPtrDiff_t iDstOffset =
734 11290800 : (static_cast<GPtrDiff_t>(iDstX) -
735 11290800 : static_cast<GPtrDiff_t>(nLBlockX) * nBlockXSize +
736 11290800 : (static_cast<GPtrDiff_t>(iDstY) -
737 11290800 : static_cast<GPtrDiff_t>(nLBlockY) * nBlockYSize) *
738 11290800 : nBlockXSize) *
739 11290800 : nBandDataSize;
740 :
741 11290800 : if (eDataType == eBufType)
742 : {
743 11287700 : memcpy(pabyDstBlock + iDstOffset,
744 11287700 : static_cast<GByte *>(pData) + iBufOffset,
745 : nBandDataSize);
746 : }
747 : else
748 : {
749 : /* type to type conversion ... ouch, this is expensive way
750 : of handling single words */
751 3096 : GDALCopyWords64(static_cast<GByte *>(pData) + iBufOffset,
752 3096 : eBufType, 0, pabyDstBlock + iDstOffset,
753 : eDataType, 0, 1);
754 : }
755 : }
756 :
757 1093360 : if (psExtraArg->pfnProgress != nullptr &&
758 0 : !psExtraArg->pfnProgress(1.0 * (iDstY - nYOff + 1) / nYSize, "",
759 : psExtraArg->pProgressData))
760 : {
761 0 : eErr = CE_Failure;
762 0 : break;
763 : }
764 : }
765 : }
766 : else
767 : {
768 412797 : if (psExtraArg->eResampleAlg != GRIORA_NearestNeighbour)
769 : {
770 46692 : if ((psExtraArg->eResampleAlg == GRIORA_Cubic ||
771 15098 : psExtraArg->eResampleAlg == GRIORA_CubicSpline ||
772 15045 : psExtraArg->eResampleAlg == GRIORA_Bilinear ||
773 31641 : psExtraArg->eResampleAlg == GRIORA_Lanczos) &&
774 4763 : GetColorTable() != nullptr)
775 : {
776 0 : CPLError(CE_Warning, CPLE_NotSupported,
777 : "Resampling method not supported on paletted band. "
778 : "Falling back to nearest neighbour");
779 : }
780 15800 : else if (psExtraArg->eResampleAlg == GRIORA_Gauss &&
781 3 : GDALDataTypeIsComplex(eDataType))
782 : {
783 0 : CPLError(CE_Warning, CPLE_NotSupported,
784 : "Resampling method not supported on complex data type "
785 : "band. Falling back to nearest neighbour");
786 : }
787 : else
788 : {
789 15797 : return RasterIOResampled(eRWFlag, nXOff, nYOff, nXSize, nYSize,
790 : pData, nBufXSize, nBufYSize, eBufType,
791 15797 : nPixelSpace, nLineSpace, psExtraArg);
792 : }
793 : }
794 :
795 397000 : int nLimitBlockY = 0;
796 397000 : const bool bByteCopy = eDataType == eBufType && nBandDataSize == 1;
797 397000 : int nStartBlockX = -nBlockXSize;
798 397000 : constexpr double EPS = 1e-10;
799 397000 : int nLBlockY = -1;
800 397000 : const double dfSrcXStart = 0.5 * dfSrcXInc + dfXOff + EPS;
801 397000 : const bool bIntegerXFactor =
802 372767 : bUseIntegerRequestCoords &&
803 670836 : static_cast<int>(dfSrcXInc) == dfSrcXInc &&
804 273836 : static_cast<int>(dfSrcXInc) < INT_MAX / nBandDataSize;
805 :
806 : /* --------------------------------------------------------------------
807 : */
808 : /* Read case */
809 : /* Loop over buffer computing source locations. */
810 : /* --------------------------------------------------------------------
811 : */
812 2367100 : for (int iBufYOff = 0; iBufYOff < nBufYSize; iBufYOff++)
813 : {
814 : // Add small epsilon to avoid some numeric precision issues.
815 1970110 : const double dfSrcY = (iBufYOff + 0.5) * dfSrcYInc + dfYOff + EPS;
816 1970110 : const int iSrcY = static_cast<int>(std::min(
817 1970110 : std::max(0.0, dfSrcY), static_cast<double>(nRasterYSize - 1)));
818 :
819 1970110 : GPtrDiff_t iBufOffset = static_cast<GPtrDiff_t>(iBufYOff) *
820 : static_cast<GPtrDiff_t>(nLineSpace);
821 :
822 1970110 : if (iSrcY >= nLimitBlockY)
823 : {
824 438018 : nLBlockY = iSrcY / nBlockYSize;
825 438018 : nLimitBlockY = nLBlockY * nBlockYSize;
826 438018 : if (nLimitBlockY < INT_MAX - nBlockYSize)
827 438018 : nLimitBlockY += nBlockYSize;
828 : else
829 0 : nLimitBlockY = INT_MAX;
830 : // Make sure a new block is loaded.
831 438018 : nStartBlockX = -nBlockXSize;
832 : }
833 1532090 : else if (static_cast<int>(dfSrcXStart) < nStartBlockX)
834 : {
835 : // Make sure a new block is loaded.
836 437363 : nStartBlockX = -nBlockXSize;
837 : }
838 :
839 1970110 : GPtrDiff_t iSrcOffsetCst = (iSrcY - nLBlockY * nBlockYSize) *
840 1970110 : static_cast<GPtrDiff_t>(nBlockXSize);
841 :
842 1970110 : if (bIntegerXFactor)
843 : {
844 695850 : int iSrcX = static_cast<int>(dfSrcXStart);
845 695850 : const int nSrcXInc = static_cast<int>(dfSrcXInc);
846 695850 : GByte *pabyDstData = static_cast<GByte *>(pData) + iBufOffset;
847 695850 : bool bRet = false;
848 695850 : if (bByteCopy)
849 : {
850 585842 : bRet = DownsamplingIntegerXFactor<true, 1>(
851 : this, iSrcX, nSrcXInc, iSrcOffsetCst, pabyDstData,
852 : static_cast<int>(nPixelSpace), nBufXSize, GDT_UInt8,
853 : GDT_UInt8, nStartBlockX, nBlockXSize, poBlock,
854 : nLBlockY);
855 : }
856 110008 : else if (eDataType == eBufType)
857 : {
858 109783 : switch (nBandDataSize)
859 : {
860 109630 : case 2:
861 109630 : bRet = DownsamplingIntegerXFactor<true, 2>(
862 : this, iSrcX, nSrcXInc, iSrcOffsetCst,
863 : pabyDstData, static_cast<int>(nPixelSpace),
864 : nBufXSize, eDataType, eDataType, nStartBlockX,
865 : nBlockXSize, poBlock, nLBlockY);
866 109630 : break;
867 55 : case 4:
868 55 : bRet = DownsamplingIntegerXFactor<true, 4>(
869 : this, iSrcX, nSrcXInc, iSrcOffsetCst,
870 : pabyDstData, static_cast<int>(nPixelSpace),
871 : nBufXSize, eDataType, eDataType, nStartBlockX,
872 : nBlockXSize, poBlock, nLBlockY);
873 55 : break;
874 96 : case 8:
875 96 : bRet = DownsamplingIntegerXFactor<true, 8>(
876 : this, iSrcX, nSrcXInc, iSrcOffsetCst,
877 : pabyDstData, static_cast<int>(nPixelSpace),
878 : nBufXSize, eDataType, eDataType, nStartBlockX,
879 : nBlockXSize, poBlock, nLBlockY);
880 96 : break;
881 2 : case 16:
882 2 : bRet = DownsamplingIntegerXFactor<true, 16>(
883 : this, iSrcX, nSrcXInc, iSrcOffsetCst,
884 : pabyDstData, static_cast<int>(nPixelSpace),
885 : nBufXSize, eDataType, eDataType, nStartBlockX,
886 : nBlockXSize, poBlock, nLBlockY);
887 2 : break;
888 0 : default:
889 0 : CPLAssert(false);
890 : break;
891 : }
892 : }
893 : else
894 : {
895 225 : bRet = DownsamplingIntegerXFactor<false, 0>(
896 : this, iSrcX, nSrcXInc, iSrcOffsetCst, pabyDstData,
897 : static_cast<int>(nPixelSpace), nBufXSize, eDataType,
898 : eBufType, nStartBlockX, nBlockXSize, poBlock, nLBlockY);
899 : }
900 695850 : if (!bRet)
901 1 : eErr = CE_Failure;
902 : }
903 : else
904 : {
905 1274260 : double dfSrcX = dfSrcXStart;
906 503811000 : for (int iBufXOff = 0; iBufXOff < nBufXSize;
907 502537000 : iBufXOff++, dfSrcX += dfSrcXInc)
908 : {
909 : // TODO?: try to avoid the clamping for most iterations
910 : const int iSrcX = static_cast<int>(
911 1005070000 : std::min(std::max(0.0, dfSrcX),
912 502537000 : static_cast<double>(nRasterXSize - 1)));
913 :
914 : /* --------------------------------------------------------------------
915 : */
916 : /* Ensure we have the appropriate block loaded. */
917 : /* --------------------------------------------------------------------
918 : */
919 502537000 : if (iSrcX >= nBlockXSize + nStartBlockX)
920 : {
921 1697820 : const int nLBlockX = iSrcX / nBlockXSize;
922 1697820 : nStartBlockX = nLBlockX * nBlockXSize;
923 :
924 1697820 : if (poBlock != nullptr)
925 1574650 : poBlock->DropLock();
926 :
927 1697820 : poBlock = GetLockedBlockRef(nLBlockX, nLBlockY, FALSE);
928 1697820 : if (poBlock == nullptr)
929 : {
930 9 : eErr = CE_Failure;
931 9 : break;
932 : }
933 :
934 : pabySrcBlock =
935 1697810 : static_cast<GByte *>(poBlock->GetDataRef());
936 : }
937 502537000 : const GPtrDiff_t nDiffX =
938 502537000 : static_cast<GPtrDiff_t>(iSrcX - nStartBlockX);
939 :
940 : /* --------------------------------------------------------------------
941 : */
942 : /* Copy over this pixel of data. */
943 : /* --------------------------------------------------------------------
944 : */
945 :
946 502537000 : if (bByteCopy)
947 : {
948 442592000 : GPtrDiff_t iSrcOffset = nDiffX + iSrcOffsetCst;
949 442592000 : static_cast<GByte *>(pData)[iBufOffset] =
950 442592000 : pabySrcBlock[iSrcOffset];
951 : }
952 59944700 : else if (eDataType == eBufType)
953 : {
954 50322800 : GPtrDiff_t iSrcOffset =
955 50322800 : (nDiffX + iSrcOffsetCst) * nBandDataSize;
956 50322800 : memcpy(static_cast<GByte *>(pData) + iBufOffset,
957 50322800 : pabySrcBlock + iSrcOffset, nBandDataSize);
958 : }
959 : else
960 : {
961 : // Type to type conversion ...
962 9621890 : GPtrDiff_t iSrcOffset =
963 9621890 : (nDiffX + iSrcOffsetCst) * nBandDataSize;
964 9621890 : GDALCopyWords64(pabySrcBlock + iSrcOffset, eDataType, 0,
965 : static_cast<GByte *>(pData) +
966 9621890 : iBufOffset,
967 : eBufType, 0, 1);
968 : }
969 :
970 502537000 : iBufOffset += static_cast<int>(nPixelSpace);
971 : }
972 : }
973 1970110 : if (eErr == CE_Failure)
974 11 : break;
975 :
976 2191530 : if (psExtraArg->pfnProgress != nullptr &&
977 221434 : !psExtraArg->pfnProgress(1.0 * (iBufYOff + 1) / nBufYSize, "",
978 : psExtraArg->pProgressData))
979 : {
980 1 : eErr = CE_Failure;
981 1 : break;
982 : }
983 : }
984 : }
985 :
986 563655 : if (poBlock != nullptr)
987 563645 : poBlock->DropLock();
988 :
989 563655 : return eErr;
990 : }
991 :
992 : /************************************************************************/
993 : /* GDALRasterIOTransformer() */
994 : /************************************************************************/
995 :
996 : struct GDALRasterIOTransformerStruct
997 : {
998 : double dfXOff;
999 : double dfYOff;
1000 : double dfXRatioDstToSrc;
1001 : double dfYRatioDstToSrc;
1002 : };
1003 :
1004 6897 : static int GDALRasterIOTransformer(void *pTransformerArg, int bDstToSrc,
1005 : int nPointCount, double *x, double *y,
1006 : double * /* z */, int *panSuccess)
1007 : {
1008 6897 : GDALRasterIOTransformerStruct *psParams =
1009 : static_cast<GDALRasterIOTransformerStruct *>(pTransformerArg);
1010 6897 : if (bDstToSrc)
1011 : {
1012 311993 : for (int i = 0; i < nPointCount; i++)
1013 : {
1014 305684 : x[i] = x[i] * psParams->dfXRatioDstToSrc + psParams->dfXOff;
1015 305684 : y[i] = y[i] * psParams->dfYRatioDstToSrc + psParams->dfYOff;
1016 305684 : panSuccess[i] = TRUE;
1017 : }
1018 : }
1019 : else
1020 : {
1021 1176 : for (int i = 0; i < nPointCount; i++)
1022 : {
1023 588 : x[i] = (x[i] - psParams->dfXOff) / psParams->dfXRatioDstToSrc;
1024 588 : y[i] = (y[i] - psParams->dfYOff) / psParams->dfYRatioDstToSrc;
1025 588 : panSuccess[i] = TRUE;
1026 : }
1027 : }
1028 6897 : return TRUE;
1029 : }
1030 :
1031 : /************************************************************************/
1032 : /* RasterIOResampled() */
1033 : /************************************************************************/
1034 :
1035 : //! @cond Doxygen_Suppress
1036 15797 : CPLErr GDALRasterBand::RasterIOResampled(
1037 : GDALRWFlag /* eRWFlag */, int nXOff, int nYOff, int nXSize, int nYSize,
1038 : void *pData, int nBufXSize, int nBufYSize, GDALDataType eBufType,
1039 : GSpacing nPixelSpace, GSpacing nLineSpace, GDALRasterIOExtraArg *psExtraArg)
1040 : {
1041 : // Determine if we use warping resampling or overview resampling
1042 : const bool bUseWarp =
1043 15797 : (GDALDataTypeIsComplex(eDataType) &&
1044 15956 : psExtraArg->eResampleAlg != GRIORA_NearestNeighbour &&
1045 159 : psExtraArg->eResampleAlg != GRIORA_Mode);
1046 :
1047 15797 : double dfXOff = nXOff;
1048 15797 : double dfYOff = nYOff;
1049 15797 : double dfXSize = nXSize;
1050 15797 : double dfYSize = nYSize;
1051 15797 : if (psExtraArg->bFloatingPointWindowValidity)
1052 : {
1053 15051 : dfXOff = psExtraArg->dfXOff;
1054 15051 : dfYOff = psExtraArg->dfYOff;
1055 15051 : dfXSize = psExtraArg->dfXSize;
1056 15051 : dfYSize = psExtraArg->dfYSize;
1057 : }
1058 :
1059 15797 : const double dfXRatioDstToSrc = dfXSize / nBufXSize;
1060 15797 : const double dfYRatioDstToSrc = dfYSize / nBufYSize;
1061 :
1062 : // Determine the coordinates in the "virtual" output raster to see
1063 : // if there are not integers, in which case we will use them as a shift
1064 : // so that subwindow extracts give the exact same results as entire raster
1065 : // scaling.
1066 15797 : double dfDestXOff = dfXOff / dfXRatioDstToSrc;
1067 15797 : bool bHasXOffVirtual = false;
1068 15797 : int nDestXOffVirtual = 0;
1069 15797 : if (fabs(dfDestXOff - static_cast<int>(dfDestXOff + 0.5)) < 1e-8)
1070 : {
1071 15469 : bHasXOffVirtual = true;
1072 15469 : dfXOff = nXOff;
1073 15469 : nDestXOffVirtual = static_cast<int>(dfDestXOff + 0.5);
1074 : }
1075 :
1076 15797 : double dfDestYOff = dfYOff / dfYRatioDstToSrc;
1077 15797 : bool bHasYOffVirtual = false;
1078 15797 : int nDestYOffVirtual = 0;
1079 15797 : if (fabs(dfDestYOff - static_cast<int>(dfDestYOff + 0.5)) < 1e-8)
1080 : {
1081 15465 : bHasYOffVirtual = true;
1082 15465 : dfYOff = nYOff;
1083 15465 : nDestYOffVirtual = static_cast<int>(dfDestYOff + 0.5);
1084 : }
1085 :
1086 : // Create a MEM dataset that wraps the output buffer.
1087 : GDALDataset *poMEMDS;
1088 15797 : void *pTempBuffer = nullptr;
1089 15797 : GSpacing nPSMem = nPixelSpace;
1090 15797 : GSpacing nLSMem = nLineSpace;
1091 15797 : void *pDataMem = pData;
1092 15797 : GDALDataType eDTMem = eBufType;
1093 15797 : if (eBufType != eDataType && !GDAL_GET_OPERATE_IN_BUF_TYPE(*psExtraArg))
1094 : {
1095 4 : nPSMem = GDALGetDataTypeSizeBytes(eDataType);
1096 4 : nLSMem = nPSMem * nBufXSize;
1097 : pTempBuffer =
1098 4 : VSI_MALLOC2_VERBOSE(nBufYSize, static_cast<size_t>(nLSMem));
1099 4 : if (pTempBuffer == nullptr)
1100 0 : return CE_Failure;
1101 4 : pDataMem = pTempBuffer;
1102 4 : eDTMem = eDataType;
1103 : }
1104 :
1105 : poMEMDS =
1106 15797 : MEMDataset::Create("", nDestXOffVirtual + nBufXSize,
1107 : nDestYOffVirtual + nBufYSize, 0, eDTMem, nullptr);
1108 15797 : GByte *pabyData = static_cast<GByte *>(pDataMem) -
1109 15797 : nPSMem * nDestXOffVirtual - nLSMem * nDestYOffVirtual;
1110 15797 : GDALRasterBandH hMEMBand = MEMCreateRasterBandEx(
1111 : poMEMDS, 1, pabyData, eDTMem, nPSMem, nLSMem, false);
1112 15797 : poMEMDS->SetBand(1, GDALRasterBand::FromHandle(hMEMBand));
1113 :
1114 15797 : const char *pszNBITS = GetMetadataItem("NBITS", "IMAGE_STRUCTURE");
1115 15797 : const int nNBITS = pszNBITS ? atoi(pszNBITS) : 0;
1116 15797 : if (pszNBITS)
1117 6 : GDALRasterBand::FromHandle(hMEMBand)->SetMetadataItem(
1118 6 : "NBITS", pszNBITS, "IMAGE_STRUCTURE");
1119 :
1120 15797 : CPLErr eErr = CE_None;
1121 :
1122 : // Do the resampling.
1123 15797 : if (bUseWarp)
1124 : {
1125 149 : int bHasNoData = FALSE;
1126 149 : double dfNoDataValue = GetNoDataValue(&bHasNoData);
1127 :
1128 149 : VRTDatasetH hVRTDS = nullptr;
1129 149 : GDALRasterBandH hVRTBand = nullptr;
1130 149 : if (GetDataset() == nullptr)
1131 : {
1132 : /* Create VRT dataset that wraps the whole dataset */
1133 0 : hVRTDS = VRTCreate(nRasterXSize, nRasterYSize);
1134 0 : VRTAddBand(hVRTDS, eDataType, nullptr);
1135 0 : hVRTBand = GDALGetRasterBand(hVRTDS, 1);
1136 0 : VRTAddSimpleSource(hVRTBand, this, 0, 0, nRasterXSize, nRasterYSize,
1137 : 0, 0, nRasterXSize, nRasterYSize, nullptr,
1138 : VRT_NODATA_UNSET);
1139 :
1140 : /* Add a mask band if needed */
1141 0 : if (GetMaskFlags() != GMF_ALL_VALID)
1142 : {
1143 0 : GDALDataset::FromHandle(hVRTDS)->CreateMaskBand(0);
1144 : VRTSourcedRasterBand *poVRTMaskBand =
1145 : reinterpret_cast<VRTSourcedRasterBand *>(
1146 : reinterpret_cast<GDALRasterBand *>(hVRTBand)
1147 0 : ->GetMaskBand());
1148 0 : poVRTMaskBand->AddMaskBandSource(this, 0, 0, nRasterXSize,
1149 0 : nRasterYSize, 0, 0,
1150 0 : nRasterXSize, nRasterYSize);
1151 : }
1152 : }
1153 :
1154 149 : GDALWarpOptions *psWarpOptions = GDALCreateWarpOptions();
1155 149 : switch (psExtraArg->eResampleAlg)
1156 : {
1157 0 : case GRIORA_NearestNeighbour:
1158 0 : psWarpOptions->eResampleAlg = GRA_NearestNeighbour;
1159 0 : break;
1160 147 : case GRIORA_Bilinear:
1161 147 : psWarpOptions->eResampleAlg = GRA_Bilinear;
1162 147 : break;
1163 0 : case GRIORA_Cubic:
1164 0 : psWarpOptions->eResampleAlg = GRA_Cubic;
1165 0 : break;
1166 0 : case GRIORA_CubicSpline:
1167 0 : psWarpOptions->eResampleAlg = GRA_CubicSpline;
1168 0 : break;
1169 0 : case GRIORA_Lanczos:
1170 0 : psWarpOptions->eResampleAlg = GRA_Lanczos;
1171 0 : break;
1172 0 : case GRIORA_Average:
1173 0 : psWarpOptions->eResampleAlg = GRA_Average;
1174 0 : break;
1175 2 : case GRIORA_RMS:
1176 2 : psWarpOptions->eResampleAlg = GRA_RMS;
1177 2 : break;
1178 0 : case GRIORA_Mode:
1179 0 : psWarpOptions->eResampleAlg = GRA_Mode;
1180 0 : break;
1181 0 : default:
1182 0 : CPLAssert(false);
1183 : psWarpOptions->eResampleAlg = GRA_NearestNeighbour;
1184 : break;
1185 : }
1186 149 : psWarpOptions->hSrcDS = hVRTDS ? hVRTDS : GetDataset();
1187 149 : psWarpOptions->hDstDS = poMEMDS;
1188 149 : psWarpOptions->nBandCount = 1;
1189 149 : int nSrcBandNumber = hVRTDS ? 1 : nBand;
1190 149 : int nDstBandNumber = 1;
1191 149 : psWarpOptions->panSrcBands = &nSrcBandNumber;
1192 149 : psWarpOptions->panDstBands = &nDstBandNumber;
1193 298 : psWarpOptions->pfnProgress = psExtraArg->pfnProgress
1194 149 : ? psExtraArg->pfnProgress
1195 : : GDALDummyProgress;
1196 149 : psWarpOptions->pProgressArg = psExtraArg->pProgressData;
1197 149 : psWarpOptions->pfnTransformer = GDALRasterIOTransformer;
1198 149 : if (bHasNoData)
1199 : {
1200 0 : psWarpOptions->papszWarpOptions = CSLSetNameValue(
1201 : psWarpOptions->papszWarpOptions, "INIT_DEST", "NO_DATA");
1202 0 : if (psWarpOptions->padfSrcNoDataReal == nullptr)
1203 : {
1204 0 : psWarpOptions->padfSrcNoDataReal =
1205 0 : static_cast<double *>(CPLMalloc(sizeof(double)));
1206 0 : psWarpOptions->padfSrcNoDataReal[0] = dfNoDataValue;
1207 : }
1208 :
1209 0 : if (psWarpOptions->padfDstNoDataReal == nullptr)
1210 : {
1211 0 : psWarpOptions->padfDstNoDataReal =
1212 0 : static_cast<double *>(CPLMalloc(sizeof(double)));
1213 0 : psWarpOptions->padfDstNoDataReal[0] = dfNoDataValue;
1214 : }
1215 : }
1216 :
1217 : GDALRasterIOTransformerStruct sTransformer;
1218 149 : sTransformer.dfXOff = bHasXOffVirtual ? 0 : dfXOff;
1219 149 : sTransformer.dfYOff = bHasYOffVirtual ? 0 : dfYOff;
1220 149 : sTransformer.dfXRatioDstToSrc = dfXRatioDstToSrc;
1221 149 : sTransformer.dfYRatioDstToSrc = dfYRatioDstToSrc;
1222 149 : psWarpOptions->pTransformerArg = &sTransformer;
1223 :
1224 : GDALWarpOperationH hWarpOperation =
1225 149 : GDALCreateWarpOperation(psWarpOptions);
1226 149 : eErr = GDALChunkAndWarpImage(hWarpOperation, nDestXOffVirtual,
1227 : nDestYOffVirtual, nBufXSize, nBufYSize);
1228 149 : GDALDestroyWarpOperation(hWarpOperation);
1229 :
1230 149 : psWarpOptions->panSrcBands = nullptr;
1231 149 : psWarpOptions->panDstBands = nullptr;
1232 149 : GDALDestroyWarpOptions(psWarpOptions);
1233 :
1234 149 : if (hVRTDS)
1235 0 : GDALClose(hVRTDS);
1236 : }
1237 : else
1238 : {
1239 : const char *pszResampling =
1240 15648 : GDALRasterIOGetResampleAlg(psExtraArg->eResampleAlg);
1241 15648 : int nKernelRadius = 0;
1242 : GDALResampleFunction pfnResampleFunc =
1243 15648 : GDALGetResampleFunction(pszResampling, &nKernelRadius);
1244 15648 : CPLAssert(pfnResampleFunc);
1245 : GDALDataType eWrkDataType =
1246 15648 : GDALGetOvrWorkDataType(pszResampling, eDataType);
1247 15648 : int nHasNoData = 0;
1248 15648 : double dfNoDataValue = GetNoDataValue(&nHasNoData);
1249 15648 : const bool bHasNoData = CPL_TO_BOOL(nHasNoData);
1250 15648 : if (!bHasNoData)
1251 15516 : dfNoDataValue = 0.0;
1252 :
1253 15648 : int nDstBlockXSize = nBufXSize;
1254 15648 : int nDstBlockYSize = nBufYSize;
1255 15648 : int nFullResXChunk = 0;
1256 15648 : int nFullResYChunk = 0;
1257 : while (true)
1258 : {
1259 15659 : nFullResXChunk = static_cast<int>(std::min<double>(
1260 15659 : 3 + nDstBlockXSize * dfXRatioDstToSrc, nRasterXSize));
1261 15659 : nFullResYChunk = static_cast<int>(std::min<double>(
1262 15659 : 3 + nDstBlockYSize * dfYRatioDstToSrc, nRasterYSize));
1263 15659 : if ((nDstBlockXSize == 1 && nDstBlockYSize == 1) ||
1264 15601 : (static_cast<GIntBig>(nFullResXChunk) * nFullResYChunk <=
1265 : 1024 * 1024))
1266 : break;
1267 : // When operating on the full width of a raster whose block width is
1268 : // the raster width, prefer doing chunks in height.
1269 11 : if (nFullResXChunk >= nXSize && nXSize == nBlockXSize &&
1270 : nDstBlockYSize > 1)
1271 0 : nDstBlockYSize /= 2;
1272 : /* Otherwise cut the maximal dimension */
1273 11 : else if (nDstBlockXSize > 1 &&
1274 0 : (nFullResXChunk > nFullResYChunk || nDstBlockYSize == 1))
1275 11 : nDstBlockXSize /= 2;
1276 : else
1277 0 : nDstBlockYSize /= 2;
1278 : }
1279 :
1280 : const int nOvrXFactor =
1281 15648 : std::max(1, static_cast<int>(0.5 + dfXRatioDstToSrc));
1282 : const int nOvrYFactor =
1283 15648 : std::max(1, static_cast<int>(0.5 + dfYRatioDstToSrc));
1284 : const int nFullResXSizeQueried = static_cast<int>(
1285 31296 : std::min<int64_t>(nFullResXChunk + static_cast<int64_t>(2) *
1286 15648 : nKernelRadius * nOvrXFactor,
1287 15648 : nRasterXSize));
1288 : const int nFullResYSizeQueried = static_cast<int>(
1289 31296 : std::min<int64_t>(nFullResYChunk + static_cast<int64_t>(2) *
1290 15648 : nKernelRadius * nOvrYFactor,
1291 15648 : nRasterYSize));
1292 :
1293 : void *pChunk =
1294 15648 : VSI_MALLOC3_VERBOSE(GDALGetDataTypeSizeBytes(eWrkDataType),
1295 : nFullResXSizeQueried, nFullResYSizeQueried);
1296 15648 : GByte *pabyChunkNoDataMask = nullptr;
1297 :
1298 15648 : GDALRasterBand *poMaskBand = GetMaskBand();
1299 15648 : int l_nMaskFlags = GetMaskFlags();
1300 :
1301 15648 : bool bUseNoDataMask = ((l_nMaskFlags & GMF_ALL_VALID) == 0);
1302 15648 : if (bUseNoDataMask)
1303 : {
1304 7525 : pabyChunkNoDataMask = static_cast<GByte *>(VSI_MALLOC2_VERBOSE(
1305 : nFullResXSizeQueried, nFullResYSizeQueried));
1306 : }
1307 15648 : if (pChunk == nullptr ||
1308 7525 : (bUseNoDataMask && pabyChunkNoDataMask == nullptr))
1309 : {
1310 0 : GDALClose(poMEMDS);
1311 0 : CPLFree(pChunk);
1312 0 : CPLFree(pabyChunkNoDataMask);
1313 0 : VSIFree(pTempBuffer);
1314 0 : return CE_Failure;
1315 : }
1316 :
1317 : const int64_t nTotalBlocks =
1318 15648 : static_cast<int64_t>(cpl::div_round_up(nBufXSize, nDstBlockXSize)) *
1319 15648 : cpl::div_round_up(nBufYSize, nDstBlockYSize);
1320 15648 : int64_t nBlocksDone = 0;
1321 :
1322 31296 : for (int nDstYOff = 0; nDstYOff < nBufYSize && eErr == CE_None;
1323 15648 : nDstYOff += nDstBlockYSize)
1324 : {
1325 : int nDstYCount;
1326 15648 : if (nDstYOff + nDstBlockYSize <= nBufYSize)
1327 15648 : nDstYCount = nDstBlockYSize;
1328 : else
1329 0 : nDstYCount = nBufYSize - nDstYOff;
1330 :
1331 15648 : int nChunkYOff =
1332 15648 : nYOff + static_cast<int>(nDstYOff * dfYRatioDstToSrc);
1333 15648 : int nChunkYOff2 = nYOff + 1 +
1334 15648 : static_cast<int>(ceil((nDstYOff + nDstYCount) *
1335 : dfYRatioDstToSrc));
1336 15648 : if (nChunkYOff2 > nRasterYSize)
1337 789 : nChunkYOff2 = nRasterYSize;
1338 15648 : int nYCount = nChunkYOff2 - nChunkYOff;
1339 15648 : CPLAssert(nYCount <= nFullResYChunk);
1340 :
1341 15648 : int nChunkYOffQueried = nChunkYOff - nKernelRadius * nOvrYFactor;
1342 15648 : int nChunkYSizeQueried = nYCount + 2 * nKernelRadius * nOvrYFactor;
1343 15648 : if (nChunkYOffQueried < 0)
1344 : {
1345 498 : nChunkYSizeQueried += nChunkYOffQueried;
1346 498 : nChunkYOffQueried = 0;
1347 : }
1348 15648 : if (nChunkYSizeQueried + nChunkYOffQueried > nRasterYSize)
1349 607 : nChunkYSizeQueried = nRasterYSize - nChunkYOffQueried;
1350 15648 : CPLAssert(nChunkYSizeQueried <= nFullResYSizeQueried);
1351 :
1352 15648 : int nDstXOff = 0;
1353 31296 : for (nDstXOff = 0; nDstXOff < nBufXSize && eErr == CE_None;
1354 15648 : nDstXOff += nDstBlockXSize)
1355 : {
1356 15648 : int nDstXCount = 0;
1357 15648 : if (nDstXOff + nDstBlockXSize <= nBufXSize)
1358 15648 : nDstXCount = nDstBlockXSize;
1359 : else
1360 0 : nDstXCount = nBufXSize - nDstXOff;
1361 :
1362 15648 : int nChunkXOff =
1363 15648 : nXOff + static_cast<int>(nDstXOff * dfXRatioDstToSrc);
1364 15648 : int nChunkXOff2 =
1365 15648 : nXOff + 1 +
1366 15648 : static_cast<int>(
1367 15648 : ceil((nDstXOff + nDstXCount) * dfXRatioDstToSrc));
1368 15648 : if (nChunkXOff2 > nRasterXSize)
1369 9827 : nChunkXOff2 = nRasterXSize;
1370 15648 : int nXCount = nChunkXOff2 - nChunkXOff;
1371 15648 : CPLAssert(nXCount <= nFullResXChunk);
1372 :
1373 15648 : int nChunkXOffQueried =
1374 15648 : nChunkXOff - nKernelRadius * nOvrXFactor;
1375 15648 : int nChunkXSizeQueried =
1376 15648 : nXCount + 2 * nKernelRadius * nOvrXFactor;
1377 15648 : if (nChunkXOffQueried < 0)
1378 : {
1379 3310 : nChunkXSizeQueried += nChunkXOffQueried;
1380 3310 : nChunkXOffQueried = 0;
1381 : }
1382 15648 : if (nChunkXSizeQueried + nChunkXOffQueried > nRasterXSize)
1383 3806 : nChunkXSizeQueried = nRasterXSize - nChunkXOffQueried;
1384 15648 : CPLAssert(nChunkXSizeQueried <= nFullResXSizeQueried);
1385 :
1386 : // Read the source buffers.
1387 15648 : eErr = RasterIO(GF_Read, nChunkXOffQueried, nChunkYOffQueried,
1388 : nChunkXSizeQueried, nChunkYSizeQueried, pChunk,
1389 : nChunkXSizeQueried, nChunkYSizeQueried,
1390 : eWrkDataType, 0, 0, nullptr);
1391 :
1392 15648 : bool bSkipResample = false;
1393 15648 : bool bNoDataMaskFullyOpaque = false;
1394 15648 : if (eErr == CE_None && bUseNoDataMask)
1395 : {
1396 7525 : eErr = poMaskBand->RasterIO(
1397 : GF_Read, nChunkXOffQueried, nChunkYOffQueried,
1398 : nChunkXSizeQueried, nChunkYSizeQueried,
1399 : pabyChunkNoDataMask, nChunkXSizeQueried,
1400 : nChunkYSizeQueried, GDT_UInt8, 0, 0, nullptr);
1401 :
1402 : /* Optimizations if mask if fully opaque or transparent */
1403 7525 : int nPixels = nChunkXSizeQueried * nChunkYSizeQueried;
1404 7525 : GByte bVal = pabyChunkNoDataMask[0];
1405 7525 : int i = 1;
1406 15237000 : for (; i < nPixels; i++)
1407 : {
1408 15230700 : if (pabyChunkNoDataMask[i] != bVal)
1409 1168 : break;
1410 : }
1411 7525 : if (i == nPixels)
1412 : {
1413 6357 : if (bVal == 0)
1414 : {
1415 12094 : for (int j = 0; j < nDstYCount; j++)
1416 : {
1417 6377 : GDALCopyWords64(&dfNoDataValue, GDT_Float64, 0,
1418 : static_cast<GByte *>(pDataMem) +
1419 6377 : nLSMem * (j + nDstYOff) +
1420 6377 : nDstXOff * nPSMem,
1421 : eDTMem,
1422 : static_cast<int>(nPSMem),
1423 : nDstXCount);
1424 : }
1425 5717 : bSkipResample = true;
1426 : }
1427 : else
1428 : {
1429 640 : bNoDataMaskFullyOpaque = true;
1430 : }
1431 : }
1432 : }
1433 :
1434 15648 : if (!bSkipResample && eErr == CE_None)
1435 : {
1436 9928 : const bool bPropagateNoData = false;
1437 9928 : void *pDstBuffer = nullptr;
1438 9928 : GDALDataType eDstBufferDataType = GDT_Unknown;
1439 : GDALRasterBand *poMEMBand =
1440 9928 : GDALRasterBand::FromHandle(hMEMBand);
1441 9928 : GDALOverviewResampleArgs args;
1442 9928 : args.eSrcDataType = eDataType;
1443 9928 : args.eOvrDataType = poMEMBand->GetRasterDataType();
1444 9928 : args.nOvrXSize = poMEMBand->GetXSize();
1445 9928 : args.nOvrYSize = poMEMBand->GetYSize();
1446 9928 : args.nOvrNBITS = nNBITS;
1447 9928 : args.dfXRatioDstToSrc = dfXRatioDstToSrc;
1448 9928 : args.dfYRatioDstToSrc = dfYRatioDstToSrc;
1449 9928 : args.dfSrcXDelta =
1450 9928 : dfXOff - nXOff; /* == 0 if bHasXOffVirtual */
1451 9928 : args.dfSrcYDelta =
1452 9928 : dfYOff - nYOff; /* == 0 if bHasYOffVirtual */
1453 9928 : args.eWrkDataType = eWrkDataType;
1454 9928 : args.pabyChunkNodataMask =
1455 9928 : bNoDataMaskFullyOpaque ? nullptr : pabyChunkNoDataMask;
1456 9928 : args.nChunkXOff =
1457 9928 : nChunkXOffQueried - (bHasXOffVirtual ? 0 : nXOff);
1458 9928 : args.nChunkXSize = nChunkXSizeQueried;
1459 9928 : args.nChunkYOff =
1460 9928 : nChunkYOffQueried - (bHasYOffVirtual ? 0 : nYOff);
1461 9928 : args.nChunkYSize = nChunkYSizeQueried;
1462 9928 : args.nDstXOff = nDstXOff + nDestXOffVirtual;
1463 9928 : args.nDstXOff2 = nDstXOff + nDestXOffVirtual + nDstXCount;
1464 9928 : args.nDstYOff = nDstYOff + nDestYOffVirtual;
1465 9928 : args.nDstYOff2 = nDstYOff + nDestYOffVirtual + nDstYCount;
1466 9928 : args.pszResampling = pszResampling;
1467 9928 : args.bHasNoData = bHasNoData;
1468 9928 : args.dfNoDataValue = dfNoDataValue;
1469 9928 : args.poColorTable = GetColorTable();
1470 9928 : args.bPropagateNoData = bPropagateNoData;
1471 9928 : eErr = pfnResampleFunc(args, pChunk, &pDstBuffer,
1472 : &eDstBufferDataType);
1473 9928 : if (eErr == CE_None)
1474 : {
1475 9928 : eErr = poMEMBand->RasterIO(
1476 : GF_Write, nDstXOff + nDestXOffVirtual,
1477 : nDstYOff + nDestYOffVirtual, nDstXCount, nDstYCount,
1478 : pDstBuffer, nDstXCount, nDstYCount,
1479 : eDstBufferDataType, 0, 0, nullptr);
1480 : }
1481 9928 : CPLFree(pDstBuffer);
1482 : }
1483 :
1484 15648 : nBlocksDone++;
1485 28106 : if (eErr == CE_None && psExtraArg->pfnProgress != nullptr &&
1486 12458 : !psExtraArg->pfnProgress(
1487 12458 : static_cast<double>(nBlocksDone) /
1488 12458 : static_cast<double>(nTotalBlocks),
1489 : "", psExtraArg->pProgressData))
1490 : {
1491 1 : eErr = CE_Failure;
1492 : }
1493 : }
1494 : }
1495 :
1496 15648 : CPLFree(pChunk);
1497 15648 : CPLFree(pabyChunkNoDataMask);
1498 : }
1499 :
1500 15797 : if (pTempBuffer)
1501 : {
1502 4 : CPL_IGNORE_RET_VAL(poMEMDS->GetRasterBand(1)->RasterIO(
1503 : GF_Read, nDestXOffVirtual, nDestYOffVirtual, nBufXSize, nBufYSize,
1504 : pData, nBufXSize, nBufYSize, eBufType, nPixelSpace, nLineSpace,
1505 : nullptr));
1506 : }
1507 15797 : GDALClose(poMEMDS);
1508 15797 : VSIFree(pTempBuffer);
1509 :
1510 15797 : return eErr;
1511 : }
1512 :
1513 : /************************************************************************/
1514 : /* RasterIOResampled() */
1515 : /************************************************************************/
1516 :
1517 2431 : CPLErr GDALDataset::RasterIOResampled(
1518 : GDALRWFlag /* eRWFlag */, int nXOff, int nYOff, int nXSize, int nYSize,
1519 : void *pData, int nBufXSize, int nBufYSize, GDALDataType eBufType,
1520 : int nBandCount, const int *panBandMap, GSpacing nPixelSpace,
1521 : GSpacing nLineSpace, GSpacing nBandSpace, GDALRasterIOExtraArg *psExtraArg)
1522 :
1523 : {
1524 : #if 0
1525 : // Determine if we use warping resampling or overview resampling
1526 : bool bUseWarp = false;
1527 : if( GDALDataTypeIsComplex( eDataType ) )
1528 : bUseWarp = true;
1529 : #endif
1530 :
1531 2431 : double dfXOff = nXOff;
1532 2431 : double dfYOff = nYOff;
1533 2431 : double dfXSize = nXSize;
1534 2431 : double dfYSize = nYSize;
1535 2431 : if (psExtraArg->bFloatingPointWindowValidity)
1536 : {
1537 2304 : dfXOff = psExtraArg->dfXOff;
1538 2304 : dfYOff = psExtraArg->dfYOff;
1539 2304 : dfXSize = psExtraArg->dfXSize;
1540 2304 : dfYSize = psExtraArg->dfYSize;
1541 : }
1542 :
1543 2431 : const double dfXRatioDstToSrc = dfXSize / nBufXSize;
1544 2431 : const double dfYRatioDstToSrc = dfYSize / nBufYSize;
1545 :
1546 : // Determine the coordinates in the "virtual" output raster to see
1547 : // if there are not integers, in which case we will use them as a shift
1548 : // so that subwindow extracts give the exact same results as entire raster
1549 : // scaling.
1550 2431 : double dfDestXOff = dfXOff / dfXRatioDstToSrc;
1551 2431 : bool bHasXOffVirtual = false;
1552 2431 : int nDestXOffVirtual = 0;
1553 2431 : if (fabs(dfDestXOff - static_cast<int>(dfDestXOff + 0.5)) < 1e-8)
1554 : {
1555 2306 : bHasXOffVirtual = true;
1556 2306 : dfXOff = nXOff;
1557 2306 : nDestXOffVirtual = static_cast<int>(dfDestXOff + 0.5);
1558 : }
1559 :
1560 2431 : double dfDestYOff = dfYOff / dfYRatioDstToSrc;
1561 2431 : bool bHasYOffVirtual = false;
1562 2431 : int nDestYOffVirtual = 0;
1563 2431 : if (fabs(dfDestYOff - static_cast<int>(dfDestYOff + 0.5)) < 1e-8)
1564 : {
1565 2266 : bHasYOffVirtual = true;
1566 2266 : dfYOff = nYOff;
1567 2266 : nDestYOffVirtual = static_cast<int>(dfDestYOff + 0.5);
1568 : }
1569 :
1570 : // Create a MEM dataset that wraps the output buffer.
1571 2431 : std::unique_ptr<void, VSIFreeReleaser> pTempBuffer;
1572 2431 : GSpacing nPSMem = nPixelSpace;
1573 2431 : GSpacing nLSMem = nLineSpace;
1574 2431 : GSpacing nBandSpaceMEM = nBandSpace;
1575 2431 : void *pDataMem = pData;
1576 2431 : GDALDataType eDTMem = eBufType;
1577 2431 : GDALRasterBand *poFirstSrcBand = GetRasterBand(panBandMap[0]);
1578 2431 : const GDALDataType eDataType = poFirstSrcBand->GetRasterDataType();
1579 2431 : if (eBufType != eDataType && !GDAL_GET_OPERATE_IN_BUF_TYPE(*psExtraArg))
1580 : {
1581 2 : nPSMem = GDALGetDataTypeSizeBytes(eDataType);
1582 2 : nLSMem = nPSMem * nBufXSize;
1583 2 : nBandSpaceMEM = nLSMem * nBandCount;
1584 2 : pTempBuffer.reset(VSI_MALLOC3_VERBOSE(nBandCount, nBufYSize,
1585 : static_cast<size_t>(nLSMem)));
1586 2 : if (pTempBuffer == nullptr)
1587 0 : return CE_Failure;
1588 2 : pDataMem = pTempBuffer.get();
1589 2 : eDTMem = eDataType;
1590 : }
1591 :
1592 : auto poMEMDS = std::unique_ptr<GDALDataset>(
1593 2431 : MEMDataset::Create("", nDestXOffVirtual + nBufXSize,
1594 4862 : nDestYOffVirtual + nBufYSize, 0, eDTMem, nullptr));
1595 : #ifdef GDAL_ENABLE_RESAMPLING_MULTIBAND
1596 : std::vector<GDALRasterBand *> apoDstBands(nBandCount);
1597 : #endif
1598 2431 : int nNBITS = 0;
1599 9052 : for (int i = 0; i < nBandCount; i++)
1600 : {
1601 6621 : GByte *const pBandData = static_cast<GByte *>(pDataMem) -
1602 6621 : nPSMem * nDestXOffVirtual -
1603 6621 : nLSMem * nDestYOffVirtual + nBandSpaceMEM * i;
1604 6621 : auto poMEMBand = GDALRasterBand::FromHandle(MEMCreateRasterBandEx(
1605 : poMEMDS.get(), i + 1, pBandData, eDTMem, nPSMem, nLSMem, false));
1606 6621 : poMEMDS->SetBand(i + 1, poMEMBand);
1607 :
1608 6621 : GDALRasterBand *poSrcBand = GetRasterBand(panBandMap[i]);
1609 : #ifdef GDAL_ENABLE_RESAMPLING_MULTIBAND
1610 : apoDstBands[i] = poMEMBand;
1611 : #endif
1612 : const char *pszNBITS =
1613 6621 : poSrcBand->GetMetadataItem("NBITS", "IMAGE_STRUCTURE");
1614 6621 : if (pszNBITS)
1615 : {
1616 0 : nNBITS = atoi(pszNBITS);
1617 0 : poMEMDS->GetRasterBand(i + 1)->SetMetadataItem("NBITS", pszNBITS,
1618 0 : "IMAGE_STRUCTURE");
1619 : }
1620 : }
1621 :
1622 2431 : CPLErr eErr = CE_None;
1623 :
1624 : // TODO(schwehr): Why disabled? Why not just delete?
1625 : // Looks like this code was initially added as disable by copying
1626 : // from RasterIO here:
1627 : // https://trac.osgeo.org/gdal/changeset/29572
1628 : #if 0
1629 : // Do the resampling.
1630 : if( bUseWarp )
1631 : {
1632 : VRTDatasetH hVRTDS = nullptr;
1633 : GDALRasterBandH hVRTBand = nullptr;
1634 : if( GetDataset() == nullptr )
1635 : {
1636 : /* Create VRT dataset that wraps the whole dataset */
1637 : hVRTDS = VRTCreate(nRasterXSize, nRasterYSize);
1638 : VRTAddBand( hVRTDS, eDataType, nullptr );
1639 : hVRTBand = GDALGetRasterBand(hVRTDS, 1);
1640 : VRTAddSimpleSource( (VRTSourcedRasterBandH)hVRTBand,
1641 : (GDALRasterBandH)this,
1642 : 0, 0,
1643 : nRasterXSize, nRasterYSize,
1644 : 0, 0,
1645 : nRasterXSize, nRasterYSize,
1646 : nullptr, VRT_NODATA_UNSET );
1647 :
1648 : /* Add a mask band if needed */
1649 : if( GetMaskFlags() != GMF_ALL_VALID )
1650 : {
1651 : ((GDALDataset*)hVRTDS)->CreateMaskBand(0);
1652 : VRTSourcedRasterBand* poVRTMaskBand =
1653 : (VRTSourcedRasterBand*)(((GDALRasterBand*)hVRTBand)->GetMaskBand());
1654 : poVRTMaskBand->
1655 : AddMaskBandSource( this,
1656 : 0, 0,
1657 : nRasterXSize, nRasterYSize,
1658 : 0, 0,
1659 : nRasterXSize, nRasterYSize);
1660 : }
1661 : }
1662 :
1663 : GDALWarpOptions* psWarpOptions = GDALCreateWarpOptions();
1664 : psWarpOptions->eResampleAlg = (GDALResampleAlg)psExtraArg->eResampleAlg;
1665 : psWarpOptions->hSrcDS = (GDALDatasetH) (hVRTDS ? hVRTDS : GetDataset());
1666 : psWarpOptions->hDstDS = (GDALDatasetH) poMEMDS;
1667 : psWarpOptions->nBandCount = 1;
1668 : int nSrcBandNumber = (hVRTDS ? 1 : nBand);
1669 : int nDstBandNumber = 1;
1670 : psWarpOptions->panSrcBands = &nSrcBandNumber;
1671 : psWarpOptions->panDstBands = &nDstBandNumber;
1672 : psWarpOptions->pfnProgress = psExtraArg->pfnProgress ?
1673 : psExtraArg->pfnProgress : GDALDummyProgress;
1674 : psWarpOptions->pProgressArg = psExtraArg->pProgressData;
1675 : psWarpOptions->pfnTransformer = GDALRasterIOTransformer;
1676 : GDALRasterIOTransformerStruct sTransformer;
1677 : sTransformer.dfXOff = bHasXOffVirtual ? 0 : dfXOff;
1678 : sTransformer.dfYOff = bHasYOffVirtual ? 0 : dfYOff;
1679 : sTransformer.dfXRatioDstToSrc = dfXRatioDstToSrc;
1680 : sTransformer.dfYRatioDstToSrc = dfYRatioDstToSrc;
1681 : psWarpOptions->pTransformerArg = &sTransformer;
1682 :
1683 : GDALWarpOperationH hWarpOperation = GDALCreateWarpOperation(psWarpOptions);
1684 : eErr = GDALChunkAndWarpImage( hWarpOperation,
1685 : nDestXOffVirtual, nDestYOffVirtual,
1686 : nBufXSize, nBufYSize );
1687 : GDALDestroyWarpOperation( hWarpOperation );
1688 :
1689 : psWarpOptions->panSrcBands = nullptr;
1690 : psWarpOptions->panDstBands = nullptr;
1691 : GDALDestroyWarpOptions( psWarpOptions );
1692 :
1693 : if( hVRTDS )
1694 : GDALClose(hVRTDS);
1695 : }
1696 : else
1697 : #endif
1698 : {
1699 : const char *pszResampling =
1700 2431 : GDALRasterIOGetResampleAlg(psExtraArg->eResampleAlg);
1701 :
1702 : int nBlockXSize, nBlockYSize;
1703 2431 : poFirstSrcBand->GetBlockSize(&nBlockXSize, &nBlockYSize);
1704 :
1705 : int nKernelRadius;
1706 : GDALResampleFunction pfnResampleFunc =
1707 2431 : GDALGetResampleFunction(pszResampling, &nKernelRadius);
1708 2431 : CPLAssert(pfnResampleFunc);
1709 : #ifdef GDAL_ENABLE_RESAMPLING_MULTIBAND
1710 : GDALResampleFunctionMultiBands pfnResampleFuncMultiBands =
1711 : GDALGetResampleFunctionMultiBands(pszResampling, &nKernelRadius);
1712 : #endif
1713 : GDALDataType eWrkDataType =
1714 2431 : GDALGetOvrWorkDataType(pszResampling, eDataType);
1715 :
1716 2431 : int nDstBlockXSize = nBufXSize;
1717 2431 : int nDstBlockYSize = nBufYSize;
1718 : int nFullResXChunk, nFullResYChunk;
1719 : while (true)
1720 : {
1721 2431 : nFullResXChunk = static_cast<int>(std::min<double>(
1722 2431 : 3 + nDstBlockXSize * dfXRatioDstToSrc, nRasterXSize));
1723 2431 : nFullResYChunk = static_cast<int>(std::min<double>(
1724 2431 : 3 + nDstBlockYSize * dfYRatioDstToSrc, nRasterYSize));
1725 2431 : if ((nDstBlockXSize == 1 && nDstBlockYSize == 1) ||
1726 2429 : (static_cast<GIntBig>(nFullResXChunk) * nFullResYChunk <=
1727 : 1024 * 1024))
1728 : break;
1729 : // When operating on the full width of a raster whose block width is
1730 : // the raster width, prefer doing chunks in height.
1731 0 : if (nFullResXChunk >= nXSize && nXSize == nBlockXSize &&
1732 : nDstBlockYSize > 1)
1733 0 : nDstBlockYSize /= 2;
1734 : /* Otherwise cut the maximal dimension */
1735 0 : else if (nDstBlockXSize > 1 &&
1736 0 : (nFullResXChunk > nFullResYChunk || nDstBlockYSize == 1))
1737 0 : nDstBlockXSize /= 2;
1738 : else
1739 0 : nDstBlockYSize /= 2;
1740 : }
1741 :
1742 : const int nOvrFactor =
1743 7293 : std::max(1, std::max(static_cast<int>(0.5 + dfXRatioDstToSrc),
1744 2431 : static_cast<int>(0.5 + dfYRatioDstToSrc)));
1745 : const int nFullResXSizeQueried = static_cast<int>(
1746 4862 : std::min<int64_t>(nFullResXChunk + static_cast<int64_t>(2) *
1747 2431 : nKernelRadius * nOvrFactor,
1748 2431 : nRasterXSize));
1749 : const int nFullResYSizeQueried = static_cast<int>(
1750 4862 : std::min<int64_t>(nFullResYChunk + static_cast<int64_t>(2) *
1751 2431 : nKernelRadius * nOvrFactor,
1752 2431 : nRasterYSize));
1753 :
1754 2431 : void *pChunk = VSI_MALLOC3_VERBOSE(
1755 : cpl::fits_on<int>(GDALGetDataTypeSizeBytes(eWrkDataType) *
1756 : nBandCount),
1757 : nFullResXSizeQueried, nFullResYSizeQueried);
1758 2431 : GByte *pabyChunkNoDataMask = nullptr;
1759 :
1760 2431 : GDALRasterBand *poMaskBand = poFirstSrcBand->GetMaskBand();
1761 2431 : int nMaskFlags = poFirstSrcBand->GetMaskFlags();
1762 :
1763 2431 : bool bUseNoDataMask = ((nMaskFlags & GMF_ALL_VALID) == 0);
1764 2431 : if (bUseNoDataMask)
1765 : {
1766 2156 : pabyChunkNoDataMask = static_cast<GByte *>(VSI_MALLOC2_VERBOSE(
1767 : nFullResXSizeQueried, nFullResYSizeQueried));
1768 : }
1769 2431 : if (pChunk == nullptr ||
1770 2156 : (bUseNoDataMask && pabyChunkNoDataMask == nullptr))
1771 : {
1772 0 : CPLFree(pChunk);
1773 0 : CPLFree(pabyChunkNoDataMask);
1774 0 : return CE_Failure;
1775 : }
1776 :
1777 : const int64_t nTotalBlocks =
1778 2431 : static_cast<int64_t>(cpl::div_round_up(nBufXSize, nDstBlockXSize)) *
1779 2431 : cpl::div_round_up(nBufYSize, nDstBlockYSize);
1780 2431 : int64_t nBlocksDone = 0;
1781 :
1782 4862 : for (int nDstYOff = 0; nDstYOff < nBufYSize && eErr == CE_None;
1783 2431 : nDstYOff += nDstBlockYSize)
1784 : {
1785 : int nDstYCount;
1786 2431 : if (nDstYOff + nDstBlockYSize <= nBufYSize)
1787 2431 : nDstYCount = nDstBlockYSize;
1788 : else
1789 0 : nDstYCount = nBufYSize - nDstYOff;
1790 :
1791 2431 : int nChunkYOff =
1792 2431 : nYOff + static_cast<int>(nDstYOff * dfYRatioDstToSrc);
1793 2431 : int nChunkYOff2 = nYOff + 1 +
1794 2431 : static_cast<int>(ceil((nDstYOff + nDstYCount) *
1795 : dfYRatioDstToSrc));
1796 2431 : if (nChunkYOff2 > nRasterYSize)
1797 146 : nChunkYOff2 = nRasterYSize;
1798 2431 : int nYCount = nChunkYOff2 - nChunkYOff;
1799 2431 : CPLAssert(nYCount <= nFullResYChunk);
1800 :
1801 2431 : int nChunkYOffQueried = nChunkYOff - nKernelRadius * nOvrFactor;
1802 2431 : int nChunkYSizeQueried = nYCount + 2 * nKernelRadius * nOvrFactor;
1803 2431 : if (nChunkYOffQueried < 0)
1804 : {
1805 149 : nChunkYSizeQueried += nChunkYOffQueried;
1806 149 : nChunkYOffQueried = 0;
1807 : }
1808 2431 : if (nChunkYSizeQueried + nChunkYOffQueried > nRasterYSize)
1809 170 : nChunkYSizeQueried = nRasterYSize - nChunkYOffQueried;
1810 2431 : CPLAssert(nChunkYSizeQueried <= nFullResYSizeQueried);
1811 :
1812 : int nDstXOff;
1813 4862 : for (nDstXOff = 0; nDstXOff < nBufXSize && eErr == CE_None;
1814 2431 : nDstXOff += nDstBlockXSize)
1815 : {
1816 : int nDstXCount;
1817 2431 : if (nDstXOff + nDstBlockXSize <= nBufXSize)
1818 2431 : nDstXCount = nDstBlockXSize;
1819 : else
1820 0 : nDstXCount = nBufXSize - nDstXOff;
1821 :
1822 2431 : int nChunkXOff =
1823 2431 : nXOff + static_cast<int>(nDstXOff * dfXRatioDstToSrc);
1824 2431 : int nChunkXOff2 =
1825 2431 : nXOff + 1 +
1826 2431 : static_cast<int>(
1827 2431 : ceil((nDstXOff + nDstXCount) * dfXRatioDstToSrc));
1828 2431 : if (nChunkXOff2 > nRasterXSize)
1829 1672 : nChunkXOff2 = nRasterXSize;
1830 2431 : int nXCount = nChunkXOff2 - nChunkXOff;
1831 2431 : CPLAssert(nXCount <= nFullResXChunk);
1832 :
1833 2431 : int nChunkXOffQueried = nChunkXOff - nKernelRadius * nOvrFactor;
1834 2431 : int nChunkXSizeQueried =
1835 2431 : nXCount + 2 * nKernelRadius * nOvrFactor;
1836 2431 : if (nChunkXOffQueried < 0)
1837 : {
1838 1162 : nChunkXSizeQueried += nChunkXOffQueried;
1839 1162 : nChunkXOffQueried = 0;
1840 : }
1841 2431 : if (nChunkXSizeQueried + nChunkXOffQueried > nRasterXSize)
1842 1680 : nChunkXSizeQueried = nRasterXSize - nChunkXOffQueried;
1843 2431 : CPLAssert(nChunkXSizeQueried <= nFullResXSizeQueried);
1844 :
1845 2431 : bool bSkipResample = false;
1846 2431 : bool bNoDataMaskFullyOpaque = false;
1847 2431 : if (eErr == CE_None && bUseNoDataMask)
1848 : {
1849 2156 : eErr = poMaskBand->RasterIO(
1850 : GF_Read, nChunkXOffQueried, nChunkYOffQueried,
1851 : nChunkXSizeQueried, nChunkYSizeQueried,
1852 : pabyChunkNoDataMask, nChunkXSizeQueried,
1853 : nChunkYSizeQueried, GDT_UInt8, 0, 0, nullptr);
1854 :
1855 : /* Optimizations if mask if fully opaque or transparent */
1856 2156 : const int nPixels = nChunkXSizeQueried * nChunkYSizeQueried;
1857 2156 : const GByte bVal = pabyChunkNoDataMask[0];
1858 2156 : int i = 1; // Used after for.
1859 49799600 : for (; i < nPixels; i++)
1860 : {
1861 49798500 : if (pabyChunkNoDataMask[i] != bVal)
1862 1031 : break;
1863 : }
1864 2156 : if (i == nPixels)
1865 : {
1866 1125 : if (bVal == 0)
1867 : {
1868 953 : GByte abyZero[16] = {0};
1869 3100 : for (int iBand = 0; iBand < nBandCount; iBand++)
1870 : {
1871 6979 : for (int j = 0; j < nDstYCount; j++)
1872 : {
1873 4832 : GDALCopyWords64(
1874 : abyZero, GDT_UInt8, 0,
1875 : static_cast<GByte *>(pDataMem) +
1876 4832 : iBand * nBandSpaceMEM +
1877 4832 : nLSMem * (j + nDstYOff) +
1878 4832 : nDstXOff * nPSMem,
1879 : eBufType, static_cast<int>(nPSMem),
1880 : nDstXCount);
1881 : }
1882 : }
1883 953 : bSkipResample = true;
1884 : }
1885 : else
1886 : {
1887 172 : bNoDataMaskFullyOpaque = true;
1888 : }
1889 : }
1890 : }
1891 :
1892 2431 : if (!bSkipResample && eErr == CE_None)
1893 : {
1894 : /* Read the source buffers */
1895 1475 : eErr = RasterIO(
1896 : GF_Read, nChunkXOffQueried, nChunkYOffQueried,
1897 : nChunkXSizeQueried, nChunkYSizeQueried, pChunk,
1898 : nChunkXSizeQueried, nChunkYSizeQueried, eWrkDataType,
1899 : nBandCount, panBandMap, 0, 0, 0, nullptr);
1900 : }
1901 :
1902 : #ifdef GDAL_ENABLE_RESAMPLING_MULTIBAND
1903 : if (pfnResampleFuncMultiBands && !bSkipResample &&
1904 : eErr == CE_None)
1905 : {
1906 : eErr = pfnResampleFuncMultiBands(
1907 : dfXRatioDstToSrc, dfYRatioDstToSrc,
1908 : dfXOff - nXOff, /* == 0 if bHasXOffVirtual */
1909 : dfYOff - nYOff, /* == 0 if bHasYOffVirtual */
1910 : eWrkDataType, (GByte *)pChunk, nBandCount,
1911 : bNoDataMaskFullyOpaque ? nullptr : pabyChunkNoDataMask,
1912 : nChunkXOffQueried - (bHasXOffVirtual ? 0 : nXOff),
1913 : nChunkXSizeQueried,
1914 : nChunkYOffQueried - (bHasYOffVirtual ? 0 : nYOff),
1915 : nChunkYSizeQueried, nDstXOff + nDestXOffVirtual,
1916 : nDstXOff + nDestXOffVirtual + nDstXCount,
1917 : nDstYOff + nDestYOffVirtual,
1918 : nDstYOff + nDestYOffVirtual + nDstYCount,
1919 : apoDstBands.data(), pszResampling, FALSE /*bHasNoData*/,
1920 : 0.0 /* dfNoDataValue */, nullptr /* color table*/,
1921 : eDataType);
1922 : }
1923 : else
1924 : #endif
1925 : {
1926 : size_t nChunkBandOffset =
1927 2431 : static_cast<size_t>(nChunkXSizeQueried) *
1928 2431 : nChunkYSizeQueried *
1929 2431 : GDALGetDataTypeSizeBytes(eWrkDataType);
1930 6896 : for (int i = 0;
1931 6896 : i < nBandCount && !bSkipResample && eErr == CE_None;
1932 : i++)
1933 : {
1934 4465 : const bool bPropagateNoData = false;
1935 4465 : void *pDstBuffer = nullptr;
1936 4465 : GDALDataType eDstBufferDataType = GDT_Unknown;
1937 : GDALRasterBand *poMEMBand =
1938 4465 : poMEMDS->GetRasterBand(i + 1);
1939 4465 : GDALOverviewResampleArgs args;
1940 4465 : args.eSrcDataType = eDataType;
1941 4465 : args.eOvrDataType = poMEMBand->GetRasterDataType();
1942 4465 : args.nOvrXSize = poMEMBand->GetXSize();
1943 4465 : args.nOvrYSize = poMEMBand->GetYSize();
1944 4465 : args.nOvrNBITS = nNBITS;
1945 4465 : args.dfXRatioDstToSrc = dfXRatioDstToSrc;
1946 4465 : args.dfYRatioDstToSrc = dfYRatioDstToSrc;
1947 4465 : args.dfSrcXDelta =
1948 4465 : dfXOff - nXOff; /* == 0 if bHasXOffVirtual */
1949 4465 : args.dfSrcYDelta =
1950 4465 : dfYOff - nYOff; /* == 0 if bHasYOffVirtual */
1951 4465 : args.eWrkDataType = eWrkDataType;
1952 4465 : args.pabyChunkNodataMask = bNoDataMaskFullyOpaque
1953 4465 : ? nullptr
1954 : : pabyChunkNoDataMask;
1955 4465 : args.nChunkXOff =
1956 4465 : nChunkXOffQueried - (bHasXOffVirtual ? 0 : nXOff);
1957 4465 : args.nChunkXSize = nChunkXSizeQueried;
1958 4465 : args.nChunkYOff =
1959 4465 : nChunkYOffQueried - (bHasYOffVirtual ? 0 : nYOff);
1960 4465 : args.nChunkYSize = nChunkYSizeQueried;
1961 4465 : args.nDstXOff = nDstXOff + nDestXOffVirtual;
1962 4465 : args.nDstXOff2 =
1963 4465 : nDstXOff + nDestXOffVirtual + nDstXCount;
1964 4465 : args.nDstYOff = nDstYOff + nDestYOffVirtual;
1965 4465 : args.nDstYOff2 =
1966 4465 : nDstYOff + nDestYOffVirtual + nDstYCount;
1967 4465 : args.pszResampling = pszResampling;
1968 4465 : args.bHasNoData = false;
1969 4465 : args.dfNoDataValue = 0.0;
1970 4465 : args.poColorTable = nullptr;
1971 4465 : args.bPropagateNoData = bPropagateNoData;
1972 :
1973 : eErr =
1974 8930 : pfnResampleFunc(args,
1975 4465 : reinterpret_cast<GByte *>(pChunk) +
1976 4465 : i * nChunkBandOffset,
1977 : &pDstBuffer, &eDstBufferDataType);
1978 4465 : if (eErr == CE_None)
1979 : {
1980 4465 : eErr = poMEMBand->RasterIO(
1981 : GF_Write, nDstXOff + nDestXOffVirtual,
1982 : nDstYOff + nDestYOffVirtual, nDstXCount,
1983 : nDstYCount, pDstBuffer, nDstXCount, nDstYCount,
1984 : eDstBufferDataType, 0, 0, nullptr);
1985 : }
1986 4465 : CPLFree(pDstBuffer);
1987 : }
1988 : }
1989 :
1990 2431 : nBlocksDone++;
1991 4356 : if (eErr == CE_None && psExtraArg->pfnProgress != nullptr &&
1992 1925 : !psExtraArg->pfnProgress(
1993 1925 : static_cast<double>(nBlocksDone) /
1994 1925 : static_cast<double>(nTotalBlocks),
1995 : "", psExtraArg->pProgressData))
1996 : {
1997 0 : eErr = CE_Failure;
1998 : }
1999 : }
2000 : }
2001 :
2002 2431 : CPLFree(pChunk);
2003 2431 : CPLFree(pabyChunkNoDataMask);
2004 : }
2005 :
2006 2431 : if (pTempBuffer)
2007 : {
2008 2 : CPL_IGNORE_RET_VAL(poMEMDS->RasterIO(
2009 : GF_Read, nDestXOffVirtual, nDestYOffVirtual, nBufXSize, nBufYSize,
2010 : pData, nBufXSize, nBufYSize, eBufType, nBandCount, nullptr,
2011 : nPixelSpace, nLineSpace, nBandSpace, nullptr));
2012 : }
2013 :
2014 2431 : return eErr;
2015 : }
2016 :
2017 : //! @endcond
2018 :
2019 : /************************************************************************/
2020 : /* GDALSwapWords() */
2021 : /************************************************************************/
2022 :
2023 : /**
2024 : * Byte swap words in-place.
2025 : *
2026 : * This function will byte swap a set of 2, 4 or 8 byte words "in place" in
2027 : * a memory array. No assumption is made that the words being swapped are
2028 : * word aligned in memory. Use the CPL_LSB and CPL_MSB macros from cpl_port.h
2029 : * to determine if the current platform is big endian or little endian. Use
2030 : * The macros like CPL_SWAP32() to byte swap single values without the overhead
2031 : * of a function call.
2032 : *
2033 : * @param pData pointer to start of data buffer.
2034 : * @param nWordSize size of words being swapped in bytes. Normally 2, 4 or 8.
2035 : * @param nWordCount the number of words to be swapped in this call.
2036 : * @param nWordSkip the byte offset from the start of one word to the start of
2037 : * the next. For packed buffers this is the same as nWordSize.
2038 : */
2039 :
2040 497405 : void CPL_STDCALL GDALSwapWords(void *pData, int nWordSize, int nWordCount,
2041 : int nWordSkip)
2042 :
2043 : {
2044 497405 : if (nWordCount > 0)
2045 497405 : VALIDATE_POINTER0(pData, "GDALSwapWords");
2046 :
2047 497405 : GByte *pabyData = static_cast<GByte *>(pData);
2048 :
2049 497405 : switch (nWordSize)
2050 : {
2051 7234 : case 1:
2052 7234 : break;
2053 :
2054 477161 : case 2:
2055 477161 : CPLAssert(nWordSkip >= 2 || nWordCount == 1);
2056 228194000 : for (int i = 0; i < nWordCount; i++)
2057 : {
2058 227716000 : CPL_SWAP16PTR(pabyData);
2059 227716000 : pabyData += nWordSkip;
2060 : }
2061 477161 : break;
2062 :
2063 10584 : case 4:
2064 10584 : CPLAssert(nWordSkip >= 4 || nWordCount == 1);
2065 10584 : if (CPL_IS_ALIGNED(pabyData, 4) && (nWordSkip % 4) == 0)
2066 : {
2067 29140600 : for (int i = 0; i < nWordCount; i++)
2068 : {
2069 29130000 : *reinterpret_cast<GUInt32 *>(pabyData) = CPL_SWAP32(
2070 : *reinterpret_cast<const GUInt32 *>(pabyData));
2071 29130000 : pabyData += nWordSkip;
2072 10581 : }
2073 : }
2074 : else
2075 : {
2076 9 : for (int i = 0; i < nWordCount; i++)
2077 : {
2078 6 : CPL_SWAP32PTR(pabyData);
2079 6 : pabyData += nWordSkip;
2080 : }
2081 : }
2082 10584 : break;
2083 :
2084 2426 : case 8:
2085 2426 : CPLAssert(nWordSkip >= 8 || nWordCount == 1);
2086 2426 : if (CPL_IS_ALIGNED(pabyData, 8) && (nWordSkip % 8) == 0)
2087 : {
2088 3356900 : for (int i = 0; i < nWordCount; i++)
2089 : {
2090 3354480 : *reinterpret_cast<GUInt64 *>(pabyData) = CPL_SWAP64(
2091 : *reinterpret_cast<const GUInt64 *>(pabyData));
2092 3354480 : pabyData += nWordSkip;
2093 2425 : }
2094 : }
2095 : else
2096 : {
2097 3 : for (int i = 0; i < nWordCount; i++)
2098 : {
2099 2 : CPL_SWAP64PTR(pabyData);
2100 2 : pabyData += nWordSkip;
2101 : }
2102 : }
2103 2426 : break;
2104 :
2105 0 : default:
2106 0 : CPLAssert(false);
2107 : }
2108 : }
2109 :
2110 : /************************************************************************/
2111 : /* GDALSwapWordsEx() */
2112 : /************************************************************************/
2113 :
2114 : /**
2115 : * Byte swap words in-place.
2116 : *
2117 : * This function will byte swap a set of 2, 4 or 8 byte words "in place" in
2118 : * a memory array. No assumption is made that the words being swapped are
2119 : * word aligned in memory. Use the CPL_LSB and CPL_MSB macros from cpl_port.h
2120 : * to determine if the current platform is big endian or little endian. Use
2121 : * The macros like CPL_SWAP32() to byte swap single values without the overhead
2122 : * of a function call.
2123 : *
2124 : * @param pData pointer to start of data buffer.
2125 : * @param nWordSize size of words being swapped in bytes. Normally 2, 4 or 8.
2126 : * @param nWordCount the number of words to be swapped in this call.
2127 : * @param nWordSkip the byte offset from the start of one word to the start of
2128 : * the next. For packed buffers this is the same as nWordSize.
2129 : */
2130 6130 : void CPL_STDCALL GDALSwapWordsEx(void *pData, int nWordSize, size_t nWordCount,
2131 : int nWordSkip)
2132 : {
2133 6130 : GByte *pabyData = static_cast<GByte *>(pData);
2134 12260 : while (nWordCount)
2135 : {
2136 : // Pick-up a multiple of 8 as max chunk size.
2137 6130 : const int nWordCountSmall =
2138 6130 : (nWordCount > (1 << 30)) ? (1 << 30) : static_cast<int>(nWordCount);
2139 6130 : GDALSwapWords(pabyData, nWordSize, nWordCountSmall, nWordSkip);
2140 6130 : pabyData += static_cast<size_t>(nWordSkip) * nWordCountSmall;
2141 6130 : nWordCount -= nWordCountSmall;
2142 : }
2143 6130 : }
2144 :
2145 : // Place the new GDALCopyWords helpers in an anonymous namespace
2146 : namespace
2147 : {
2148 :
2149 : /************************************************************************/
2150 : /* GDALCopyWordsT() */
2151 : /************************************************************************/
2152 : /**
2153 : * Template function, used to copy data from pSrcData into buffer
2154 : * pDstData, with stride nSrcPixelStride in the source data and
2155 : * stride nDstPixelStride in the destination data. This template can
2156 : * deal with the case where the input data type is real or complex and
2157 : * the output is real.
2158 : *
2159 : * @param pSrcData the source data buffer
2160 : * @param nSrcPixelStride the stride, in the buffer pSrcData for pixels
2161 : * of interest.
2162 : * @param pDstData the destination buffer.
2163 : * @param nDstPixelStride the stride in the buffer pDstData for pixels of
2164 : * interest.
2165 : * @param nWordCount the total number of pixel words to copy
2166 : *
2167 : * @code
2168 : * // Assume an input buffer of type GUInt16 named pBufferIn
2169 : * GByte *pBufferOut = new GByte[numBytesOut];
2170 : * GDALCopyWordsT<GUInt16, GByte>(pSrcData, 2, pDstData, 1, numBytesOut);
2171 : * @endcode
2172 : * @note
2173 : * This is a private function, and should not be exposed outside of
2174 : * rasterio.cpp. External users should call the GDALCopyWords driver function.
2175 : */
2176 :
2177 : template <class Tin, class Tout>
2178 49014325 : static void inline GDALCopyWordsGenericT(const Tin *const CPL_RESTRICT pSrcData,
2179 : int nSrcPixelStride,
2180 : Tout *const CPL_RESTRICT pDstData,
2181 : int nDstPixelStride,
2182 : GPtrDiff_t nWordCount)
2183 : {
2184 49014325 : decltype(nWordCount) nDstOffset = 0;
2185 :
2186 49014325 : const char *const pSrcDataPtr = reinterpret_cast<const char *>(pSrcData);
2187 49014325 : char *const pDstDataPtr = reinterpret_cast<char *>(pDstData);
2188 356710337 : for (decltype(nWordCount) n = 0; n < nWordCount; n++)
2189 : {
2190 307695964 : const Tin tValue =
2191 307695964 : *reinterpret_cast<const Tin *>(pSrcDataPtr + (n * nSrcPixelStride));
2192 307695964 : Tout *const pOutPixel =
2193 307695964 : reinterpret_cast<Tout *>(pDstDataPtr + nDstOffset);
2194 :
2195 307695964 : GDALCopyWord(tValue, *pOutPixel);
2196 :
2197 307695964 : nDstOffset += nDstPixelStride;
2198 : }
2199 49014325 : }
2200 :
2201 : template <class Tin, class Tout>
2202 29777128 : static void CPL_NOINLINE GDALCopyWordsT(const Tin *const CPL_RESTRICT pSrcData,
2203 : int nSrcPixelStride,
2204 : Tout *const CPL_RESTRICT pDstData,
2205 : int nDstPixelStride,
2206 : GPtrDiff_t nWordCount)
2207 : {
2208 29777128 : GDALCopyWordsGenericT(pSrcData, nSrcPixelStride, pDstData, nDstPixelStride,
2209 : nWordCount);
2210 29777128 : }
2211 :
2212 : template <class Tin, class Tout>
2213 5101323 : static void inline GDALCopyWordsT_8atatime(
2214 : const Tin *const CPL_RESTRICT pSrcData, int nSrcPixelStride,
2215 : Tout *const CPL_RESTRICT pDstData, int nDstPixelStride,
2216 : GPtrDiff_t nWordCount)
2217 : {
2218 5101323 : decltype(nWordCount) nDstOffset = 0;
2219 :
2220 5101323 : const char *const pSrcDataPtr = reinterpret_cast<const char *>(pSrcData);
2221 5101323 : char *const pDstDataPtr = reinterpret_cast<char *>(pDstData);
2222 5101323 : decltype(nWordCount) n = 0;
2223 5101323 : if (nSrcPixelStride == static_cast<int>(sizeof(Tin)) &&
2224 : nDstPixelStride == static_cast<int>(sizeof(Tout)))
2225 : {
2226 53187487 : for (; n < nWordCount - 7; n += 8)
2227 : {
2228 52636036 : const Tin *pInValues = reinterpret_cast<const Tin *>(
2229 52636036 : pSrcDataPtr + (n * nSrcPixelStride));
2230 52636036 : Tout *const pOutPixels =
2231 52636036 : reinterpret_cast<Tout *>(pDstDataPtr + nDstOffset);
2232 :
2233 52636036 : GDALCopy8Words(pInValues, pOutPixels);
2234 :
2235 52636036 : nDstOffset += 8 * nDstPixelStride;
2236 : }
2237 : }
2238 10498785 : for (; n < nWordCount; n++)
2239 : {
2240 5397472 : const Tin tValue =
2241 5397472 : *reinterpret_cast<const Tin *>(pSrcDataPtr + (n * nSrcPixelStride));
2242 5397472 : Tout *const pOutPixel =
2243 5397472 : reinterpret_cast<Tout *>(pDstDataPtr + nDstOffset);
2244 :
2245 5397472 : GDALCopyWord(tValue, *pOutPixel);
2246 :
2247 5397472 : nDstOffset += nDstPixelStride;
2248 : }
2249 5101323 : }
2250 :
2251 : #ifdef HAVE_SSE2
2252 :
2253 : template <class Tout>
2254 1042126 : void GDALCopyWordsByteTo16Bit(const GByte *const CPL_RESTRICT pSrcData,
2255 : int nSrcPixelStride,
2256 : Tout *const CPL_RESTRICT pDstData,
2257 : int nDstPixelStride, GPtrDiff_t nWordCount)
2258 : {
2259 : static_assert(std::is_integral<Tout>::value &&
2260 : sizeof(Tout) == sizeof(uint16_t),
2261 : "Bad Tout");
2262 1042126 : if (nSrcPixelStride == static_cast<int>(sizeof(*pSrcData)) &&
2263 : nDstPixelStride == static_cast<int>(sizeof(*pDstData)))
2264 : {
2265 35752 : decltype(nWordCount) n = 0;
2266 35752 : const __m128i xmm_zero = _mm_setzero_si128();
2267 35752 : GByte *CPL_RESTRICT pabyDstDataPtr =
2268 : reinterpret_cast<GByte *>(pDstData);
2269 1478148 : for (; n < nWordCount - 15; n += 16)
2270 : {
2271 1442396 : __m128i xmm = _mm_loadu_si128(
2272 1442396 : reinterpret_cast<const __m128i *>(pSrcData + n));
2273 1442396 : __m128i xmm0 = _mm_unpacklo_epi8(xmm, xmm_zero);
2274 1442396 : __m128i xmm1 = _mm_unpackhi_epi8(xmm, xmm_zero);
2275 : _mm_storeu_si128(
2276 1442396 : reinterpret_cast<__m128i *>(pabyDstDataPtr + n * 2), xmm0);
2277 : _mm_storeu_si128(
2278 1442396 : reinterpret_cast<__m128i *>(pabyDstDataPtr + n * 2 + 16), xmm1);
2279 : }
2280 111662 : for (; n < nWordCount; n++)
2281 : {
2282 75910 : pDstData[n] = pSrcData[n];
2283 35752 : }
2284 : }
2285 : else
2286 : {
2287 1006371 : GDALCopyWordsGenericT(pSrcData, nSrcPixelStride, pDstData,
2288 : nDstPixelStride, nWordCount);
2289 : }
2290 1042126 : }
2291 :
2292 : template <>
2293 1029400 : CPL_NOINLINE void GDALCopyWordsT(const GByte *const CPL_RESTRICT pSrcData,
2294 : int nSrcPixelStride,
2295 : GUInt16 *const CPL_RESTRICT pDstData,
2296 : int nDstPixelStride, GPtrDiff_t nWordCount)
2297 : {
2298 1029400 : GDALCopyWordsByteTo16Bit(pSrcData, nSrcPixelStride, pDstData,
2299 : nDstPixelStride, nWordCount);
2300 1029400 : }
2301 :
2302 : template <>
2303 12726 : CPL_NOINLINE void GDALCopyWordsT(const GByte *const CPL_RESTRICT pSrcData,
2304 : int nSrcPixelStride,
2305 : GInt16 *const CPL_RESTRICT pDstData,
2306 : int nDstPixelStride, GPtrDiff_t nWordCount)
2307 : {
2308 12726 : GDALCopyWordsByteTo16Bit(pSrcData, nSrcPixelStride, pDstData,
2309 : nDstPixelStride, nWordCount);
2310 12726 : }
2311 :
2312 : template <class Tout>
2313 16237376 : void GDALCopyWordsByteTo32Bit(const GByte *const CPL_RESTRICT pSrcData,
2314 : int nSrcPixelStride,
2315 : Tout *const CPL_RESTRICT pDstData,
2316 : int nDstPixelStride, GPtrDiff_t nWordCount)
2317 : {
2318 : static_assert(std::is_integral<Tout>::value &&
2319 : sizeof(Tout) == sizeof(uint32_t),
2320 : "Bad Tout");
2321 16237376 : if (nSrcPixelStride == static_cast<int>(sizeof(*pSrcData)) &&
2322 : nDstPixelStride == static_cast<int>(sizeof(*pDstData)))
2323 : {
2324 6532886 : decltype(nWordCount) n = 0;
2325 6532886 : const __m128i xmm_zero = _mm_setzero_si128();
2326 6532886 : GByte *CPL_RESTRICT pabyDstDataPtr =
2327 : reinterpret_cast<GByte *>(pDstData);
2328 74248527 : for (; n < nWordCount - 15; n += 16)
2329 : {
2330 67715661 : __m128i xmm = _mm_loadu_si128(
2331 67715661 : reinterpret_cast<const __m128i *>(pSrcData + n));
2332 67715661 : __m128i xmm_low = _mm_unpacklo_epi8(xmm, xmm_zero);
2333 67715661 : __m128i xmm_high = _mm_unpackhi_epi8(xmm, xmm_zero);
2334 67715661 : __m128i xmm0 = _mm_unpacklo_epi16(xmm_low, xmm_zero);
2335 67715661 : __m128i xmm1 = _mm_unpackhi_epi16(xmm_low, xmm_zero);
2336 67715661 : __m128i xmm2 = _mm_unpacklo_epi16(xmm_high, xmm_zero);
2337 67715661 : __m128i xmm3 = _mm_unpackhi_epi16(xmm_high, xmm_zero);
2338 : _mm_storeu_si128(
2339 67715661 : reinterpret_cast<__m128i *>(pabyDstDataPtr + n * 4), xmm0);
2340 : _mm_storeu_si128(
2341 67715661 : reinterpret_cast<__m128i *>(pabyDstDataPtr + n * 4 + 16), xmm1);
2342 : _mm_storeu_si128(
2343 67715661 : reinterpret_cast<__m128i *>(pabyDstDataPtr + n * 4 + 32), xmm2);
2344 : _mm_storeu_si128(
2345 67715661 : reinterpret_cast<__m128i *>(pabyDstDataPtr + n * 4 + 48), xmm3);
2346 : }
2347 14828116 : for (; n < nWordCount; n++)
2348 : {
2349 8295270 : pDstData[n] = pSrcData[n];
2350 6532886 : }
2351 : }
2352 : else
2353 : {
2354 9704510 : GDALCopyWordsGenericT(pSrcData, nSrcPixelStride, pDstData,
2355 : nDstPixelStride, nWordCount);
2356 : }
2357 16237376 : }
2358 :
2359 : template <>
2360 476 : CPL_NOINLINE void GDALCopyWordsT(const GByte *const CPL_RESTRICT pSrcData,
2361 : int nSrcPixelStride,
2362 : GUInt32 *const CPL_RESTRICT pDstData,
2363 : int nDstPixelStride, GPtrDiff_t nWordCount)
2364 : {
2365 476 : GDALCopyWordsByteTo32Bit(pSrcData, nSrcPixelStride, pDstData,
2366 : nDstPixelStride, nWordCount);
2367 476 : }
2368 :
2369 : template <>
2370 16236900 : CPL_NOINLINE void GDALCopyWordsT(const GByte *const CPL_RESTRICT pSrcData,
2371 : int nSrcPixelStride,
2372 : GInt32 *const CPL_RESTRICT pDstData,
2373 : int nDstPixelStride, GPtrDiff_t nWordCount)
2374 : {
2375 16236900 : GDALCopyWordsByteTo32Bit(pSrcData, nSrcPixelStride, pDstData,
2376 : nDstPixelStride, nWordCount);
2377 16236900 : }
2378 :
2379 : template <>
2380 2851030 : CPL_NOINLINE void GDALCopyWordsT(const GByte *const CPL_RESTRICT pSrcData,
2381 : int nSrcPixelStride,
2382 : float *const CPL_RESTRICT pDstData,
2383 : int nDstPixelStride, GPtrDiff_t nWordCount)
2384 : {
2385 2851030 : if (nSrcPixelStride == static_cast<int>(sizeof(*pSrcData)) &&
2386 : nDstPixelStride == static_cast<int>(sizeof(*pDstData)))
2387 : {
2388 228149 : decltype(nWordCount) n = 0;
2389 228149 : const __m128i xmm_zero = _mm_setzero_si128();
2390 228149 : GByte *CPL_RESTRICT pabyDstDataPtr =
2391 : reinterpret_cast<GByte *>(pDstData);
2392 2267080 : for (; n < nWordCount - 15; n += 16)
2393 : {
2394 2038930 : __m128i xmm = _mm_loadu_si128(
2395 2038930 : reinterpret_cast<const __m128i *>(pSrcData + n));
2396 2038930 : __m128i xmm_low = _mm_unpacklo_epi8(xmm, xmm_zero);
2397 2038930 : __m128i xmm_high = _mm_unpackhi_epi8(xmm, xmm_zero);
2398 2038930 : __m128i xmm0 = _mm_unpacklo_epi16(xmm_low, xmm_zero);
2399 2038930 : __m128i xmm1 = _mm_unpackhi_epi16(xmm_low, xmm_zero);
2400 2038930 : __m128i xmm2 = _mm_unpacklo_epi16(xmm_high, xmm_zero);
2401 2038930 : __m128i xmm3 = _mm_unpackhi_epi16(xmm_high, xmm_zero);
2402 2038930 : __m128 xmm0_f = _mm_cvtepi32_ps(xmm0);
2403 2038930 : __m128 xmm1_f = _mm_cvtepi32_ps(xmm1);
2404 2038930 : __m128 xmm2_f = _mm_cvtepi32_ps(xmm2);
2405 2038930 : __m128 xmm3_f = _mm_cvtepi32_ps(xmm3);
2406 2038930 : _mm_storeu_ps(reinterpret_cast<float *>(pabyDstDataPtr + n * 4),
2407 : xmm0_f);
2408 : _mm_storeu_ps(
2409 2038930 : reinterpret_cast<float *>(pabyDstDataPtr + n * 4 + 16), xmm1_f);
2410 : _mm_storeu_ps(
2411 2038930 : reinterpret_cast<float *>(pabyDstDataPtr + n * 4 + 32), xmm2_f);
2412 : _mm_storeu_ps(
2413 2038930 : reinterpret_cast<float *>(pabyDstDataPtr + n * 4 + 48), xmm3_f);
2414 : }
2415 951237 : for (; n < nWordCount; n++)
2416 : {
2417 723088 : pDstData[n] = pSrcData[n];
2418 228149 : }
2419 : }
2420 : else
2421 : {
2422 2622880 : GDALCopyWordsGenericT(pSrcData, nSrcPixelStride, pDstData,
2423 : nDstPixelStride, nWordCount);
2424 : }
2425 2851030 : }
2426 :
2427 : template <>
2428 170938 : CPL_NOINLINE void GDALCopyWordsT(const GByte *const CPL_RESTRICT pSrcData,
2429 : int nSrcPixelStride,
2430 : double *const CPL_RESTRICT pDstData,
2431 : int nDstPixelStride, GPtrDiff_t nWordCount)
2432 : {
2433 170938 : if (nSrcPixelStride == static_cast<int>(sizeof(*pSrcData)) &&
2434 : nDstPixelStride == static_cast<int>(sizeof(*pDstData)))
2435 : {
2436 147140 : decltype(nWordCount) n = 0;
2437 147140 : const __m128i xmm_zero = _mm_setzero_si128();
2438 147140 : GByte *CPL_RESTRICT pabyDstDataPtr =
2439 : reinterpret_cast<GByte *>(pDstData);
2440 3127410 : for (; n < nWordCount - 15; n += 16)
2441 : {
2442 2980270 : __m128i xmm = _mm_loadu_si128(
2443 2980270 : reinterpret_cast<const __m128i *>(pSrcData + n));
2444 2980270 : __m128i xmm_low = _mm_unpacklo_epi8(xmm, xmm_zero);
2445 2980270 : __m128i xmm_high = _mm_unpackhi_epi8(xmm, xmm_zero);
2446 2980270 : __m128i xmm0 = _mm_unpacklo_epi16(xmm_low, xmm_zero);
2447 2980270 : __m128i xmm1 = _mm_unpackhi_epi16(xmm_low, xmm_zero);
2448 2980270 : __m128i xmm2 = _mm_unpacklo_epi16(xmm_high, xmm_zero);
2449 2980270 : __m128i xmm3 = _mm_unpackhi_epi16(xmm_high, xmm_zero);
2450 :
2451 : #if defined(__AVX2__) && defined(slightly_slower_than_SSE2)
2452 : _mm256_storeu_pd(reinterpret_cast<double *>(pabyDstDataPtr + n * 8),
2453 : _mm256_cvtepi32_pd(xmm0));
2454 : _mm256_storeu_pd(
2455 : reinterpret_cast<double *>(pabyDstDataPtr + n * 8 + 32),
2456 : _mm256_cvtepi32_pd(xmm1));
2457 : _mm256_storeu_pd(
2458 : reinterpret_cast<double *>(pabyDstDataPtr + n * 8 + 64),
2459 : _mm256_cvtepi32_pd(xmm2));
2460 : _mm256_storeu_pd(
2461 : reinterpret_cast<double *>(pabyDstDataPtr + n * 8 + 96),
2462 : _mm256_cvtepi32_pd(xmm3));
2463 : #else
2464 2980270 : __m128d xmm0_low_d = _mm_cvtepi32_pd(xmm0);
2465 2980270 : __m128d xmm1_low_d = _mm_cvtepi32_pd(xmm1);
2466 2980270 : __m128d xmm2_low_d = _mm_cvtepi32_pd(xmm2);
2467 2980270 : __m128d xmm3_low_d = _mm_cvtepi32_pd(xmm3);
2468 2980270 : xmm0 = _mm_srli_si128(xmm0, 8);
2469 2980270 : xmm1 = _mm_srli_si128(xmm1, 8);
2470 2980270 : xmm2 = _mm_srli_si128(xmm2, 8);
2471 2980270 : xmm3 = _mm_srli_si128(xmm3, 8);
2472 2980270 : __m128d xmm0_high_d = _mm_cvtepi32_pd(xmm0);
2473 2980270 : __m128d xmm1_high_d = _mm_cvtepi32_pd(xmm1);
2474 2980270 : __m128d xmm2_high_d = _mm_cvtepi32_pd(xmm2);
2475 2980270 : __m128d xmm3_high_d = _mm_cvtepi32_pd(xmm3);
2476 :
2477 2980270 : _mm_storeu_pd(reinterpret_cast<double *>(pabyDstDataPtr + n * 8),
2478 : xmm0_low_d);
2479 : _mm_storeu_pd(
2480 2980270 : reinterpret_cast<double *>(pabyDstDataPtr + n * 8 + 16),
2481 : xmm0_high_d);
2482 : _mm_storeu_pd(
2483 2980270 : reinterpret_cast<double *>(pabyDstDataPtr + n * 8 + 32),
2484 : xmm1_low_d);
2485 : _mm_storeu_pd(
2486 2980270 : reinterpret_cast<double *>(pabyDstDataPtr + n * 8 + 48),
2487 : xmm1_high_d);
2488 : _mm_storeu_pd(
2489 2980270 : reinterpret_cast<double *>(pabyDstDataPtr + n * 8 + 64),
2490 : xmm2_low_d);
2491 : _mm_storeu_pd(
2492 2980270 : reinterpret_cast<double *>(pabyDstDataPtr + n * 8 + 80),
2493 : xmm2_high_d);
2494 : _mm_storeu_pd(
2495 2980270 : reinterpret_cast<double *>(pabyDstDataPtr + n * 8 + 96),
2496 : xmm3_low_d);
2497 : _mm_storeu_pd(
2498 2980270 : reinterpret_cast<double *>(pabyDstDataPtr + n * 8 + 112),
2499 : xmm3_high_d);
2500 : #endif
2501 : }
2502 280823 : for (; n < nWordCount; n++)
2503 : {
2504 133683 : pDstData[n] = pSrcData[n];
2505 147140 : }
2506 : }
2507 : else
2508 : {
2509 23798 : GDALCopyWordsGenericT(pSrcData, nSrcPixelStride, pDstData,
2510 : nDstPixelStride, nWordCount);
2511 : }
2512 170938 : }
2513 :
2514 : template <>
2515 148 : CPL_NOINLINE void GDALCopyWordsT(const uint8_t *const CPL_RESTRICT pSrcData,
2516 : int nSrcPixelStride,
2517 : int8_t *const CPL_RESTRICT pDstData,
2518 : int nDstPixelStride, GPtrDiff_t nWordCount)
2519 : {
2520 148 : if (nSrcPixelStride == static_cast<int>(sizeof(*pSrcData)) &&
2521 : nDstPixelStride == static_cast<int>(sizeof(*pDstData)))
2522 : {
2523 142 : decltype(nWordCount) n = 0;
2524 142 : const __m128i xmm_127 = _mm_set1_epi8(127);
2525 146 : for (; n < nWordCount - 31; n += 32)
2526 : {
2527 8 : __m128i xmm0 = _mm_loadu_si128(
2528 4 : reinterpret_cast<const __m128i *>(pSrcData + n));
2529 4 : __m128i xmm1 = _mm_loadu_si128(
2530 4 : reinterpret_cast<const __m128i *>(pSrcData + n + 16));
2531 4 : xmm0 = _mm_min_epu8(xmm0, xmm_127);
2532 4 : xmm1 = _mm_min_epu8(xmm1, xmm_127);
2533 4 : _mm_storeu_si128(reinterpret_cast<__m128i *>(pDstData + n), xmm0);
2534 4 : _mm_storeu_si128(reinterpret_cast<__m128i *>(pDstData + n + 16),
2535 : xmm1);
2536 : }
2537 2424 : for (; n < nWordCount; n++)
2538 : {
2539 2282 : pDstData[n] =
2540 2282 : pSrcData[n] >= 127 ? 127 : static_cast<int8_t>(pSrcData[n]);
2541 142 : }
2542 : }
2543 : else
2544 : {
2545 6 : GDALCopyWordsGenericT(pSrcData, nSrcPixelStride, pDstData,
2546 : nDstPixelStride, nWordCount);
2547 : }
2548 148 : }
2549 :
2550 : template <>
2551 62 : CPL_NOINLINE void GDALCopyWordsT(const int8_t *const CPL_RESTRICT pSrcData,
2552 : int nSrcPixelStride,
2553 : uint8_t *const CPL_RESTRICT pDstData,
2554 : int nDstPixelStride, GPtrDiff_t nWordCount)
2555 : {
2556 62 : if (nSrcPixelStride == static_cast<int>(sizeof(*pSrcData)) &&
2557 : nDstPixelStride == static_cast<int>(sizeof(*pDstData)))
2558 : {
2559 56 : decltype(nWordCount) n = 0;
2560 : #if !(defined(__SSE4_1__) || defined(USE_NEON_OPTIMIZATIONS))
2561 56 : const __m128i xmm_INT8_to_UINT8 = _mm_set1_epi8(-128);
2562 : #endif
2563 117 : for (; n < nWordCount - 31; n += 32)
2564 : {
2565 122 : __m128i xmm0 = _mm_loadu_si128(
2566 61 : reinterpret_cast<const __m128i *>(pSrcData + n));
2567 61 : __m128i xmm1 = _mm_loadu_si128(
2568 61 : reinterpret_cast<const __m128i *>(pSrcData + n + 16));
2569 : #if defined(__SSE4_1__) || defined(USE_NEON_OPTIMIZATIONS)
2570 : xmm0 = _mm_max_epi8(xmm0, _mm_setzero_si128());
2571 : xmm1 = _mm_max_epi8(xmm1, _mm_setzero_si128());
2572 : #else
2573 61 : xmm0 = _mm_add_epi8(xmm0, xmm_INT8_to_UINT8);
2574 61 : xmm1 = _mm_add_epi8(xmm1, xmm_INT8_to_UINT8);
2575 61 : xmm0 = _mm_max_epu8(xmm0, xmm_INT8_to_UINT8);
2576 61 : xmm1 = _mm_max_epu8(xmm1, xmm_INT8_to_UINT8);
2577 61 : xmm0 = _mm_sub_epi8(xmm0, xmm_INT8_to_UINT8);
2578 61 : xmm1 = _mm_sub_epi8(xmm1, xmm_INT8_to_UINT8);
2579 : #endif
2580 61 : _mm_storeu_si128(reinterpret_cast<__m128i *>(pDstData + n), xmm0);
2581 61 : _mm_storeu_si128(reinterpret_cast<__m128i *>(pDstData + n + 16),
2582 : xmm1);
2583 : }
2584 352 : for (; n < nWordCount; n++)
2585 : {
2586 296 : pDstData[n] =
2587 296 : pSrcData[n] < 0 ? 0 : static_cast<uint8_t>(pSrcData[n]);
2588 56 : }
2589 : }
2590 : else
2591 : {
2592 6 : GDALCopyWordsGenericT(pSrcData, nSrcPixelStride, pDstData,
2593 : nDstPixelStride, nWordCount);
2594 : }
2595 62 : }
2596 :
2597 : template <>
2598 6037 : CPL_NOINLINE void GDALCopyWordsT(const uint16_t *const CPL_RESTRICT pSrcData,
2599 : int nSrcPixelStride,
2600 : uint8_t *const CPL_RESTRICT pDstData,
2601 : int nDstPixelStride, GPtrDiff_t nWordCount)
2602 : {
2603 6037 : if (nSrcPixelStride == static_cast<int>(sizeof(*pSrcData)) &&
2604 : nDstPixelStride == static_cast<int>(sizeof(*pDstData)))
2605 : {
2606 5062 : decltype(nWordCount) n = 0;
2607 : #if defined(__SSE4_1__) || defined(USE_NEON_OPTIMIZATIONS)
2608 : const auto xmm_MAX_INT16 = _mm_set1_epi16(32767);
2609 : #else
2610 : // In SSE2, min_epu16 does not exist, so shift from
2611 : // UInt16 to SInt16 to be able to use min_epi16
2612 5062 : const __m128i xmm_UINT16_to_INT16 = _mm_set1_epi16(-32768);
2613 5062 : const __m128i xmm_m255_shifted = _mm_set1_epi16(255 - 32768);
2614 : #endif
2615 71888 : for (; n < nWordCount - 15; n += 16)
2616 : {
2617 133652 : __m128i xmm0 = _mm_loadu_si128(
2618 66826 : reinterpret_cast<const __m128i *>(pSrcData + n));
2619 66826 : __m128i xmm1 = _mm_loadu_si128(
2620 66826 : reinterpret_cast<const __m128i *>(pSrcData + n + 8));
2621 : #if defined(__SSE4_1__) || defined(USE_NEON_OPTIMIZATIONS)
2622 : xmm0 = _mm_min_epu16(xmm0, xmm_MAX_INT16);
2623 : xmm1 = _mm_min_epu16(xmm1, xmm_MAX_INT16);
2624 : #else
2625 66826 : xmm0 = _mm_add_epi16(xmm0, xmm_UINT16_to_INT16);
2626 66826 : xmm1 = _mm_add_epi16(xmm1, xmm_UINT16_to_INT16);
2627 66826 : xmm0 = _mm_min_epi16(xmm0, xmm_m255_shifted);
2628 66826 : xmm1 = _mm_min_epi16(xmm1, xmm_m255_shifted);
2629 66826 : xmm0 = _mm_sub_epi16(xmm0, xmm_UINT16_to_INT16);
2630 66826 : xmm1 = _mm_sub_epi16(xmm1, xmm_UINT16_to_INT16);
2631 : #endif
2632 66826 : xmm0 = _mm_packus_epi16(xmm0, xmm1);
2633 66826 : _mm_storeu_si128(reinterpret_cast<__m128i *>(pDstData + n), xmm0);
2634 : }
2635 16403 : for (; n < nWordCount; n++)
2636 : {
2637 11341 : pDstData[n] =
2638 11341 : pSrcData[n] >= 255 ? 255 : static_cast<uint8_t>(pSrcData[n]);
2639 5062 : }
2640 : }
2641 : else
2642 : {
2643 975 : GDALCopyWordsGenericT(pSrcData, nSrcPixelStride, pDstData,
2644 : nDstPixelStride, nWordCount);
2645 : }
2646 6037 : }
2647 :
2648 : template <>
2649 46 : CPL_NOINLINE void GDALCopyWordsT(const uint16_t *const CPL_RESTRICT pSrcData,
2650 : int nSrcPixelStride,
2651 : int16_t *const CPL_RESTRICT pDstData,
2652 : int nDstPixelStride, GPtrDiff_t nWordCount)
2653 : {
2654 46 : if (nSrcPixelStride == static_cast<int>(sizeof(*pSrcData)) &&
2655 : nDstPixelStride == static_cast<int>(sizeof(*pDstData)))
2656 : {
2657 40 : decltype(nWordCount) n = 0;
2658 : #if defined(__SSE4_1__) || defined(USE_NEON_OPTIMIZATIONS)
2659 : const __m128i xmm_MAX_INT16 = _mm_set1_epi16(32767);
2660 : #else
2661 : // In SSE2, min_epu16 does not exist, so shift from
2662 : // UInt16 to SInt16 to be able to use min_epi16
2663 40 : const __m128i xmm_UINT16_to_INT16 = _mm_set1_epi16(-32768);
2664 40 : const __m128i xmm_32767_shifted = _mm_set1_epi16(32767 - 32768);
2665 : #endif
2666 169 : for (; n < nWordCount - 15; n += 16)
2667 : {
2668 258 : __m128i xmm0 = _mm_loadu_si128(
2669 129 : reinterpret_cast<const __m128i *>(pSrcData + n));
2670 129 : __m128i xmm1 = _mm_loadu_si128(
2671 129 : reinterpret_cast<const __m128i *>(pSrcData + n + 8));
2672 : #if defined(__SSE4_1__) || defined(USE_NEON_OPTIMIZATIONS)
2673 : xmm0 = _mm_min_epu16(xmm0, xmm_MAX_INT16);
2674 : xmm1 = _mm_min_epu16(xmm1, xmm_MAX_INT16);
2675 : #else
2676 129 : xmm0 = _mm_add_epi16(xmm0, xmm_UINT16_to_INT16);
2677 129 : xmm1 = _mm_add_epi16(xmm1, xmm_UINT16_to_INT16);
2678 129 : xmm0 = _mm_min_epi16(xmm0, xmm_32767_shifted);
2679 129 : xmm1 = _mm_min_epi16(xmm1, xmm_32767_shifted);
2680 129 : xmm0 = _mm_sub_epi16(xmm0, xmm_UINT16_to_INT16);
2681 129 : xmm1 = _mm_sub_epi16(xmm1, xmm_UINT16_to_INT16);
2682 : #endif
2683 129 : _mm_storeu_si128(reinterpret_cast<__m128i *>(pDstData + n), xmm0);
2684 129 : _mm_storeu_si128(reinterpret_cast<__m128i *>(pDstData + n + 8),
2685 : xmm1);
2686 : }
2687 191 : for (; n < nWordCount; n++)
2688 : {
2689 282 : pDstData[n] = pSrcData[n] >= 32767
2690 : ? 32767
2691 131 : : static_cast<int16_t>(pSrcData[n]);
2692 40 : }
2693 : }
2694 : else
2695 : {
2696 6 : GDALCopyWordsGenericT(pSrcData, nSrcPixelStride, pDstData,
2697 : nDstPixelStride, nWordCount);
2698 : }
2699 46 : }
2700 :
2701 : template <>
2702 136 : CPL_NOINLINE void GDALCopyWordsT(const int16_t *const CPL_RESTRICT pSrcData,
2703 : int nSrcPixelStride,
2704 : uint16_t *const CPL_RESTRICT pDstData,
2705 : int nDstPixelStride, GPtrDiff_t nWordCount)
2706 : {
2707 136 : if (nSrcPixelStride == static_cast<int>(sizeof(*pSrcData)) &&
2708 : nDstPixelStride == static_cast<int>(sizeof(*pDstData)))
2709 : {
2710 93 : decltype(nWordCount) n = 0;
2711 93 : const __m128i xmm_zero = _mm_setzero_si128();
2712 278 : for (; n < nWordCount - 15; n += 16)
2713 : {
2714 370 : __m128i xmm0 = _mm_loadu_si128(
2715 185 : reinterpret_cast<const __m128i *>(pSrcData + n));
2716 185 : __m128i xmm1 = _mm_loadu_si128(
2717 185 : reinterpret_cast<const __m128i *>(pSrcData + n + 8));
2718 185 : xmm0 = _mm_max_epi16(xmm0, xmm_zero);
2719 185 : xmm1 = _mm_max_epi16(xmm1, xmm_zero);
2720 185 : _mm_storeu_si128(reinterpret_cast<__m128i *>(pDstData + n), xmm0);
2721 185 : _mm_storeu_si128(reinterpret_cast<__m128i *>(pDstData + n + 8),
2722 : xmm1);
2723 : }
2724 471 : for (; n < nWordCount; n++)
2725 : {
2726 378 : pDstData[n] =
2727 378 : pSrcData[n] < 0 ? 0 : static_cast<uint16_t>(pSrcData[n]);
2728 93 : }
2729 : }
2730 : else
2731 : {
2732 43 : GDALCopyWordsGenericT(pSrcData, nSrcPixelStride, pDstData,
2733 : nDstPixelStride, nWordCount);
2734 : }
2735 136 : }
2736 :
2737 : #if defined(__SSE4_1__) || defined(USE_NEON_OPTIMIZATIONS)
2738 :
2739 : template <>
2740 : CPL_NOINLINE void GDALCopyWordsT(const uint32_t *const CPL_RESTRICT pSrcData,
2741 : int nSrcPixelStride,
2742 : int32_t *const CPL_RESTRICT pDstData,
2743 : int nDstPixelStride, GPtrDiff_t nWordCount)
2744 : {
2745 : if (nSrcPixelStride == static_cast<int>(sizeof(*pSrcData)) &&
2746 : nDstPixelStride == static_cast<int>(sizeof(*pDstData)))
2747 : {
2748 : decltype(nWordCount) n = 0;
2749 : const __m128i xmm_MAX_INT = _mm_set1_epi32(INT_MAX);
2750 : for (; n < nWordCount - 8; n += 7)
2751 : {
2752 : __m128i xmm0 = _mm_loadu_si128(
2753 : reinterpret_cast<const __m128i *>(pSrcData + n));
2754 : __m128i xmm1 = _mm_loadu_si128(
2755 : reinterpret_cast<const __m128i *>(pSrcData + n + 4));
2756 : xmm0 = _mm_min_epu32(xmm0, xmm_MAX_INT);
2757 : xmm1 = _mm_min_epu32(xmm1, xmm_MAX_INT);
2758 : _mm_storeu_si128(reinterpret_cast<__m128i *>(pDstData + n), xmm0);
2759 : _mm_storeu_si128(reinterpret_cast<__m128i *>(pDstData + n + 4),
2760 : xmm1);
2761 : }
2762 : for (; n < nWordCount; n++)
2763 : {
2764 : pDstData[n] = pSrcData[n] >= INT_MAX
2765 : ? INT_MAX
2766 : : static_cast<int32_t>(pSrcData[n]);
2767 : }
2768 : }
2769 : else
2770 : {
2771 : GDALCopyWordsGenericT(pSrcData, nSrcPixelStride, pDstData,
2772 : nDstPixelStride, nWordCount);
2773 : }
2774 : }
2775 :
2776 : template <>
2777 : CPL_NOINLINE void GDALCopyWordsT(const int32_t *const CPL_RESTRICT pSrcData,
2778 : int nSrcPixelStride,
2779 : uint32_t *const CPL_RESTRICT pDstData,
2780 : int nDstPixelStride, GPtrDiff_t nWordCount)
2781 : {
2782 : if (nSrcPixelStride == static_cast<int>(sizeof(*pSrcData)) &&
2783 : nDstPixelStride == static_cast<int>(sizeof(*pDstData)))
2784 : {
2785 : decltype(nWordCount) n = 0;
2786 : const __m128i xmm_zero = _mm_setzero_si128();
2787 : for (; n < nWordCount - 7; n += 8)
2788 : {
2789 : __m128i xmm0 = _mm_loadu_si128(
2790 : reinterpret_cast<const __m128i *>(pSrcData + n));
2791 : __m128i xmm1 = _mm_loadu_si128(
2792 : reinterpret_cast<const __m128i *>(pSrcData + n + 4));
2793 : xmm0 = _mm_max_epi32(xmm0, xmm_zero);
2794 : xmm1 = _mm_max_epi32(xmm1, xmm_zero);
2795 : _mm_storeu_si128(reinterpret_cast<__m128i *>(pDstData + n), xmm0);
2796 : _mm_storeu_si128(reinterpret_cast<__m128i *>(pDstData + n + 4),
2797 : xmm1);
2798 : }
2799 : for (; n < nWordCount; n++)
2800 : {
2801 : pDstData[n] =
2802 : pSrcData[n] < 0 ? 0 : static_cast<uint32_t>(pSrcData[n]);
2803 : }
2804 : }
2805 : else
2806 : {
2807 : GDALCopyWordsGenericT(pSrcData, nSrcPixelStride, pDstData,
2808 : nDstPixelStride, nWordCount);
2809 : }
2810 : }
2811 :
2812 : #endif // defined(__SSE4_1__) || defined(USE_NEON_OPTIMIZATIONS)
2813 :
2814 : template <>
2815 403 : CPL_NOINLINE void GDALCopyWordsT(const uint16_t *const CPL_RESTRICT pSrcData,
2816 : int nSrcPixelStride,
2817 : float *const CPL_RESTRICT pDstData,
2818 : int nDstPixelStride, GPtrDiff_t nWordCount)
2819 : {
2820 403 : if (nSrcPixelStride == static_cast<int>(sizeof(*pSrcData)) &&
2821 : nDstPixelStride == static_cast<int>(sizeof(*pDstData)))
2822 : {
2823 397 : decltype(nWordCount) n = 0;
2824 397 : const __m128i xmm_zero = _mm_setzero_si128();
2825 397 : GByte *CPL_RESTRICT pabyDstDataPtr =
2826 : reinterpret_cast<GByte *>(pDstData);
2827 1688 : for (; n < nWordCount - 7; n += 8)
2828 : {
2829 1291 : __m128i xmm = _mm_loadu_si128(
2830 1291 : reinterpret_cast<const __m128i *>(pSrcData + n));
2831 1291 : __m128i xmm0 = _mm_unpacklo_epi16(xmm, xmm_zero);
2832 1291 : __m128i xmm1 = _mm_unpackhi_epi16(xmm, xmm_zero);
2833 1291 : __m128 xmm0_f = _mm_cvtepi32_ps(xmm0);
2834 1291 : __m128 xmm1_f = _mm_cvtepi32_ps(xmm1);
2835 1291 : _mm_storeu_ps(reinterpret_cast<float *>(pabyDstDataPtr + n * 4),
2836 : xmm0_f);
2837 : _mm_storeu_ps(
2838 1291 : reinterpret_cast<float *>(pabyDstDataPtr + n * 4 + 16), xmm1_f);
2839 : }
2840 1415 : for (; n < nWordCount; n++)
2841 : {
2842 1018 : pDstData[n] = pSrcData[n];
2843 397 : }
2844 : }
2845 : else
2846 : {
2847 6 : GDALCopyWordsGenericT(pSrcData, nSrcPixelStride, pDstData,
2848 : nDstPixelStride, nWordCount);
2849 : }
2850 403 : }
2851 :
2852 : template <>
2853 1076640 : CPL_NOINLINE void GDALCopyWordsT(const int16_t *const CPL_RESTRICT pSrcData,
2854 : int nSrcPixelStride,
2855 : float *const CPL_RESTRICT pDstData,
2856 : int nDstPixelStride, GPtrDiff_t nWordCount)
2857 : {
2858 1076640 : if (nSrcPixelStride == static_cast<int>(sizeof(*pSrcData)) &&
2859 : nDstPixelStride == static_cast<int>(sizeof(*pDstData)))
2860 : {
2861 86742 : decltype(nWordCount) n = 0;
2862 86742 : GByte *CPL_RESTRICT pabyDstDataPtr =
2863 : reinterpret_cast<GByte *>(pDstData);
2864 586119 : for (; n < nWordCount - 7; n += 8)
2865 : {
2866 499377 : __m128i xmm = _mm_loadu_si128(
2867 499377 : reinterpret_cast<const __m128i *>(pSrcData + n));
2868 499377 : const auto sign = _mm_srai_epi16(xmm, 15);
2869 499377 : __m128i xmm0 = _mm_unpacklo_epi16(xmm, sign);
2870 499377 : __m128i xmm1 = _mm_unpackhi_epi16(xmm, sign);
2871 499377 : __m128 xmm0_f = _mm_cvtepi32_ps(xmm0);
2872 499377 : __m128 xmm1_f = _mm_cvtepi32_ps(xmm1);
2873 499377 : _mm_storeu_ps(reinterpret_cast<float *>(pabyDstDataPtr + n * 4),
2874 : xmm0_f);
2875 : _mm_storeu_ps(
2876 499377 : reinterpret_cast<float *>(pabyDstDataPtr + n * 4 + 16), xmm1_f);
2877 : }
2878 253882 : for (; n < nWordCount; n++)
2879 : {
2880 167140 : pDstData[n] = pSrcData[n];
2881 86742 : }
2882 : }
2883 : else
2884 : {
2885 989901 : GDALCopyWordsGenericT(pSrcData, nSrcPixelStride, pDstData,
2886 : nDstPixelStride, nWordCount);
2887 : }
2888 1076640 : }
2889 :
2890 : template <>
2891 449 : CPL_NOINLINE void GDALCopyWordsT(const uint16_t *const CPL_RESTRICT pSrcData,
2892 : int nSrcPixelStride,
2893 : double *const CPL_RESTRICT pDstData,
2894 : int nDstPixelStride, GPtrDiff_t nWordCount)
2895 : {
2896 449 : if (nSrcPixelStride == static_cast<int>(sizeof(*pSrcData)) &&
2897 : nDstPixelStride == static_cast<int>(sizeof(*pDstData)))
2898 : {
2899 313 : decltype(nWordCount) n = 0;
2900 313 : const __m128i xmm_zero = _mm_setzero_si128();
2901 313 : GByte *CPL_RESTRICT pabyDstDataPtr =
2902 : reinterpret_cast<GByte *>(pDstData);
2903 829 : for (; n < nWordCount - 7; n += 8)
2904 : {
2905 516 : __m128i xmm = _mm_loadu_si128(
2906 516 : reinterpret_cast<const __m128i *>(pSrcData + n));
2907 516 : __m128i xmm0 = _mm_unpacklo_epi16(xmm, xmm_zero);
2908 516 : __m128i xmm1 = _mm_unpackhi_epi16(xmm, xmm_zero);
2909 :
2910 516 : __m128d xmm0_low_d = _mm_cvtepi32_pd(xmm0);
2911 516 : __m128d xmm1_low_d = _mm_cvtepi32_pd(xmm1);
2912 516 : xmm0 = _mm_srli_si128(xmm0, 8);
2913 516 : xmm1 = _mm_srli_si128(xmm1, 8);
2914 516 : __m128d xmm0_high_d = _mm_cvtepi32_pd(xmm0);
2915 516 : __m128d xmm1_high_d = _mm_cvtepi32_pd(xmm1);
2916 :
2917 516 : _mm_storeu_pd(reinterpret_cast<double *>(pabyDstDataPtr + n * 8),
2918 : xmm0_low_d);
2919 : _mm_storeu_pd(
2920 516 : reinterpret_cast<double *>(pabyDstDataPtr + n * 8 + 16),
2921 : xmm0_high_d);
2922 : _mm_storeu_pd(
2923 516 : reinterpret_cast<double *>(pabyDstDataPtr + n * 8 + 32),
2924 : xmm1_low_d);
2925 : _mm_storeu_pd(
2926 516 : reinterpret_cast<double *>(pabyDstDataPtr + n * 8 + 48),
2927 : xmm1_high_d);
2928 : }
2929 1082 : for (; n < nWordCount; n++)
2930 : {
2931 769 : pDstData[n] = pSrcData[n];
2932 313 : }
2933 : }
2934 : else
2935 : {
2936 136 : GDALCopyWordsGenericT(pSrcData, nSrcPixelStride, pDstData,
2937 : nDstPixelStride, nWordCount);
2938 : }
2939 449 : }
2940 :
2941 : template <>
2942 4923280 : CPL_NOINLINE void GDALCopyWordsT(const int16_t *const CPL_RESTRICT pSrcData,
2943 : int nSrcPixelStride,
2944 : double *const CPL_RESTRICT pDstData,
2945 : int nDstPixelStride, GPtrDiff_t nWordCount)
2946 : {
2947 4923280 : if (nSrcPixelStride == static_cast<int>(sizeof(*pSrcData)) &&
2948 : nDstPixelStride == static_cast<int>(sizeof(*pDstData)))
2949 : {
2950 34874 : decltype(nWordCount) n = 0;
2951 34874 : GByte *CPL_RESTRICT pabyDstDataPtr =
2952 : reinterpret_cast<GByte *>(pDstData);
2953 403828 : for (; n < nWordCount - 7; n += 8)
2954 : {
2955 368954 : __m128i xmm = _mm_loadu_si128(
2956 368954 : reinterpret_cast<const __m128i *>(pSrcData + n));
2957 368954 : const auto sign = _mm_srai_epi16(xmm, 15);
2958 368954 : __m128i xmm0 = _mm_unpacklo_epi16(xmm, sign);
2959 368954 : __m128i xmm1 = _mm_unpackhi_epi16(xmm, sign);
2960 :
2961 368954 : __m128d xmm0_low_d = _mm_cvtepi32_pd(xmm0);
2962 368954 : __m128d xmm1_low_d = _mm_cvtepi32_pd(xmm1);
2963 368954 : xmm0 = _mm_srli_si128(xmm0, 8);
2964 368954 : xmm1 = _mm_srli_si128(xmm1, 8);
2965 368954 : __m128d xmm0_high_d = _mm_cvtepi32_pd(xmm0);
2966 368954 : __m128d xmm1_high_d = _mm_cvtepi32_pd(xmm1);
2967 :
2968 368954 : _mm_storeu_pd(reinterpret_cast<double *>(pabyDstDataPtr + n * 8),
2969 : xmm0_low_d);
2970 : _mm_storeu_pd(
2971 368954 : reinterpret_cast<double *>(pabyDstDataPtr + n * 8 + 16),
2972 : xmm0_high_d);
2973 : _mm_storeu_pd(
2974 368954 : reinterpret_cast<double *>(pabyDstDataPtr + n * 8 + 32),
2975 : xmm1_low_d);
2976 : _mm_storeu_pd(
2977 368954 : reinterpret_cast<double *>(pabyDstDataPtr + n * 8 + 48),
2978 : xmm1_high_d);
2979 : }
2980 255934 : for (; n < nWordCount; n++)
2981 : {
2982 221060 : pDstData[n] = pSrcData[n];
2983 34874 : }
2984 : }
2985 : else
2986 : {
2987 4888400 : GDALCopyWordsGenericT(pSrcData, nSrcPixelStride, pDstData,
2988 : nDstPixelStride, nWordCount);
2989 : }
2990 4923280 : }
2991 :
2992 : // ---- AVX2 helpers for int32 narrowing (runtime dispatch) ----
2993 :
2994 : #if defined(HAVE_AVX2_DISPATCH)
2995 : #if !defined(_MSC_VER)
2996 : __attribute__((target("avx2")))
2997 : #endif
2998 12723 : static void GDALCopyWordsInt32ToUInt8_AVX2(const int32_t *CPL_RESTRICT pSrc,
2999 : uint8_t *CPL_RESTRICT pDst,
3000 : GPtrDiff_t nWordCount)
3001 : {
3002 12723 : const __m256i permuteIdx = _mm256_setr_epi32(0, 4, 1, 5, 2, 6, 3, 7);
3003 12723 : GPtrDiff_t n = 0;
3004 958119 : for (; n < nWordCount - 31; n += 32)
3005 : {
3006 : __m256i v0 =
3007 945396 : _mm256_loadu_si256(reinterpret_cast<const __m256i *>(pSrc + n));
3008 : __m256i v1 =
3009 945396 : _mm256_loadu_si256(reinterpret_cast<const __m256i *>(pSrc + n + 8));
3010 945396 : __m256i v2 = _mm256_loadu_si256(
3011 945396 : reinterpret_cast<const __m256i *>(pSrc + n + 16));
3012 945396 : __m256i v3 = _mm256_loadu_si256(
3013 945396 : reinterpret_cast<const __m256i *>(pSrc + n + 24));
3014 : // Clamp to [0, 255]
3015 : // Pack int32 -> int16 -> uint8, then fix cross-lane ordering
3016 945396 : __m256i ab16 = _mm256_packs_epi32(v0, v1);
3017 945396 : __m256i cd16 = _mm256_packs_epi32(v2, v3);
3018 945396 : __m256i bytes = _mm256_packus_epi16(ab16, cd16);
3019 945396 : bytes = _mm256_permutevar8x32_epi32(bytes, permuteIdx);
3020 945396 : _mm256_storeu_si256(reinterpret_cast<__m256i *>(pDst + n), bytes);
3021 : }
3022 68589 : for (; n < nWordCount; n++)
3023 : {
3024 70955 : pDst[n] = pSrc[n] <= 0 ? 0
3025 15089 : : pSrc[n] >= 255 ? 255
3026 1075 : : static_cast<uint8_t>(pSrc[n]);
3027 : }
3028 12723 : }
3029 :
3030 : #if !defined(_MSC_VER)
3031 : __attribute__((target("avx2")))
3032 : #endif
3033 10277 : static void GDALCopyWordsInt32ToUInt16_AVX2(const int32_t *CPL_RESTRICT pSrc,
3034 : uint16_t *CPL_RESTRICT pDst,
3035 : GPtrDiff_t nWordCount)
3036 : {
3037 : // _mm256_packus_epi32(v0, v1) produces per-lane interleaved result:
3038 : // [v0_lo4, v1_lo4, v0_hi4, v1_hi4] (in uint16 pairs per 32-bit lane)
3039 : // Permute to deinterleave: all v0 values first, then all v1 values
3040 10277 : const __m256i permuteIdx = _mm256_setr_epi32(0, 1, 4, 5, 2, 3, 6, 7);
3041 10277 : GPtrDiff_t n = 0;
3042 670572 : for (; n < nWordCount - 15; n += 16)
3043 : {
3044 : __m256i v0 =
3045 660295 : _mm256_loadu_si256(reinterpret_cast<const __m256i *>(pSrc + n));
3046 : __m256i v1 =
3047 1320590 : _mm256_loadu_si256(reinterpret_cast<const __m256i *>(pSrc + n + 8));
3048 : // Clamp to [0, 65535]: _mm256_packus_epi32 saturates uint
3049 660295 : __m256i packed = _mm256_packus_epi32(v0, v1);
3050 : // Fix cross-lane interleave from packus
3051 660295 : packed = _mm256_permutevar8x32_epi32(packed, permuteIdx);
3052 660295 : _mm256_storeu_si256(reinterpret_cast<__m256i *>(pDst + n), packed);
3053 : }
3054 163928 : for (; n < nWordCount; n++)
3055 : {
3056 307282 : pDst[n] = pSrc[n] <= 0 ? 0
3057 153631 : : pSrc[n] >= 65535 ? 65535
3058 153599 : : static_cast<uint16_t>(pSrc[n]);
3059 : }
3060 10277 : }
3061 : #endif // HAVE_AVX2_DISPATCH
3062 :
3063 : // ---- int32 -> uint8 with clamping to [0, 255] ----
3064 : template <>
3065 12837 : CPL_NOINLINE void GDALCopyWordsT(const int32_t *const CPL_RESTRICT pSrcData,
3066 : int nSrcPixelStride,
3067 : uint8_t *const CPL_RESTRICT pDstData,
3068 : int nDstPixelStride, GPtrDiff_t nWordCount)
3069 : {
3070 12837 : if (nSrcPixelStride == static_cast<int>(sizeof(*pSrcData)) &&
3071 : nDstPixelStride == static_cast<int>(sizeof(*pDstData)))
3072 : {
3073 : #if defined(HAVE_AVX2_DISPATCH)
3074 12723 : if (CPLHaveRuntimeAVX2())
3075 : {
3076 12723 : GDALCopyWordsInt32ToUInt8_AVX2(pSrcData, pDstData, nWordCount);
3077 12723 : return;
3078 : }
3079 : #endif
3080 : #ifdef HAVE_SSE2
3081 : // SSE2 path: 16 pixels per iteration
3082 0 : decltype(nWordCount) n = 0;
3083 0 : for (; n < nWordCount - 15; n += 16)
3084 : {
3085 0 : __m128i v0 = _mm_loadu_si128(
3086 0 : reinterpret_cast<const __m128i *>(pSrcData + n));
3087 0 : __m128i v1 = _mm_loadu_si128(
3088 0 : reinterpret_cast<const __m128i *>(pSrcData + n + 4));
3089 0 : __m128i v2 = _mm_loadu_si128(
3090 0 : reinterpret_cast<const __m128i *>(pSrcData + n + 8));
3091 0 : __m128i v3 = _mm_loadu_si128(
3092 0 : reinterpret_cast<const __m128i *>(pSrcData + n + 12));
3093 : // Pack int32->int16 with signed saturation to [-32768,32767] range
3094 0 : __m128i lo16 = _mm_packs_epi32(v0, v1);
3095 0 : __m128i hi16 = _mm_packs_epi32(v2, v3);
3096 : // Pack int16->uint8 with unsigned saturation to [0,255] range
3097 0 : __m128i bytes = _mm_packus_epi16(lo16, hi16);
3098 0 : _mm_storeu_si128(reinterpret_cast<__m128i *>(pDstData + n), bytes);
3099 : }
3100 0 : for (; n < nWordCount; n++)
3101 : #else
3102 : for (decltype(nWordCount) n = 0; n < nWordCount; n++)
3103 : #endif
3104 : {
3105 0 : pDstData[n] = pSrcData[n] <= 0 ? 0
3106 0 : : pSrcData[n] >= 255
3107 : ? 255
3108 0 : : static_cast<uint8_t>(pSrcData[n]);
3109 0 : }
3110 : }
3111 : else
3112 : {
3113 114 : GDALCopyWordsGenericT(pSrcData, nSrcPixelStride, pDstData,
3114 : nDstPixelStride, nWordCount);
3115 : }
3116 : }
3117 :
3118 : // ---- int32 -> uint16 with clamping to [0, 65535] ----
3119 : template <>
3120 10322 : CPL_NOINLINE void GDALCopyWordsT(const int32_t *const CPL_RESTRICT pSrcData,
3121 : int nSrcPixelStride,
3122 : uint16_t *const CPL_RESTRICT pDstData,
3123 : int nDstPixelStride, GPtrDiff_t nWordCount)
3124 : {
3125 10322 : if (nSrcPixelStride == static_cast<int>(sizeof(*pSrcData)) &&
3126 : nDstPixelStride == static_cast<int>(sizeof(*pDstData)))
3127 : {
3128 : #if defined(HAVE_AVX2_DISPATCH)
3129 10277 : if (CPLHaveRuntimeAVX2())
3130 : {
3131 10277 : GDALCopyWordsInt32ToUInt16_AVX2(pSrcData, pDstData, nWordCount);
3132 10277 : return;
3133 : }
3134 : #endif
3135 0 : decltype(nWordCount) n = 0;
3136 : #if defined(__SSE4_1__) || defined(USE_NEON_OPTIMIZATIONS)
3137 : // SSE4.1: _mm_packus_epi32 directly handles uint saturation
3138 : for (; n < nWordCount - 7; n += 8)
3139 : {
3140 : __m128i v0 = _mm_loadu_si128(
3141 : reinterpret_cast<const __m128i *>(pSrcData + n));
3142 : __m128i v1 = _mm_loadu_si128(
3143 : reinterpret_cast<const __m128i *>(pSrcData + n + 4));
3144 : __m128i packed = _mm_packus_epi32(v0, v1);
3145 : _mm_storeu_si128(reinterpret_cast<__m128i *>(pDstData + n), packed);
3146 : }
3147 : #else
3148 : // SSE2: clamp to [0, 65535], bias to signed range, pack, unbias
3149 0 : const __m128i xmm_65535 = _mm_set1_epi32(65535);
3150 0 : const __m128i xmm_bias32 = _mm_set1_epi32(32768);
3151 0 : const __m128i xmm_bias16 = _mm_set1_epi16(-32768);
3152 0 : for (; n < nWordCount - 7; n += 8)
3153 : {
3154 0 : __m128i v0 = _mm_loadu_si128(
3155 0 : reinterpret_cast<const __m128i *>(pSrcData + n));
3156 0 : __m128i v1 = _mm_loadu_si128(
3157 0 : reinterpret_cast<const __m128i *>(pSrcData + n + 4));
3158 : // max(v, 0)
3159 0 : v0 = _mm_andnot_si128(_mm_srai_epi32(v0, 31), v0);
3160 0 : v1 = _mm_andnot_si128(_mm_srai_epi32(v1, 31), v1);
3161 : // min(v, 65535)
3162 0 : __m128i gt0 = _mm_cmpgt_epi32(v0, xmm_65535);
3163 0 : __m128i gt1 = _mm_cmpgt_epi32(v1, xmm_65535);
3164 0 : v0 = _mm_or_si128(_mm_andnot_si128(gt0, v0),
3165 : _mm_and_si128(gt0, xmm_65535));
3166 0 : v1 = _mm_or_si128(_mm_andnot_si128(gt1, v1),
3167 : _mm_and_si128(gt1, xmm_65535));
3168 : // Shift [0, 65535] -> [-32768, 32767] for _mm_packs_epi32
3169 0 : v0 = _mm_sub_epi32(v0, xmm_bias32);
3170 0 : v1 = _mm_sub_epi32(v1, xmm_bias32);
3171 0 : __m128i packed = _mm_packs_epi32(v0, v1);
3172 : // Shift back: sub_epi16(x, -32768) == add 32768 (mod 2^16)
3173 0 : packed = _mm_sub_epi16(packed, xmm_bias16);
3174 0 : _mm_storeu_si128(reinterpret_cast<__m128i *>(pDstData + n), packed);
3175 : }
3176 : #endif
3177 0 : for (; n < nWordCount; n++)
3178 : {
3179 0 : pDstData[n] = pSrcData[n] <= 0 ? 0
3180 0 : : pSrcData[n] >= 65535
3181 : ? 65535
3182 0 : : static_cast<uint16_t>(pSrcData[n]);
3183 0 : }
3184 : }
3185 : else
3186 : {
3187 45 : GDALCopyWordsGenericT(pSrcData, nSrcPixelStride, pDstData,
3188 : nDstPixelStride, nWordCount);
3189 : }
3190 : }
3191 :
3192 : #endif // HAVE_SSE2
3193 :
3194 : template <>
3195 4437360 : CPL_NOINLINE void GDALCopyWordsT(const double *const CPL_RESTRICT pSrcData,
3196 : int nSrcPixelStride,
3197 : GByte *const CPL_RESTRICT pDstData,
3198 : int nDstPixelStride, GPtrDiff_t nWordCount)
3199 : {
3200 4437360 : GDALCopyWordsT_8atatime(pSrcData, nSrcPixelStride, pDstData,
3201 : nDstPixelStride, nWordCount);
3202 4437360 : }
3203 :
3204 : template <>
3205 38388 : CPL_NOINLINE void GDALCopyWordsT(const double *const CPL_RESTRICT pSrcData,
3206 : int nSrcPixelStride,
3207 : GUInt16 *const CPL_RESTRICT pDstData,
3208 : int nDstPixelStride, GPtrDiff_t nWordCount)
3209 : {
3210 38388 : GDALCopyWordsT_8atatime(pSrcData, nSrcPixelStride, pDstData,
3211 : nDstPixelStride, nWordCount);
3212 38388 : }
3213 :
3214 : template <>
3215 55871 : CPL_NOINLINE void GDALCopyWordsT(const float *const CPL_RESTRICT pSrcData,
3216 : int nSrcPixelStride,
3217 : double *const CPL_RESTRICT pDstData,
3218 : int nDstPixelStride, GPtrDiff_t nWordCount)
3219 : {
3220 55871 : GDALCopyWordsT_8atatime(pSrcData, nSrcPixelStride, pDstData,
3221 : nDstPixelStride, nWordCount);
3222 55871 : }
3223 :
3224 : template <>
3225 122866 : CPL_NOINLINE void GDALCopyWordsT(const double *const CPL_RESTRICT pSrcData,
3226 : int nSrcPixelStride,
3227 : float *const CPL_RESTRICT pDstData,
3228 : int nDstPixelStride, GPtrDiff_t nWordCount)
3229 : {
3230 122866 : GDALCopyWordsT_8atatime(pSrcData, nSrcPixelStride, pDstData,
3231 : nDstPixelStride, nWordCount);
3232 122866 : }
3233 :
3234 : template <>
3235 412 : CPL_NOINLINE void GDALCopyWordsT(const GFloat16 *const CPL_RESTRICT pSrcData,
3236 : int nSrcPixelStride,
3237 : float *const CPL_RESTRICT pDstData,
3238 : int nDstPixelStride, GPtrDiff_t nWordCount)
3239 : {
3240 412 : GDALCopyWordsT_8atatime(pSrcData, nSrcPixelStride, pDstData,
3241 : nDstPixelStride, nWordCount);
3242 412 : }
3243 :
3244 : template <>
3245 544 : CPL_NOINLINE void GDALCopyWordsT(const GFloat16 *const CPL_RESTRICT pSrcData,
3246 : int nSrcPixelStride,
3247 : double *const CPL_RESTRICT pDstData,
3248 : int nDstPixelStride, GPtrDiff_t nWordCount)
3249 : {
3250 544 : GDALCopyWordsT_8atatime(pSrcData, nSrcPixelStride, pDstData,
3251 : nDstPixelStride, nWordCount);
3252 544 : }
3253 :
3254 : template <>
3255 324209 : CPL_NOINLINE void GDALCopyWordsT(const float *const CPL_RESTRICT pSrcData,
3256 : int nSrcPixelStride,
3257 : GByte *const CPL_RESTRICT pDstData,
3258 : int nDstPixelStride, GPtrDiff_t nWordCount)
3259 : {
3260 324209 : GDALCopyWordsT_8atatime(pSrcData, nSrcPixelStride, pDstData,
3261 : nDstPixelStride, nWordCount);
3262 324209 : }
3263 :
3264 : template <>
3265 55 : CPL_NOINLINE void GDALCopyWordsT(const float *const CPL_RESTRICT pSrcData,
3266 : int nSrcPixelStride,
3267 : GInt8 *const CPL_RESTRICT pDstData,
3268 : int nDstPixelStride, GPtrDiff_t nWordCount)
3269 : {
3270 55 : GDALCopyWordsT_8atatime(pSrcData, nSrcPixelStride, pDstData,
3271 : nDstPixelStride, nWordCount);
3272 55 : }
3273 :
3274 : template <>
3275 15785 : CPL_NOINLINE void GDALCopyWordsT(const float *const CPL_RESTRICT pSrcData,
3276 : int nSrcPixelStride,
3277 : GInt16 *const CPL_RESTRICT pDstData,
3278 : int nDstPixelStride, GPtrDiff_t nWordCount)
3279 : {
3280 15785 : GDALCopyWordsT_8atatime(pSrcData, nSrcPixelStride, pDstData,
3281 : nDstPixelStride, nWordCount);
3282 15785 : }
3283 :
3284 : template <>
3285 61713 : CPL_NOINLINE void GDALCopyWordsT(const float *const CPL_RESTRICT pSrcData,
3286 : int nSrcPixelStride,
3287 : GUInt16 *const CPL_RESTRICT pDstData,
3288 : int nDstPixelStride, GPtrDiff_t nWordCount)
3289 : {
3290 61713 : GDALCopyWordsT_8atatime(pSrcData, nSrcPixelStride, pDstData,
3291 : nDstPixelStride, nWordCount);
3292 61713 : }
3293 :
3294 : template <>
3295 43985 : CPL_NOINLINE void GDALCopyWordsT(const float *const CPL_RESTRICT pSrcData,
3296 : int nSrcPixelStride,
3297 : GInt32 *const CPL_RESTRICT pDstData,
3298 : int nDstPixelStride, GPtrDiff_t nWordCount)
3299 : {
3300 43985 : GDALCopyWordsT_8atatime(pSrcData, nSrcPixelStride, pDstData,
3301 : nDstPixelStride, nWordCount);
3302 43985 : }
3303 :
3304 : template <>
3305 72 : CPL_NOINLINE void GDALCopyWordsT(const float *const CPL_RESTRICT pSrcData,
3306 : int nSrcPixelStride,
3307 : GFloat16 *const CPL_RESTRICT pDstData,
3308 : int nDstPixelStride, GPtrDiff_t nWordCount)
3309 : {
3310 72 : GDALCopyWordsT_8atatime(pSrcData, nSrcPixelStride, pDstData,
3311 : nDstPixelStride, nWordCount);
3312 72 : }
3313 :
3314 : template <>
3315 63 : CPL_NOINLINE void GDALCopyWordsT(const double *const CPL_RESTRICT pSrcData,
3316 : int nSrcPixelStride,
3317 : GFloat16 *const CPL_RESTRICT pDstData,
3318 : int nDstPixelStride, GPtrDiff_t nWordCount)
3319 : {
3320 63 : GDALCopyWordsT_8atatime(pSrcData, nSrcPixelStride, pDstData,
3321 : nDstPixelStride, nWordCount);
3322 63 : }
3323 :
3324 : /************************************************************************/
3325 : /* GDALCopyWordsComplexT() */
3326 : /************************************************************************/
3327 : /**
3328 : * Template function, used to copy data from pSrcData into buffer
3329 : * pDstData, with stride nSrcPixelStride in the source data and
3330 : * stride nDstPixelStride in the destination data. Deals with the
3331 : * complex case, where input is complex and output is complex.
3332 : *
3333 : * @param pSrcData the source data buffer
3334 : * @param nSrcPixelStride the stride, in the buffer pSrcData for pixels
3335 : * of interest.
3336 : * @param pDstData the destination buffer.
3337 : * @param nDstPixelStride the stride in the buffer pDstData for pixels of
3338 : * interest.
3339 : * @param nWordCount the total number of pixel words to copy
3340 : *
3341 : */
3342 : template <class Tin, class Tout>
3343 98788 : inline void GDALCopyWordsComplexT(const Tin *const CPL_RESTRICT pSrcData,
3344 : int nSrcPixelStride,
3345 : Tout *const CPL_RESTRICT pDstData,
3346 : int nDstPixelStride, GPtrDiff_t nWordCount)
3347 : {
3348 98788 : decltype(nWordCount) nDstOffset = 0;
3349 98788 : const char *const pSrcDataPtr = reinterpret_cast<const char *>(pSrcData);
3350 98788 : char *const pDstDataPtr = reinterpret_cast<char *>(pDstData);
3351 :
3352 5631239 : for (decltype(nWordCount) n = 0; n < nWordCount; n++)
3353 : {
3354 5532446 : const Tin *const pPixelIn =
3355 5532446 : reinterpret_cast<const Tin *>(pSrcDataPtr + n * nSrcPixelStride);
3356 5532446 : Tout *const pPixelOut =
3357 5532446 : reinterpret_cast<Tout *>(pDstDataPtr + nDstOffset);
3358 :
3359 5532446 : GDALCopyWord(pPixelIn[0], pPixelOut[0]);
3360 5532446 : GDALCopyWord(pPixelIn[1], pPixelOut[1]);
3361 :
3362 5532446 : nDstOffset += nDstPixelStride;
3363 : }
3364 98788 : }
3365 :
3366 : /************************************************************************/
3367 : /* GDALCopyWordsComplexOutT() */
3368 : /************************************************************************/
3369 : /**
3370 : * Template function, used to copy data from pSrcData into buffer
3371 : * pDstData, with stride nSrcPixelStride in the source data and
3372 : * stride nDstPixelStride in the destination data. Deals with the
3373 : * case where the value is real coming in, but complex going out.
3374 : *
3375 : * @param pSrcData the source data buffer
3376 : * @param nSrcPixelStride the stride, in the buffer pSrcData for pixels
3377 : * of interest, in bytes.
3378 : * @param pDstData the destination buffer.
3379 : * @param nDstPixelStride the stride in the buffer pDstData for pixels of
3380 : * interest, in bytes.
3381 : * @param nWordCount the total number of pixel words to copy
3382 : *
3383 : */
3384 : template <class Tin, class Tout>
3385 4762 : inline void GDALCopyWordsComplexOutT(const Tin *const CPL_RESTRICT pSrcData,
3386 : int nSrcPixelStride,
3387 : Tout *const CPL_RESTRICT pDstData,
3388 : int nDstPixelStride, GPtrDiff_t nWordCount)
3389 : {
3390 4762 : decltype(nWordCount) nDstOffset = 0;
3391 :
3392 4762 : const Tout tOutZero = static_cast<Tout>(0);
3393 :
3394 4762 : const char *const pSrcDataPtr = reinterpret_cast<const char *>(pSrcData);
3395 4762 : char *const pDstDataPtr = reinterpret_cast<char *>(pDstData);
3396 :
3397 1190408 : for (decltype(nWordCount) n = 0; n < nWordCount; n++)
3398 : {
3399 1185646 : const Tin tValue =
3400 1185646 : *reinterpret_cast<const Tin *>(pSrcDataPtr + n * nSrcPixelStride);
3401 1185646 : Tout *const pPixelOut =
3402 1185646 : reinterpret_cast<Tout *>(pDstDataPtr + nDstOffset);
3403 1185646 : GDALCopyWord(tValue, *pPixelOut);
3404 :
3405 1185646 : pPixelOut[1] = tOutZero;
3406 :
3407 1185646 : nDstOffset += nDstPixelStride;
3408 : }
3409 4762 : }
3410 :
3411 : /************************************************************************/
3412 : /* GDALCopyWordsFromT() */
3413 : /************************************************************************/
3414 : /**
3415 : * Template driver function. Given the input type T, call the appropriate
3416 : * GDALCopyWordsT function template for the desired output type. You should
3417 : * never call this function directly (call GDALCopyWords instead).
3418 : *
3419 : * @param pSrcData source data buffer
3420 : * @param nSrcPixelStride pixel stride in input buffer, in pixel words
3421 : * @param bInComplex input is complex
3422 : * @param pDstData destination data buffer
3423 : * @param eDstType destination data type
3424 : * @param nDstPixelStride pixel stride in output buffer, in pixel words
3425 : * @param nWordCount number of pixel words to be copied
3426 : */
3427 : template <class T>
3428 61313979 : inline void GDALCopyWordsFromT(const T *const CPL_RESTRICT pSrcData,
3429 : int nSrcPixelStride, bool bInComplex,
3430 : void *CPL_RESTRICT pDstData,
3431 : GDALDataType eDstType, int nDstPixelStride,
3432 : GPtrDiff_t nWordCount)
3433 : {
3434 61313979 : switch (eDstType)
3435 : {
3436 4805715 : case GDT_UInt8:
3437 4805715 : GDALCopyWordsT(pSrcData, nSrcPixelStride,
3438 : static_cast<unsigned char *>(pDstData),
3439 : nDstPixelStride, nWordCount);
3440 4805715 : break;
3441 1891 : case GDT_Int8:
3442 1891 : GDALCopyWordsT(pSrcData, nSrcPixelStride,
3443 : static_cast<signed char *>(pDstData),
3444 : nDstPixelStride, nWordCount);
3445 1891 : break;
3446 1143779 : case GDT_UInt16:
3447 1143779 : GDALCopyWordsT(pSrcData, nSrcPixelStride,
3448 : static_cast<unsigned short *>(pDstData),
3449 : nDstPixelStride, nWordCount);
3450 1143779 : break;
3451 4162728 : case GDT_Int16:
3452 4162728 : GDALCopyWordsT(pSrcData, nSrcPixelStride,
3453 : static_cast<short *>(pDstData), nDstPixelStride,
3454 : nWordCount);
3455 4162728 : break;
3456 23318 : case GDT_UInt32:
3457 23318 : GDALCopyWordsT(pSrcData, nSrcPixelStride,
3458 : static_cast<unsigned int *>(pDstData),
3459 : nDstPixelStride, nWordCount);
3460 23318 : break;
3461 29460449 : case GDT_Int32:
3462 29460449 : GDALCopyWordsT(pSrcData, nSrcPixelStride,
3463 : static_cast<int *>(pDstData), nDstPixelStride,
3464 : nWordCount);
3465 29460449 : break;
3466 1250 : case GDT_UInt64:
3467 1250 : GDALCopyWordsT(pSrcData, nSrcPixelStride,
3468 : static_cast<std::uint64_t *>(pDstData),
3469 : nDstPixelStride, nWordCount);
3470 1250 : break;
3471 5957 : case GDT_Int64:
3472 5957 : GDALCopyWordsT(pSrcData, nSrcPixelStride,
3473 : static_cast<std::int64_t *>(pDstData),
3474 : nDstPixelStride, nWordCount);
3475 5957 : break;
3476 999 : case GDT_Float16:
3477 999 : GDALCopyWordsT(pSrcData, nSrcPixelStride,
3478 : static_cast<GFloat16 *>(pDstData), nDstPixelStride,
3479 : nWordCount);
3480 999 : break;
3481 4216031 : case GDT_Float32:
3482 4216031 : GDALCopyWordsT(pSrcData, nSrcPixelStride,
3483 : static_cast<float *>(pDstData), nDstPixelStride,
3484 : nWordCount);
3485 4216031 : break;
3486 17388164 : case GDT_Float64:
3487 17388164 : GDALCopyWordsT(pSrcData, nSrcPixelStride,
3488 : static_cast<double *>(pDstData), nDstPixelStride,
3489 : nWordCount);
3490 17388164 : break;
3491 94424 : case GDT_CInt16:
3492 94424 : if (bInComplex)
3493 : {
3494 93170 : GDALCopyWordsComplexT(pSrcData, nSrcPixelStride,
3495 : static_cast<short *>(pDstData),
3496 : nDstPixelStride, nWordCount);
3497 : }
3498 : else // input is not complex, so we need to promote to a complex
3499 : // buffer
3500 : {
3501 1254 : GDALCopyWordsComplexOutT(pSrcData, nSrcPixelStride,
3502 : static_cast<short *>(pDstData),
3503 : nDstPixelStride, nWordCount);
3504 : }
3505 94424 : break;
3506 1349 : case GDT_CInt32:
3507 1349 : if (bInComplex)
3508 : {
3509 717 : GDALCopyWordsComplexT(pSrcData, nSrcPixelStride,
3510 : static_cast<int *>(pDstData),
3511 : nDstPixelStride, nWordCount);
3512 : }
3513 : else // input is not complex, so we need to promote to a complex
3514 : // buffer
3515 : {
3516 632 : GDALCopyWordsComplexOutT(pSrcData, nSrcPixelStride,
3517 : static_cast<int *>(pDstData),
3518 : nDstPixelStride, nWordCount);
3519 : }
3520 1349 : break;
3521 313 : case GDT_CFloat16:
3522 313 : if (bInComplex)
3523 : {
3524 48 : GDALCopyWordsComplexT(pSrcData, nSrcPixelStride,
3525 : static_cast<GFloat16 *>(pDstData),
3526 : nDstPixelStride, nWordCount);
3527 : }
3528 : else // input is not complex, so we need to promote to a complex
3529 : // buffer
3530 : {
3531 265 : GDALCopyWordsComplexOutT(pSrcData, nSrcPixelStride,
3532 : static_cast<GFloat16 *>(pDstData),
3533 : nDstPixelStride, nWordCount);
3534 : }
3535 313 : break;
3536 3924 : case GDT_CFloat32:
3537 3924 : if (bInComplex)
3538 : {
3539 3115 : GDALCopyWordsComplexT(pSrcData, nSrcPixelStride,
3540 : static_cast<float *>(pDstData),
3541 : nDstPixelStride, nWordCount);
3542 : }
3543 : else // input is not complex, so we need to promote to a complex
3544 : // buffer
3545 : {
3546 809 : GDALCopyWordsComplexOutT(pSrcData, nSrcPixelStride,
3547 : static_cast<float *>(pDstData),
3548 : nDstPixelStride, nWordCount);
3549 : }
3550 3924 : break;
3551 3540 : case GDT_CFloat64:
3552 3540 : if (bInComplex)
3553 : {
3554 1738 : GDALCopyWordsComplexT(pSrcData, nSrcPixelStride,
3555 : static_cast<double *>(pDstData),
3556 : nDstPixelStride, nWordCount);
3557 : }
3558 : else // input is not complex, so we need to promote to a complex
3559 : // buffer
3560 : {
3561 1802 : GDALCopyWordsComplexOutT(pSrcData, nSrcPixelStride,
3562 : static_cast<double *>(pDstData),
3563 : nDstPixelStride, nWordCount);
3564 : }
3565 3540 : break;
3566 0 : case GDT_Unknown:
3567 : case GDT_TypeCount:
3568 0 : CPLAssert(false);
3569 : }
3570 61313979 : }
3571 :
3572 : } // end anonymous namespace
3573 :
3574 : /************************************************************************/
3575 : /* GDALReplicateWord() */
3576 : /************************************************************************/
3577 :
3578 : template <class T>
3579 600406 : inline void GDALReplicateWordT(void *pDstData, int nDstPixelStride,
3580 : GPtrDiff_t nWordCount)
3581 : {
3582 600406 : const T valSet = *static_cast<const T *>(pDstData);
3583 600406 : if (nDstPixelStride == static_cast<int>(sizeof(T)))
3584 : {
3585 570592 : T *pDstPtr = static_cast<T *>(pDstData) + 1;
3586 31990099 : while (nWordCount >= 4)
3587 : {
3588 31419540 : nWordCount -= 4;
3589 31419540 : pDstPtr[0] = valSet;
3590 31419540 : pDstPtr[1] = valSet;
3591 31419540 : pDstPtr[2] = valSet;
3592 31419540 : pDstPtr[3] = valSet;
3593 31419540 : pDstPtr += 4;
3594 : }
3595 1476627 : while (nWordCount > 0)
3596 : {
3597 906035 : --nWordCount;
3598 906035 : *pDstPtr = valSet;
3599 906035 : pDstPtr++;
3600 : }
3601 : }
3602 : else
3603 : {
3604 29814 : GByte *pabyDstPtr = static_cast<GByte *>(pDstData) + nDstPixelStride;
3605 1106520 : while (nWordCount > 0)
3606 : {
3607 1076706 : --nWordCount;
3608 1076706 : *reinterpret_cast<T *>(pabyDstPtr) = valSet;
3609 1076706 : pabyDstPtr += nDstPixelStride;
3610 : }
3611 : }
3612 600406 : }
3613 :
3614 1080140 : static void GDALReplicateWord(const void *CPL_RESTRICT pSrcData,
3615 : GDALDataType eSrcType,
3616 : void *CPL_RESTRICT pDstData,
3617 : GDALDataType eDstType, int nDstPixelStride,
3618 : GPtrDiff_t nWordCount)
3619 : {
3620 : /* -----------------------------------------------------------------------
3621 : */
3622 : /* Special case when the source data is always the same value */
3623 : /* (for VRTSourcedRasterBand::IRasterIO and
3624 : * VRTDerivedRasterBand::IRasterIO*/
3625 : /* for example) */
3626 : /* -----------------------------------------------------------------------
3627 : */
3628 : // Let the general translation case do the necessary conversions
3629 : // on the first destination element.
3630 1080140 : GDALCopyWords64(pSrcData, eSrcType, 0, pDstData, eDstType, 0, 1);
3631 :
3632 : // Now copy the first element to the nWordCount - 1 following destination
3633 : // elements.
3634 1080140 : nWordCount--;
3635 1080140 : GByte *pabyDstWord = reinterpret_cast<GByte *>(pDstData) + nDstPixelStride;
3636 :
3637 1080140 : switch (eDstType)
3638 : {
3639 479649 : case GDT_UInt8:
3640 : case GDT_Int8:
3641 : {
3642 479649 : if (nDstPixelStride == 1)
3643 : {
3644 369718 : if (nWordCount > 0)
3645 369718 : memset(pabyDstWord,
3646 369718 : *reinterpret_cast<const GByte *>(pDstData),
3647 : nWordCount);
3648 : }
3649 : else
3650 : {
3651 109931 : GByte valSet = *reinterpret_cast<const GByte *>(pDstData);
3652 72735800 : while (nWordCount > 0)
3653 : {
3654 72625900 : --nWordCount;
3655 72625900 : *pabyDstWord = valSet;
3656 72625900 : pabyDstWord += nDstPixelStride;
3657 : }
3658 : }
3659 479649 : break;
3660 : }
3661 :
3662 : #define CASE_DUPLICATE_SIMPLE(enum_type, c_type) \
3663 : case enum_type: \
3664 : { \
3665 : GDALReplicateWordT<c_type>(pDstData, nDstPixelStride, nWordCount); \
3666 : break; \
3667 : }
3668 :
3669 34514 : CASE_DUPLICATE_SIMPLE(GDT_UInt16, GUInt16)
3670 202455 : CASE_DUPLICATE_SIMPLE(GDT_Int16, GInt16)
3671 74 : CASE_DUPLICATE_SIMPLE(GDT_UInt32, GUInt32)
3672 301585 : CASE_DUPLICATE_SIMPLE(GDT_Int32, GInt32)
3673 41 : CASE_DUPLICATE_SIMPLE(GDT_UInt64, std::uint64_t)
3674 1072 : CASE_DUPLICATE_SIMPLE(GDT_Int64, std::int64_t)
3675 2 : CASE_DUPLICATE_SIMPLE(GDT_Float16, GFloat16)
3676 52858 : CASE_DUPLICATE_SIMPLE(GDT_Float32, float)
3677 7805 : CASE_DUPLICATE_SIMPLE(GDT_Float64, double)
3678 :
3679 : #define CASE_DUPLICATE_COMPLEX(enum_type, c_type) \
3680 : case enum_type: \
3681 : { \
3682 : c_type valSet1 = reinterpret_cast<const c_type *>(pDstData)[0]; \
3683 : c_type valSet2 = reinterpret_cast<const c_type *>(pDstData)[1]; \
3684 : while (nWordCount > 0) \
3685 : { \
3686 : --nWordCount; \
3687 : reinterpret_cast<c_type *>(pabyDstWord)[0] = valSet1; \
3688 : reinterpret_cast<c_type *>(pabyDstWord)[1] = valSet2; \
3689 : pabyDstWord += nDstPixelStride; \
3690 : } \
3691 : break; \
3692 : }
3693 :
3694 784 : CASE_DUPLICATE_COMPLEX(GDT_CInt16, GInt16)
3695 784 : CASE_DUPLICATE_COMPLEX(GDT_CInt32, GInt32)
3696 6 : CASE_DUPLICATE_COMPLEX(GDT_CFloat16, GFloat16)
3697 790 : CASE_DUPLICATE_COMPLEX(GDT_CFloat32, float)
3698 790 : CASE_DUPLICATE_COMPLEX(GDT_CFloat64, double)
3699 :
3700 0 : case GDT_Unknown:
3701 : case GDT_TypeCount:
3702 0 : CPLAssert(false);
3703 : }
3704 1080140 : }
3705 :
3706 : /************************************************************************/
3707 : /* GDALUnrolledCopy() */
3708 : /************************************************************************/
3709 :
3710 : template <class T, int srcStride, int dstStride>
3711 : #if defined(__GNUC__) && defined(__AVX2__)
3712 : __attribute__((optimize("tree-vectorize")))
3713 : #endif
3714 3053832 : static inline void GDALUnrolledCopyGeneric(T *CPL_RESTRICT pDest,
3715 : const T *CPL_RESTRICT pSrc,
3716 : GPtrDiff_t nIters)
3717 : {
3718 : #if !(defined(__GNUC__) && defined(__AVX2__))
3719 3053832 : if (nIters >= 16)
3720 : {
3721 133693888 : for (GPtrDiff_t i = nIters / 16; i != 0; i--)
3722 : {
3723 130760733 : pDest[0 * dstStride] = pSrc[0 * srcStride];
3724 130760733 : pDest[1 * dstStride] = pSrc[1 * srcStride];
3725 130760733 : pDest[2 * dstStride] = pSrc[2 * srcStride];
3726 130760733 : pDest[3 * dstStride] = pSrc[3 * srcStride];
3727 130760733 : pDest[4 * dstStride] = pSrc[4 * srcStride];
3728 130760733 : pDest[5 * dstStride] = pSrc[5 * srcStride];
3729 130760733 : pDest[6 * dstStride] = pSrc[6 * srcStride];
3730 130760733 : pDest[7 * dstStride] = pSrc[7 * srcStride];
3731 130760733 : pDest[8 * dstStride] = pSrc[8 * srcStride];
3732 130760733 : pDest[9 * dstStride] = pSrc[9 * srcStride];
3733 130760733 : pDest[10 * dstStride] = pSrc[10 * srcStride];
3734 130760733 : pDest[11 * dstStride] = pSrc[11 * srcStride];
3735 130760733 : pDest[12 * dstStride] = pSrc[12 * srcStride];
3736 130760733 : pDest[13 * dstStride] = pSrc[13 * srcStride];
3737 130760733 : pDest[14 * dstStride] = pSrc[14 * srcStride];
3738 130760733 : pDest[15 * dstStride] = pSrc[15 * srcStride];
3739 130760733 : pDest += 16 * dstStride;
3740 130760733 : pSrc += 16 * srcStride;
3741 : }
3742 2933264 : nIters = nIters % 16;
3743 : }
3744 : #else
3745 : #pragma GCC unroll 4
3746 : #endif
3747 5214072 : for (GPtrDiff_t i = 0; i < nIters; i++)
3748 : {
3749 2160243 : pDest[i * dstStride] = *pSrc;
3750 2160243 : pSrc += srcStride;
3751 : }
3752 3053832 : }
3753 :
3754 : template <class T, int srcStride, int dstStride>
3755 3053832 : static inline void GDALUnrolledCopy(T *CPL_RESTRICT pDest,
3756 : const T *CPL_RESTRICT pSrc,
3757 : GPtrDiff_t nIters)
3758 : {
3759 3053832 : GDALUnrolledCopyGeneric<T, srcStride, dstStride>(pDest, pSrc, nIters);
3760 3053832 : }
3761 :
3762 : #if defined(__AVX2__) && defined(HAVE_SSSE3_AT_COMPILE_TIME) && \
3763 : (defined(__x86_64) || defined(_M_X64) || defined(USE_NEON_OPTIMIZATIONS))
3764 :
3765 : template <>
3766 : void GDALUnrolledCopy<GByte, 3, 1>(GByte *CPL_RESTRICT pDest,
3767 : const GByte *CPL_RESTRICT pSrc,
3768 : GPtrDiff_t nIters)
3769 : {
3770 : if (nIters > 16)
3771 : {
3772 : // The SSSE3 variant is slightly faster than what the gcc autovectorizer
3773 : // generates
3774 : GDALUnrolledCopy_GByte_3_1_SSSE3(pDest, pSrc, nIters);
3775 : }
3776 : else
3777 : {
3778 : for (GPtrDiff_t i = 0; i < nIters; i++)
3779 : {
3780 : pDest[i] = *pSrc;
3781 : pSrc += 3;
3782 : }
3783 : }
3784 : }
3785 :
3786 : #elif defined(HAVE_SSE2) && !(defined(__GNUC__) && defined(__AVX2__))
3787 :
3788 : template <>
3789 354194 : void GDALUnrolledCopy<GByte, 2, 1>(GByte *CPL_RESTRICT pDest,
3790 : const GByte *CPL_RESTRICT pSrc,
3791 : GPtrDiff_t nIters)
3792 : {
3793 354194 : decltype(nIters) i = 0;
3794 354194 : if (nIters > 16)
3795 : {
3796 194667 : const __m128i xmm_mask = _mm_set1_epi16(0xff);
3797 : // If we were sure that there would always be 1 trailing byte, we could
3798 : // check against nIters - 15
3799 2988110 : for (; i < nIters - 16; i += 16)
3800 : {
3801 : __m128i xmm0 =
3802 2793440 : _mm_loadu_si128(reinterpret_cast<__m128i const *>(pSrc + 0));
3803 : __m128i xmm1 =
3804 5586890 : _mm_loadu_si128(reinterpret_cast<__m128i const *>(pSrc + 16));
3805 : // Set higher 8bit of each int16 packed word to 0
3806 2793440 : xmm0 = _mm_and_si128(xmm0, xmm_mask);
3807 2793440 : xmm1 = _mm_and_si128(xmm1, xmm_mask);
3808 : // Pack int16 to uint8 and merge back both vector
3809 2793440 : xmm0 = _mm_packus_epi16(xmm0, xmm1);
3810 :
3811 : // Store result
3812 2793440 : _mm_storeu_si128(reinterpret_cast<__m128i *>(pDest + i), xmm0);
3813 :
3814 2793440 : pSrc += 2 * 16;
3815 : }
3816 : }
3817 4633800 : for (; i < nIters; i++)
3818 : {
3819 4279610 : pDest[i] = *pSrc;
3820 4279610 : pSrc += 2;
3821 : }
3822 354194 : }
3823 :
3824 1 : static void GDALUnrolledCopy_GByte_3_1_SSE2(GByte *CPL_RESTRICT pDest,
3825 : const GByte *CPL_RESTRICT pSrc,
3826 : GPtrDiff_t nIters)
3827 : {
3828 1 : decltype(nIters) i = 0;
3829 1 : const __m128i xmm_mask_ori = _mm_set_epi32(0, 0, 0, 255);
3830 : // If we were sure that there would always be 2 trailing bytes, we could
3831 : // check against nIters - 15
3832 2 : for (; i < nIters - 16; i += 16)
3833 : {
3834 : __m128i xmm0 =
3835 1 : _mm_loadu_si128(reinterpret_cast<__m128i const *>(pSrc + 0));
3836 : __m128i xmm1 =
3837 1 : _mm_loadu_si128(reinterpret_cast<__m128i const *>(pSrc + 16));
3838 : __m128i xmm2 =
3839 1 : _mm_loadu_si128(reinterpret_cast<__m128i const *>(pSrc + 32));
3840 :
3841 1 : auto xmm_mask0 = xmm_mask_ori;
3842 1 : auto xmm_mask1 = _mm_slli_si128(xmm_mask_ori, 6);
3843 1 : auto xmm_mask2 = _mm_slli_si128(xmm_mask_ori, 11);
3844 :
3845 1 : auto xmm = _mm_and_si128(xmm0, xmm_mask0);
3846 1 : auto xmm_res1 = _mm_and_si128(_mm_slli_si128(xmm1, 4), xmm_mask1);
3847 :
3848 1 : xmm_mask0 = _mm_slli_si128(xmm_mask0, 1);
3849 1 : xmm_mask1 = _mm_slli_si128(xmm_mask1, 1);
3850 1 : xmm0 = _mm_srli_si128(xmm0, 2);
3851 1 : xmm = _mm_or_si128(xmm, _mm_and_si128(xmm0, xmm_mask0));
3852 2 : xmm_res1 = _mm_or_si128(
3853 : xmm_res1, _mm_and_si128(_mm_slli_si128(xmm1, 2), xmm_mask1));
3854 :
3855 1 : xmm_mask0 = _mm_slli_si128(xmm_mask0, 1);
3856 1 : xmm_mask1 = _mm_slli_si128(xmm_mask1, 1);
3857 1 : xmm0 = _mm_srli_si128(xmm0, 2);
3858 2 : xmm = _mm_or_si128(xmm, _mm_and_si128(xmm0, xmm_mask0));
3859 1 : xmm_res1 = _mm_or_si128(xmm_res1, _mm_and_si128(xmm1, xmm_mask1));
3860 :
3861 1 : xmm_mask0 = _mm_slli_si128(xmm_mask0, 1);
3862 1 : xmm_mask1 = _mm_slli_si128(xmm_mask1, 1);
3863 1 : xmm0 = _mm_srli_si128(xmm0, 2);
3864 1 : xmm = _mm_or_si128(xmm, _mm_and_si128(xmm0, xmm_mask0));
3865 2 : xmm_res1 = _mm_or_si128(
3866 : xmm_res1, _mm_and_si128(_mm_srli_si128(xmm1, 2), xmm_mask1));
3867 :
3868 1 : xmm_mask0 = _mm_slli_si128(xmm_mask0, 1);
3869 1 : xmm_mask1 = _mm_slli_si128(xmm_mask1, 1);
3870 1 : xmm0 = _mm_srli_si128(xmm0, 2);
3871 1 : xmm = _mm_or_si128(xmm, _mm_and_si128(xmm0, xmm_mask0));
3872 3 : xmm_res1 = _mm_or_si128(
3873 : xmm_res1, _mm_and_si128(_mm_srli_si128(xmm1, 4), xmm_mask1));
3874 1 : xmm = _mm_or_si128(xmm, xmm_res1);
3875 :
3876 1 : xmm_mask0 = _mm_slli_si128(xmm_mask0, 1);
3877 1 : xmm0 = _mm_srli_si128(xmm0, 2);
3878 1 : xmm = _mm_or_si128(xmm, _mm_and_si128(xmm0, xmm_mask0));
3879 :
3880 2 : xmm = _mm_or_si128(xmm,
3881 : _mm_and_si128(_mm_slli_si128(xmm2, 10), xmm_mask2));
3882 :
3883 1 : xmm_mask2 = _mm_slli_si128(xmm_mask2, 1);
3884 2 : xmm = _mm_or_si128(xmm,
3885 : _mm_and_si128(_mm_slli_si128(xmm2, 8), xmm_mask2));
3886 :
3887 1 : xmm_mask2 = _mm_slli_si128(xmm_mask2, 1);
3888 2 : xmm = _mm_or_si128(xmm,
3889 : _mm_and_si128(_mm_slli_si128(xmm2, 6), xmm_mask2));
3890 :
3891 1 : xmm_mask2 = _mm_slli_si128(xmm_mask2, 1);
3892 2 : xmm = _mm_or_si128(xmm,
3893 : _mm_and_si128(_mm_slli_si128(xmm2, 4), xmm_mask2));
3894 :
3895 1 : xmm_mask2 = _mm_slli_si128(xmm_mask2, 1);
3896 2 : xmm = _mm_or_si128(xmm,
3897 : _mm_and_si128(_mm_slli_si128(xmm2, 2), xmm_mask2));
3898 :
3899 1 : _mm_storeu_si128(reinterpret_cast<__m128i *>(pDest + i), xmm);
3900 :
3901 1 : pSrc += 3 * 16;
3902 : }
3903 2 : for (; i < nIters; i++)
3904 : {
3905 1 : pDest[i] = *pSrc;
3906 1 : pSrc += 3;
3907 : }
3908 1 : }
3909 :
3910 : #ifdef HAVE_SSSE3_AT_COMPILE_TIME
3911 :
3912 : template <>
3913 193425 : void GDALUnrolledCopy<GByte, 3, 1>(GByte *CPL_RESTRICT pDest,
3914 : const GByte *CPL_RESTRICT pSrc,
3915 : GPtrDiff_t nIters)
3916 : {
3917 193425 : if (nIters > 16)
3918 : {
3919 187302 : if (CPLHaveRuntimeSSSE3())
3920 : {
3921 187301 : GDALUnrolledCopy_GByte_3_1_SSSE3(pDest, pSrc, nIters);
3922 : }
3923 : else
3924 : {
3925 1 : GDALUnrolledCopy_GByte_3_1_SSE2(pDest, pSrc, nIters);
3926 : }
3927 : }
3928 : else
3929 : {
3930 20384 : for (GPtrDiff_t i = 0; i < nIters; i++)
3931 : {
3932 14261 : pDest[i] = *pSrc;
3933 14261 : pSrc += 3;
3934 : }
3935 : }
3936 193425 : }
3937 :
3938 : #else
3939 :
3940 : template <>
3941 : void GDALUnrolledCopy<GByte, 3, 1>(GByte *CPL_RESTRICT pDest,
3942 : const GByte *CPL_RESTRICT pSrc,
3943 : GPtrDiff_t nIters)
3944 : {
3945 : GDALUnrolledCopy_GByte_3_1_SSE2(pDest, pSrc, nIters);
3946 : }
3947 : #endif
3948 :
3949 : template <>
3950 332694 : void GDALUnrolledCopy<GByte, 4, 1>(GByte *CPL_RESTRICT pDest,
3951 : const GByte *CPL_RESTRICT pSrc,
3952 : GPtrDiff_t nIters)
3953 : {
3954 332694 : decltype(nIters) i = 0;
3955 332694 : if (nIters > 16)
3956 : {
3957 327397 : const __m128i xmm_mask = _mm_set1_epi32(0xff);
3958 : // If we were sure that there would always be 3 trailing bytes, we could
3959 : // check against nIters - 15
3960 28178600 : for (; i < nIters - 16; i += 16)
3961 : {
3962 : __m128i xmm0 =
3963 27851200 : _mm_loadu_si128(reinterpret_cast<__m128i const *>(pSrc + 0));
3964 : __m128i xmm1 =
3965 27851200 : _mm_loadu_si128(reinterpret_cast<__m128i const *>(pSrc + 16));
3966 : __m128i xmm2 =
3967 27851200 : _mm_loadu_si128(reinterpret_cast<__m128i const *>(pSrc + 32));
3968 : __m128i xmm3 =
3969 55702500 : _mm_loadu_si128(reinterpret_cast<__m128i const *>(pSrc + 48));
3970 : // Set higher 24bit of each int32 packed word to 0
3971 27851200 : xmm0 = _mm_and_si128(xmm0, xmm_mask);
3972 27851200 : xmm1 = _mm_and_si128(xmm1, xmm_mask);
3973 27851200 : xmm2 = _mm_and_si128(xmm2, xmm_mask);
3974 27851200 : xmm3 = _mm_and_si128(xmm3, xmm_mask);
3975 : // Pack int32 to int16
3976 27851200 : xmm0 = _mm_packs_epi32(xmm0, xmm1);
3977 27851200 : xmm2 = _mm_packs_epi32(xmm2, xmm3);
3978 : // Pack int16 to uint8
3979 27851200 : xmm0 = _mm_packus_epi16(xmm0, xmm2);
3980 :
3981 : // Store result
3982 27851200 : _mm_storeu_si128(reinterpret_cast<__m128i *>(pDest + i), xmm0);
3983 :
3984 27851200 : pSrc += 4 * 16;
3985 : }
3986 : }
3987 5049310 : for (; i < nIters; i++)
3988 : {
3989 4716620 : pDest[i] = *pSrc;
3990 4716620 : pSrc += 4;
3991 : }
3992 332694 : }
3993 : #endif // HAVE_SSE2
3994 :
3995 : /************************************************************************/
3996 : /* GDALFastCopy() */
3997 : /************************************************************************/
3998 :
3999 : template <class T>
4000 40270700 : static inline void GDALFastCopy(T *CPL_RESTRICT pDest, int nDestStride,
4001 : const T *CPL_RESTRICT pSrc, int nSrcStride,
4002 : GPtrDiff_t nIters)
4003 : {
4004 40270700 : constexpr int sizeofT = static_cast<int>(sizeof(T));
4005 40270700 : if (nIters == 1)
4006 : {
4007 22545440 : *pDest = *pSrc;
4008 : }
4009 17725255 : else if (nDestStride == sizeofT)
4010 : {
4011 14598200 : if (nSrcStride == sizeofT)
4012 : {
4013 13508184 : memcpy(pDest, pSrc, nIters * sizeof(T));
4014 : }
4015 1090047 : else if (nSrcStride == 2 * sizeofT)
4016 : {
4017 357410 : GDALUnrolledCopy<T, 2, 1>(pDest, pSrc, nIters);
4018 : }
4019 732637 : else if (nSrcStride == 3 * sizeofT)
4020 : {
4021 290405 : GDALUnrolledCopy<T, 3, 1>(pDest, pSrc, nIters);
4022 : }
4023 442232 : else if (nSrcStride == 4 * sizeofT)
4024 : {
4025 336676 : GDALUnrolledCopy<T, 4, 1>(pDest, pSrc, nIters);
4026 : }
4027 : else
4028 : {
4029 17229290 : while (nIters-- > 0)
4030 : {
4031 17123750 : *pDest = *pSrc;
4032 17123750 : pSrc += nSrcStride / sizeofT;
4033 17123750 : pDest++;
4034 : }
4035 : }
4036 : }
4037 3126965 : else if (nSrcStride == sizeofT)
4038 : {
4039 3113959 : if (nDestStride == 2 * sizeofT)
4040 : {
4041 151764 : GDALUnrolledCopy<T, 1, 2>(pDest, pSrc, nIters);
4042 : }
4043 2962205 : else if (nDestStride == 3 * sizeofT)
4044 : {
4045 2133781 : GDALUnrolledCopy<T, 1, 3>(pDest, pSrc, nIters);
4046 : }
4047 828421 : else if (nDestStride == 4 * sizeofT)
4048 : {
4049 664109 : GDALUnrolledCopy<T, 1, 4>(pDest, pSrc, nIters);
4050 : }
4051 : else
4052 : {
4053 17169660 : while (nIters-- > 0)
4054 : {
4055 17005410 : *pDest = *pSrc;
4056 17005410 : pSrc++;
4057 17005410 : pDest += nDestStride / sizeofT;
4058 : }
4059 : }
4060 : }
4061 : else
4062 : {
4063 1220108 : while (nIters-- > 0)
4064 : {
4065 1207102 : *pDest = *pSrc;
4066 1207102 : pSrc += nSrcStride / sizeofT;
4067 1207102 : pDest += nDestStride / sizeofT;
4068 : }
4069 : }
4070 40270700 : }
4071 :
4072 : /************************************************************************/
4073 : /* GDALFastCopyByte() */
4074 : /************************************************************************/
4075 :
4076 326320 : static void GDALFastCopyByte(const GByte *CPL_RESTRICT pSrcData,
4077 : int nSrcPixelStride, GByte *CPL_RESTRICT pDstData,
4078 : int nDstPixelStride, GPtrDiff_t nWordCount)
4079 : {
4080 326320 : GDALFastCopy(pDstData, nDstPixelStride, pSrcData, nSrcPixelStride,
4081 : nWordCount);
4082 326320 : }
4083 :
4084 : /************************************************************************/
4085 : /* GDALCopyWords() */
4086 : /************************************************************************/
4087 :
4088 : /**
4089 : * Copy pixel words from buffer to buffer.
4090 : *
4091 : * @see GDALCopyWords64()
4092 : */
4093 80594200 : void CPL_STDCALL GDALCopyWords(const void *CPL_RESTRICT pSrcData,
4094 : GDALDataType eSrcType, int nSrcPixelStride,
4095 : void *CPL_RESTRICT pDstData,
4096 : GDALDataType eDstType, int nDstPixelStride,
4097 : int nWordCount)
4098 : {
4099 80594200 : GDALCopyWords64(pSrcData, eSrcType, nSrcPixelStride, pDstData, eDstType,
4100 : nDstPixelStride, nWordCount);
4101 80594200 : }
4102 :
4103 : /************************************************************************/
4104 : /* GDALCopyWords64() */
4105 : /************************************************************************/
4106 :
4107 : /**
4108 : * Copy pixel words from buffer to buffer.
4109 : *
4110 : * This function is used to copy pixel word values from one memory buffer
4111 : * to another, with support for conversion between data types, and differing
4112 : * step factors. The data type conversion is done using the following
4113 : * rules:
4114 : * <ul>
4115 : * <li>Values assigned to a lower range integer type are clipped. For
4116 : * instance assigning GDT_Int16 values to a GDT_UInt8 buffer will cause values
4117 : * less the 0 to be set to 0, and values larger than 255 to be set to 255.
4118 : * </li>
4119 : * <li>
4120 : * Assignment from floating point to integer rounds to closest integer.
4121 : * +Infinity is mapped to the largest integer. -Infinity is mapped to the
4122 : * smallest integer. NaN is mapped to 0.
4123 : * </li>
4124 : * <li>
4125 : * Assignment from non-complex to complex will result in the imaginary part
4126 : * being set to zero on output.
4127 : * </li>
4128 : * <li> Assignment from complex to
4129 : * non-complex will result in the complex portion being lost and the real
4130 : * component being preserved (<i>not magnitude!</i>).
4131 : * </li>
4132 : * </ul>
4133 : *
4134 : * No assumptions are made about the source or destination words occurring
4135 : * on word boundaries. It is assumed that all values are in native machine
4136 : * byte order.
4137 : *
4138 : * @param pSrcData Pointer to source data to be converted.
4139 : * @param eSrcType the source data type (see GDALDataType enum)
4140 : * @param nSrcPixelStride Source pixel stride (i.e. distance between 2 words),
4141 : * in bytes
4142 : * @param pDstData Pointer to buffer where destination data should go
4143 : * @param eDstType the destination data type (see GDALDataType enum)
4144 : * @param nDstPixelStride Destination pixel stride (i.e. distance between 2
4145 : * words), in bytes
4146 : * @param nWordCount number of words to be copied
4147 : *
4148 : * @note
4149 : * When adding a new data type to GDAL, you must do the following to
4150 : * support it properly within the GDALCopyWords function:
4151 : * 1. Add the data type to the switch on eSrcType in GDALCopyWords.
4152 : * This should invoke the appropriate GDALCopyWordsFromT wrapper.
4153 : * 2. Add the data type to the switch on eDstType in GDALCopyWordsFromT.
4154 : * This should call the appropriate GDALCopyWordsT template.
4155 : * 3. If appropriate, overload the appropriate CopyWord template in the
4156 : * above namespace. This will ensure that any conversion issues are
4157 : * handled (cases like the float -> int32 case, where the min/max)
4158 : * values are subject to roundoff error.
4159 : */
4160 :
4161 116978000 : void CPL_STDCALL GDALCopyWords64(const void *CPL_RESTRICT pSrcData,
4162 : GDALDataType eSrcType, int nSrcPixelStride,
4163 : void *CPL_RESTRICT pDstData,
4164 : GDALDataType eDstType, int nDstPixelStride,
4165 : GPtrDiff_t nWordCount)
4166 :
4167 : {
4168 : // On platforms where alignment matters, be careful
4169 116978000 : const int nSrcDataTypeSize = GDALGetDataTypeSizeBytes(eSrcType);
4170 116978000 : const int nDstDataTypeSize = GDALGetDataTypeSizeBytes(eDstType);
4171 116978000 : if (CPL_UNLIKELY(nSrcDataTypeSize == 0 || nDstDataTypeSize == 0))
4172 : {
4173 2 : CPLError(CE_Failure, CPLE_NotSupported,
4174 : "GDALCopyWords64(): unsupported GDT_Unknown/GDT_TypeCount "
4175 : "argument");
4176 2 : return;
4177 : }
4178 116978000 : if (!(eSrcType == eDstType && nSrcPixelStride == nDstPixelStride) &&
4179 66410400 : ((reinterpret_cast<uintptr_t>(pSrcData) % nSrcDataTypeSize) != 0 ||
4180 66410400 : (reinterpret_cast<uintptr_t>(pDstData) % nDstDataTypeSize) != 0 ||
4181 66410000 : (nSrcPixelStride % nSrcDataTypeSize) != 0 ||
4182 66409900 : (nDstPixelStride % nDstDataTypeSize) != 0))
4183 : {
4184 905 : if (eSrcType == eDstType)
4185 : {
4186 34800 : for (decltype(nWordCount) i = 0; i < nWordCount; i++)
4187 : {
4188 34000 : memcpy(static_cast<GByte *>(pDstData) + nDstPixelStride * i,
4189 : static_cast<const GByte *>(pSrcData) +
4190 34000 : nSrcPixelStride * i,
4191 : nDstDataTypeSize);
4192 : }
4193 : }
4194 : else
4195 : {
4196 210 : const auto getAlignedPtr = [](GByte *ptr, int align)
4197 : {
4198 : return ptr +
4199 210 : ((align - (reinterpret_cast<uintptr_t>(ptr) % align)) %
4200 210 : align);
4201 : };
4202 :
4203 : // The largest we need is for CFloat64 (16 bytes), so 32 bytes to
4204 : // be sure to get correctly aligned pointer.
4205 105 : constexpr size_t SIZEOF_CFLOAT64 = 2 * sizeof(double);
4206 : GByte abySrcBuffer[2 * SIZEOF_CFLOAT64];
4207 : GByte abyDstBuffer[2 * SIZEOF_CFLOAT64];
4208 : GByte *pabySrcBuffer =
4209 105 : getAlignedPtr(abySrcBuffer, nSrcDataTypeSize);
4210 : GByte *pabyDstBuffer =
4211 105 : getAlignedPtr(abyDstBuffer, nDstDataTypeSize);
4212 3360 : for (decltype(nWordCount) i = 0; i < nWordCount; i++)
4213 : {
4214 3255 : memcpy(pabySrcBuffer,
4215 : static_cast<const GByte *>(pSrcData) +
4216 3255 : nSrcPixelStride * i,
4217 : nSrcDataTypeSize);
4218 3255 : GDALCopyWords64(pabySrcBuffer, eSrcType, 0, pabyDstBuffer,
4219 : eDstType, 0, 1);
4220 3255 : memcpy(static_cast<GByte *>(pDstData) + nDstPixelStride * i,
4221 : pabyDstBuffer, nDstDataTypeSize);
4222 : }
4223 : }
4224 905 : return;
4225 : }
4226 :
4227 : // Deal with the case where we're replicating a single word into the
4228 : // provided buffer
4229 116978000 : if (nSrcPixelStride == 0 && nWordCount > 1)
4230 : {
4231 1080140 : GDALReplicateWord(pSrcData, eSrcType, pDstData, eDstType,
4232 : nDstPixelStride, nWordCount);
4233 1080140 : return;
4234 : }
4235 :
4236 115897000 : if (eSrcType == eDstType)
4237 : {
4238 54845500 : if (eSrcType == GDT_UInt8 || eSrcType == GDT_Int8)
4239 : {
4240 18148200 : GDALFastCopy(static_cast<GByte *>(pDstData), nDstPixelStride,
4241 : static_cast<const GByte *>(pSrcData), nSrcPixelStride,
4242 : nWordCount);
4243 18148200 : return;
4244 : }
4245 :
4246 36697300 : if (nSrcDataTypeSize == 2 && (nSrcPixelStride % 2) == 0 &&
4247 21796200 : (nDstPixelStride % 2) == 0)
4248 : {
4249 21796200 : GDALFastCopy(static_cast<short *>(pDstData), nDstPixelStride,
4250 : static_cast<const short *>(pSrcData), nSrcPixelStride,
4251 : nWordCount);
4252 21796200 : return;
4253 : }
4254 :
4255 14901100 : if (nWordCount == 1)
4256 : {
4257 : #if defined(CSA_BUILD) || defined(__COVERITY__)
4258 : // Avoid false positives...
4259 : memcpy(pDstData, pSrcData, nSrcDataTypeSize);
4260 : #else
4261 14411900 : if (nSrcDataTypeSize == 2)
4262 0 : memcpy(pDstData, pSrcData, 2);
4263 14411900 : else if (nSrcDataTypeSize == 4)
4264 13807600 : memcpy(pDstData, pSrcData, 4);
4265 604291 : else if (nSrcDataTypeSize == 8)
4266 587686 : memcpy(pDstData, pSrcData, 8);
4267 : else /* if( eSrcType == GDT_CFloat64 ) */
4268 16605 : memcpy(pDstData, pSrcData, 16);
4269 : #endif
4270 14411900 : return;
4271 : }
4272 :
4273 : // Let memcpy() handle the case where we're copying a packed buffer
4274 : // of pixels.
4275 489225 : if (nSrcPixelStride == nDstPixelStride)
4276 : {
4277 227381 : if (nSrcPixelStride == nSrcDataTypeSize)
4278 : {
4279 227313 : memcpy(pDstData, pSrcData, nWordCount * nSrcDataTypeSize);
4280 227313 : return;
4281 : }
4282 : }
4283 : }
4284 :
4285 : // Handle the more general case -- deals with conversion of data types
4286 : // directly.
4287 61313900 : switch (eSrcType)
4288 : {
4289 20306500 : case GDT_UInt8:
4290 20306500 : GDALCopyWordsFromT<unsigned char>(
4291 : static_cast<const unsigned char *>(pSrcData), nSrcPixelStride,
4292 : false, pDstData, eDstType, nDstPixelStride, nWordCount);
4293 20306500 : break;
4294 1786 : case GDT_Int8:
4295 1786 : GDALCopyWordsFromT<signed char>(
4296 : static_cast<const signed char *>(pSrcData), nSrcPixelStride,
4297 : false, pDstData, eDstType, nDstPixelStride, nWordCount);
4298 1786 : break;
4299 55545 : case GDT_UInt16:
4300 55545 : GDALCopyWordsFromT<unsigned short>(
4301 : static_cast<const unsigned short *>(pSrcData), nSrcPixelStride,
4302 : false, pDstData, eDstType, nDstPixelStride, nWordCount);
4303 55545 : break;
4304 6519830 : case GDT_Int16:
4305 6519830 : GDALCopyWordsFromT<short>(static_cast<const short *>(pSrcData),
4306 : nSrcPixelStride, false, pDstData,
4307 : eDstType, nDstPixelStride, nWordCount);
4308 6519830 : break;
4309 8250 : case GDT_UInt32:
4310 8250 : GDALCopyWordsFromT<unsigned int>(
4311 : static_cast<const unsigned int *>(pSrcData), nSrcPixelStride,
4312 : false, pDstData, eDstType, nDstPixelStride, nWordCount);
4313 8250 : break;
4314 12254800 : case GDT_Int32:
4315 12254800 : GDALCopyWordsFromT<int>(static_cast<const int *>(pSrcData),
4316 : nSrcPixelStride, false, pDstData, eDstType,
4317 : nDstPixelStride, nWordCount);
4318 12254800 : break;
4319 2205 : case GDT_UInt64:
4320 2205 : GDALCopyWordsFromT<std::uint64_t>(
4321 : static_cast<const std::uint64_t *>(pSrcData), nSrcPixelStride,
4322 : false, pDstData, eDstType, nDstPixelStride, nWordCount);
4323 2205 : break;
4324 11729 : case GDT_Int64:
4325 11729 : GDALCopyWordsFromT<std::int64_t>(
4326 : static_cast<const std::int64_t *>(pSrcData), nSrcPixelStride,
4327 : false, pDstData, eDstType, nDstPixelStride, nWordCount);
4328 11729 : break;
4329 1387 : case GDT_Float16:
4330 1387 : GDALCopyWordsFromT<GFloat16>(
4331 : static_cast<const GFloat16 *>(pSrcData), nSrcPixelStride, false,
4332 : pDstData, eDstType, nDstPixelStride, nWordCount);
4333 1387 : break;
4334 664922 : case GDT_Float32:
4335 664922 : GDALCopyWordsFromT<float>(static_cast<const float *>(pSrcData),
4336 : nSrcPixelStride, false, pDstData,
4337 : eDstType, nDstPixelStride, nWordCount);
4338 664922 : break;
4339 20726100 : case GDT_Float64:
4340 20726100 : GDALCopyWordsFromT<double>(static_cast<const double *>(pSrcData),
4341 : nSrcPixelStride, false, pDstData,
4342 : eDstType, nDstPixelStride, nWordCount);
4343 20726100 : break;
4344 478486 : case GDT_CInt16:
4345 478486 : GDALCopyWordsFromT<short>(static_cast<const short *>(pSrcData),
4346 : nSrcPixelStride, true, pDstData, eDstType,
4347 : nDstPixelStride, nWordCount);
4348 478486 : break;
4349 868 : case GDT_CInt32:
4350 868 : GDALCopyWordsFromT<int>(static_cast<const int *>(pSrcData),
4351 : nSrcPixelStride, true, pDstData, eDstType,
4352 : nDstPixelStride, nWordCount);
4353 868 : break;
4354 508 : case GDT_CFloat16:
4355 508 : GDALCopyWordsFromT<GFloat16>(
4356 : static_cast<const GFloat16 *>(pSrcData), nSrcPixelStride, true,
4357 : pDstData, eDstType, nDstPixelStride, nWordCount);
4358 508 : break;
4359 2437 : case GDT_CFloat32:
4360 2437 : GDALCopyWordsFromT<float>(static_cast<const float *>(pSrcData),
4361 : nSrcPixelStride, true, pDstData, eDstType,
4362 : nDstPixelStride, nWordCount);
4363 2437 : break;
4364 278617 : case GDT_CFloat64:
4365 278617 : GDALCopyWordsFromT<double>(static_cast<const double *>(pSrcData),
4366 : nSrcPixelStride, true, pDstData,
4367 : eDstType, nDstPixelStride, nWordCount);
4368 278617 : break;
4369 0 : case GDT_Unknown:
4370 : case GDT_TypeCount:
4371 0 : CPLAssert(false);
4372 : }
4373 : }
4374 :
4375 : /************************************************************************/
4376 : /* GDALCopyBits() */
4377 : /************************************************************************/
4378 :
4379 : /**
4380 : * Bitwise word copying.
4381 : *
4382 : * A function for moving sets of partial bytes around. Loosely
4383 : * speaking this is a bitwise analog to GDALCopyWords().
4384 : *
4385 : * It copies nStepCount "words" where each word is nBitCount bits long.
4386 : * The nSrcStep and nDstStep are the number of bits from the start of one
4387 : * word to the next (same as nBitCount if they are packed). The nSrcOffset
4388 : * and nDstOffset are the offset into the source and destination buffers
4389 : * to start at, also measured in bits.
4390 : *
4391 : * All bit offsets are assumed to start from the high order bit in a byte
4392 : * (i.e. most significant bit first). Currently this function is not very
4393 : * optimized, but it may be improved for some common cases in the future
4394 : * as needed.
4395 : *
4396 : * @param pabySrcData the source data buffer.
4397 : * @param nSrcOffset the offset (in bits) in pabySrcData to the start of the
4398 : * first word to copy.
4399 : * @param nSrcStep the offset in bits from the start one source word to the
4400 : * start of the next.
4401 : * @param pabyDstData the destination data buffer.
4402 : * @param nDstOffset the offset (in bits) in pabyDstData to the start of the
4403 : * first word to copy over.
4404 : * @param nDstStep the offset in bits from the start one word to the
4405 : * start of the next.
4406 : * @param nBitCount the number of bits in a word to be copied.
4407 : * @param nStepCount the number of words to copy.
4408 : */
4409 :
4410 0 : void GDALCopyBits(const GByte *pabySrcData, int nSrcOffset, int nSrcStep,
4411 : GByte *pabyDstData, int nDstOffset, int nDstStep,
4412 : int nBitCount, int nStepCount)
4413 :
4414 : {
4415 0 : VALIDATE_POINTER0(pabySrcData, "GDALCopyBits");
4416 :
4417 0 : for (int iStep = 0; iStep < nStepCount; iStep++)
4418 : {
4419 0 : for (int iBit = 0; iBit < nBitCount; iBit++)
4420 : {
4421 0 : if (pabySrcData[nSrcOffset >> 3] & (0x80 >> (nSrcOffset & 7)))
4422 0 : pabyDstData[nDstOffset >> 3] |= (0x80 >> (nDstOffset & 7));
4423 : else
4424 0 : pabyDstData[nDstOffset >> 3] &= ~(0x80 >> (nDstOffset & 7));
4425 :
4426 0 : nSrcOffset++;
4427 0 : nDstOffset++;
4428 : }
4429 :
4430 0 : nSrcOffset += (nSrcStep - nBitCount);
4431 0 : nDstOffset += (nDstStep - nBitCount);
4432 : }
4433 : }
4434 :
4435 : /************************************************************************/
4436 : /* GDALBandGetBestOverviewLevel() */
4437 : /************************************************************************/
4438 :
4439 525463 : int GDALBandGetBestOverviewLevel(GDALRasterBand *poBand,
4440 : double dfTargetDownsamplingRatio,
4441 : double dfOversamplingThreshold)
4442 : {
4443 525463 : int iBestOvr = -1;
4444 525463 : double dfBestRatio = 0;
4445 525463 : const int nOvCount = poBand->GetOverviewCount();
4446 525463 : constexpr double EPSILON = 1e-1;
4447 1053620 : for (int iOvr = -1; iOvr < nOvCount; iOvr++)
4448 : {
4449 531090 : double dfOvrRatio = 1.0;
4450 531090 : GDALRasterBand *poOvrBand = nullptr;
4451 531090 : if (iOvr >= 0)
4452 : {
4453 5627 : poOvrBand = poBand->GetOverview(iOvr);
4454 11254 : if (poOvrBand == nullptr ||
4455 11254 : poOvrBand->GetXSize() > poBand->GetXSize() ||
4456 5627 : poOvrBand->GetYSize() > poBand->GetYSize())
4457 : {
4458 0 : continue;
4459 : }
4460 22508 : dfOvrRatio = std::min(static_cast<double>(poBand->GetXSize()) /
4461 5627 : poOvrBand->GetXSize(),
4462 11254 : static_cast<double>(poBand->GetYSize()) /
4463 11254 : poOvrBand->GetYSize());
4464 : }
4465 :
4466 : // Is it nearly the requested factor and better (lower) than
4467 : // the current best factor?
4468 : // Use an epsilon because of numerical instability.
4469 531197 : if (dfOvrRatio >=
4470 531090 : dfTargetDownsamplingRatio * dfOversamplingThreshold + EPSILON ||
4471 : dfOvrRatio <= dfBestRatio)
4472 : {
4473 107 : continue;
4474 : }
4475 :
4476 530983 : if (poOvrBand)
4477 : {
4478 : // Ignore AVERAGE_BIT2GRAYSCALE overviews.
4479 : const char *pszResampling =
4480 5520 : poOvrBand->GetMetadataItem("RESAMPLING");
4481 5520 : if (pszResampling != nullptr &&
4482 71 : STARTS_WITH_CI(pszResampling, "AVERAGE_BIT2"))
4483 : {
4484 16 : continue;
4485 : }
4486 : }
4487 :
4488 530967 : iBestOvr = iOvr;
4489 530967 : dfBestRatio = dfOvrRatio;
4490 530967 : if (std::abs(dfTargetDownsamplingRatio - dfOvrRatio) < EPSILON)
4491 : {
4492 2938 : break;
4493 : }
4494 : }
4495 525463 : return iBestOvr;
4496 : }
4497 :
4498 : /************************************************************************/
4499 : /* GDALGetBestOverviewLevel() */
4500 : /* */
4501 : /* Returns the best overview level to satisfy the query or -1 if none */
4502 : /* Also updates nXOff, nYOff, nXSize, nYSize and psExtraArg when */
4503 : /* returning a valid overview level */
4504 : /************************************************************************/
4505 :
4506 0 : int GDALBandGetBestOverviewLevel(GDALRasterBand *poBand, int &nXOff, int &nYOff,
4507 : int &nXSize, int &nYSize, int nBufXSize,
4508 : int nBufYSize)
4509 : {
4510 0 : return GDALBandGetBestOverviewLevel2(poBand, nXOff, nYOff, nXSize, nYSize,
4511 0 : nBufXSize, nBufYSize, nullptr);
4512 : }
4513 :
4514 525556 : int GDALBandGetBestOverviewLevel2(GDALRasterBand *poBand, int &nXOff,
4515 : int &nYOff, int &nXSize, int &nYSize,
4516 : int nBufXSize, int nBufYSize,
4517 : GDALRasterIOExtraArg *psExtraArg)
4518 : {
4519 525556 : if (psExtraArg != nullptr && psExtraArg->nVersion > 1 &&
4520 525556 : psExtraArg->bUseOnlyThisScale)
4521 109 : return -1;
4522 : /* -------------------------------------------------------------------- */
4523 : /* Compute the desired downsampling factor. It is */
4524 : /* based on the least reduced axis, and represents the number */
4525 : /* of source pixels to one destination pixel. */
4526 : /* -------------------------------------------------------------------- */
4527 525447 : const double dfDesiredDownsamplingFactor =
4528 525447 : ((nXSize / static_cast<double>(nBufXSize)) <
4529 363107 : (nYSize / static_cast<double>(nBufYSize)) ||
4530 : nBufYSize == 1)
4531 755372 : ? nXSize / static_cast<double>(nBufXSize)
4532 133182 : : nYSize / static_cast<double>(nBufYSize);
4533 :
4534 : /* -------------------------------------------------------------------- */
4535 : /* Find the overview level that largest downsampling factor (most */
4536 : /* downsampled) that is still less than (or only a little more) */
4537 : /* downsampled than the request. */
4538 : /* -------------------------------------------------------------------- */
4539 :
4540 : const char *pszOversampligThreshold =
4541 525447 : CPLGetConfigOption("GDAL_OVERVIEW_OVERSAMPLING_THRESHOLD", nullptr);
4542 :
4543 : // Cf https://github.com/OSGeo/gdal/pull/9040#issuecomment-1898524693
4544 : const double dfOversamplingThreshold =
4545 1050880 : pszOversampligThreshold ? CPLAtof(pszOversampligThreshold)
4546 525438 : : psExtraArg && psExtraArg->eResampleAlg != GRIORA_NearestNeighbour
4547 1050880 : ? 1.0
4548 525447 : : 1.2;
4549 525447 : const int iBestOvrLevel = GDALBandGetBestOverviewLevel(
4550 : poBand, dfDesiredDownsamplingFactor, dfOversamplingThreshold);
4551 :
4552 : /* -------------------------------------------------------------------- */
4553 : /* If we didn't find an overview that helps us, just return */
4554 : /* indicating failure and the full resolution image will be used. */
4555 : /* -------------------------------------------------------------------- */
4556 525447 : if (iBestOvrLevel < 0)
4557 522454 : return -1;
4558 2993 : const GDALRasterBand *poBestOverview = poBand->GetOverview(iBestOvrLevel);
4559 :
4560 : /* -------------------------------------------------------------------- */
4561 : /* Recompute the source window in terms of the selected */
4562 : /* overview. */
4563 : /* -------------------------------------------------------------------- */
4564 : const double dfXFactor =
4565 2993 : poBand->GetXSize() / static_cast<double>(poBestOverview->GetXSize());
4566 : const double dfYFactor =
4567 2993 : poBand->GetYSize() / static_cast<double>(poBestOverview->GetYSize());
4568 2993 : CPLDebug("GDAL", "Selecting overview %d x %d", poBestOverview->GetXSize(),
4569 : poBestOverview->GetYSize());
4570 :
4571 8979 : const int nOXOff = std::min(poBestOverview->GetXSize() - 1,
4572 2993 : static_cast<int>(nXOff / dfXFactor + 0.5));
4573 8979 : const int nOYOff = std::min(poBestOverview->GetYSize() - 1,
4574 2993 : static_cast<int>(nYOff / dfYFactor + 0.5));
4575 2993 : int nOXSize = std::max(1, static_cast<int>(nXSize / dfXFactor + 0.5));
4576 2993 : int nOYSize = std::max(1, static_cast<int>(nYSize / dfYFactor + 0.5));
4577 2993 : if (nOXOff + nOXSize > poBestOverview->GetXSize())
4578 0 : nOXSize = poBestOverview->GetXSize() - nOXOff;
4579 2993 : if (nOYOff + nOYSize > poBestOverview->GetYSize())
4580 2 : nOYSize = poBestOverview->GetYSize() - nOYOff;
4581 :
4582 2993 : if (psExtraArg)
4583 : {
4584 2993 : if (psExtraArg->bFloatingPointWindowValidity)
4585 : {
4586 117 : psExtraArg->dfXOff /= dfXFactor;
4587 117 : psExtraArg->dfXSize /= dfXFactor;
4588 117 : psExtraArg->dfYOff /= dfYFactor;
4589 117 : psExtraArg->dfYSize /= dfYFactor;
4590 : }
4591 2876 : else if (psExtraArg->eResampleAlg != GRIORA_NearestNeighbour)
4592 : {
4593 16 : psExtraArg->bFloatingPointWindowValidity = true;
4594 16 : psExtraArg->dfXOff = nXOff / dfXFactor;
4595 16 : psExtraArg->dfXSize = nXSize / dfXFactor;
4596 16 : psExtraArg->dfYOff = nYOff / dfYFactor;
4597 16 : psExtraArg->dfYSize = nYSize / dfYFactor;
4598 : }
4599 : }
4600 :
4601 2993 : nXOff = nOXOff;
4602 2993 : nYOff = nOYOff;
4603 2993 : nXSize = nOXSize;
4604 2993 : nYSize = nOYSize;
4605 :
4606 2993 : return iBestOvrLevel;
4607 : }
4608 :
4609 : /************************************************************************/
4610 : /* OverviewRasterIO() */
4611 : /* */
4612 : /* Special work function to utilize available overviews to */
4613 : /* more efficiently satisfy downsampled requests. It will */
4614 : /* return CE_Failure if there are no appropriate overviews */
4615 : /* available but it doesn't emit any error messages. */
4616 : /************************************************************************/
4617 :
4618 : //! @cond Doxygen_Suppress
4619 1 : CPLErr GDALRasterBand::OverviewRasterIO(
4620 : GDALRWFlag eRWFlag, int nXOff, int nYOff, int nXSize, int nYSize,
4621 : void *pData, int nBufXSize, int nBufYSize, GDALDataType eBufType,
4622 : GSpacing nPixelSpace, GSpacing nLineSpace, GDALRasterIOExtraArg *psExtraArg)
4623 :
4624 : {
4625 : GDALRasterIOExtraArg sExtraArg;
4626 1 : GDALCopyRasterIOExtraArg(&sExtraArg, psExtraArg);
4627 :
4628 1 : const int nOverview = GDALBandGetBestOverviewLevel2(
4629 : this, nXOff, nYOff, nXSize, nYSize, nBufXSize, nBufYSize, &sExtraArg);
4630 1 : if (nOverview < 0)
4631 1 : return CE_Failure;
4632 :
4633 : /* -------------------------------------------------------------------- */
4634 : /* Recast the call in terms of the new raster layer. */
4635 : /* -------------------------------------------------------------------- */
4636 0 : GDALRasterBand *poOverviewBand = GetOverview(nOverview);
4637 0 : if (poOverviewBand == nullptr)
4638 0 : return CE_Failure;
4639 :
4640 0 : return poOverviewBand->RasterIO(eRWFlag, nXOff, nYOff, nXSize, nYSize,
4641 : pData, nBufXSize, nBufYSize, eBufType,
4642 0 : nPixelSpace, nLineSpace, &sExtraArg);
4643 : }
4644 :
4645 : /************************************************************************/
4646 : /* TryOverviewRasterIO() */
4647 : /************************************************************************/
4648 :
4649 362428 : CPLErr GDALRasterBand::TryOverviewRasterIO(
4650 : GDALRWFlag eRWFlag, int nXOff, int nYOff, int nXSize, int nYSize,
4651 : void *pData, int nBufXSize, int nBufYSize, GDALDataType eBufType,
4652 : GSpacing nPixelSpace, GSpacing nLineSpace, GDALRasterIOExtraArg *psExtraArg,
4653 : int *pbTried)
4654 : {
4655 362428 : int nXOffMod = nXOff;
4656 362428 : int nYOffMod = nYOff;
4657 362428 : int nXSizeMod = nXSize;
4658 362428 : int nYSizeMod = nYSize;
4659 : GDALRasterIOExtraArg sExtraArg;
4660 :
4661 362428 : GDALCopyRasterIOExtraArg(&sExtraArg, psExtraArg);
4662 :
4663 362428 : int iOvrLevel = GDALBandGetBestOverviewLevel2(
4664 : this, nXOffMod, nYOffMod, nXSizeMod, nYSizeMod, nBufXSize, nBufYSize,
4665 : &sExtraArg);
4666 :
4667 362428 : if (iOvrLevel >= 0)
4668 : {
4669 53 : GDALRasterBand *poOverviewBand = GetOverview(iOvrLevel);
4670 53 : if (poOverviewBand)
4671 : {
4672 53 : *pbTried = TRUE;
4673 53 : return poOverviewBand->RasterIO(
4674 : eRWFlag, nXOffMod, nYOffMod, nXSizeMod, nYSizeMod, pData,
4675 : nBufXSize, nBufYSize, eBufType, nPixelSpace, nLineSpace,
4676 53 : &sExtraArg);
4677 : }
4678 : }
4679 :
4680 362375 : *pbTried = FALSE;
4681 362375 : return CE_None;
4682 : }
4683 :
4684 : /************************************************************************/
4685 : /* TryOverviewRasterIO() */
4686 : /************************************************************************/
4687 :
4688 160153 : CPLErr GDALDataset::TryOverviewRasterIO(
4689 : GDALRWFlag eRWFlag, int nXOff, int nYOff, int nXSize, int nYSize,
4690 : void *pData, int nBufXSize, int nBufYSize, GDALDataType eBufType,
4691 : int nBandCount, const int *panBandMap, GSpacing nPixelSpace,
4692 : GSpacing nLineSpace, GSpacing nBandSpace, GDALRasterIOExtraArg *psExtraArg,
4693 : int *pbTried)
4694 : {
4695 160153 : int nXOffMod = nXOff;
4696 160153 : int nYOffMod = nYOff;
4697 160153 : int nXSizeMod = nXSize;
4698 160153 : int nYSizeMod = nYSize;
4699 : GDALRasterIOExtraArg sExtraArg;
4700 160153 : GDALCopyRasterIOExtraArg(&sExtraArg, psExtraArg);
4701 :
4702 320306 : int iOvrLevel = GDALBandGetBestOverviewLevel2(
4703 160153 : papoBands[0], nXOffMod, nYOffMod, nXSizeMod, nYSizeMod, nBufXSize,
4704 : nBufYSize, &sExtraArg);
4705 :
4706 160196 : if (iOvrLevel >= 0 && papoBands[0]->GetOverview(iOvrLevel) != nullptr &&
4707 43 : papoBands[0]->GetOverview(iOvrLevel)->GetDataset() != nullptr)
4708 : {
4709 43 : *pbTried = TRUE;
4710 43 : return papoBands[0]->GetOverview(iOvrLevel)->GetDataset()->RasterIO(
4711 : eRWFlag, nXOffMod, nYOffMod, nXSizeMod, nYSizeMod, pData, nBufXSize,
4712 : nBufYSize, eBufType, nBandCount, panBandMap, nPixelSpace,
4713 43 : nLineSpace, nBandSpace, &sExtraArg);
4714 : }
4715 : else
4716 : {
4717 160110 : *pbTried = FALSE;
4718 160110 : return CE_None;
4719 : }
4720 : }
4721 :
4722 : /************************************************************************/
4723 : /* GetBestOverviewLevel() */
4724 : /* */
4725 : /* Returns the best overview level to satisfy the query or -1 if none */
4726 : /* Also updates nXOff, nYOff, nXSize, nYSize when returning a valid */
4727 : /* overview level */
4728 : /************************************************************************/
4729 :
4730 4 : static int GDALDatasetGetBestOverviewLevel(GDALDataset *poDS, int &nXOff,
4731 : int &nYOff, int &nXSize, int &nYSize,
4732 : int nBufXSize, int nBufYSize,
4733 : int nBandCount,
4734 : const int *panBandMap,
4735 : GDALRasterIOExtraArg *psExtraArg)
4736 : {
4737 4 : int nOverviewCount = 0;
4738 4 : GDALRasterBand *poFirstBand = nullptr;
4739 :
4740 : /* -------------------------------------------------------------------- */
4741 : /* Check that all bands have the same number of overviews and */
4742 : /* that they have all the same size and block dimensions */
4743 : /* -------------------------------------------------------------------- */
4744 12 : for (int iBand = 0; iBand < nBandCount; iBand++)
4745 : {
4746 8 : GDALRasterBand *poBand = poDS->GetRasterBand(panBandMap[iBand]);
4747 8 : if (poBand == nullptr)
4748 0 : return -1;
4749 8 : if (iBand == 0)
4750 : {
4751 4 : poFirstBand = poBand;
4752 4 : nOverviewCount = poBand->GetOverviewCount();
4753 : }
4754 4 : else if (nOverviewCount != poBand->GetOverviewCount())
4755 : {
4756 0 : CPLDebug("GDAL", "GDALDataset::GetBestOverviewLevel() ... "
4757 : "mismatched overview count, use std method.");
4758 0 : return -1;
4759 : }
4760 : else
4761 : {
4762 4 : for (int iOverview = 0; iOverview < nOverviewCount; iOverview++)
4763 : {
4764 0 : GDALRasterBand *poOvrBand = poBand->GetOverview(iOverview);
4765 : GDALRasterBand *poOvrFirstBand =
4766 0 : poFirstBand->GetOverview(iOverview);
4767 0 : if (poOvrBand == nullptr || poOvrFirstBand == nullptr)
4768 0 : continue;
4769 :
4770 0 : if (poOvrFirstBand->GetXSize() != poOvrBand->GetXSize() ||
4771 0 : poOvrFirstBand->GetYSize() != poOvrBand->GetYSize())
4772 : {
4773 0 : CPLDebug("GDAL",
4774 : "GDALDataset::GetBestOverviewLevel() ... "
4775 : "mismatched overview sizes, use std method.");
4776 0 : return -1;
4777 : }
4778 0 : int nBlockXSizeFirst = 0;
4779 0 : int nBlockYSizeFirst = 0;
4780 0 : poOvrFirstBand->GetBlockSize(&nBlockXSizeFirst,
4781 : &nBlockYSizeFirst);
4782 :
4783 0 : int nBlockXSizeCurrent = 0;
4784 0 : int nBlockYSizeCurrent = 0;
4785 0 : poOvrBand->GetBlockSize(&nBlockXSizeCurrent,
4786 : &nBlockYSizeCurrent);
4787 :
4788 0 : if (nBlockXSizeFirst != nBlockXSizeCurrent ||
4789 0 : nBlockYSizeFirst != nBlockYSizeCurrent)
4790 : {
4791 0 : CPLDebug("GDAL", "GDALDataset::GetBestOverviewLevel() ... "
4792 : "mismatched block sizes, use std method.");
4793 0 : return -1;
4794 : }
4795 : }
4796 : }
4797 : }
4798 4 : if (poFirstBand == nullptr)
4799 0 : return -1;
4800 :
4801 4 : return GDALBandGetBestOverviewLevel2(poFirstBand, nXOff, nYOff, nXSize,
4802 : nYSize, nBufXSize, nBufYSize,
4803 4 : psExtraArg);
4804 : }
4805 :
4806 : /************************************************************************/
4807 : /* BlockBasedRasterIO() */
4808 : /* */
4809 : /* This convenience function implements a dataset level */
4810 : /* RasterIO() interface based on calling down to fetch blocks, */
4811 : /* much like the GDALRasterBand::IRasterIO(), but it handles */
4812 : /* all bands at once, so that a format driver that handles a */
4813 : /* request for different bands of the same block efficiently */
4814 : /* (i.e. without re-reading interleaved data) will efficiently. */
4815 : /* */
4816 : /* This method is intended to be called by an overridden */
4817 : /* IRasterIO() method in the driver specific GDALDataset */
4818 : /* derived class. */
4819 : /* */
4820 : /* Default internal implementation of RasterIO() ... utilizes */
4821 : /* the Block access methods to satisfy the request. This would */
4822 : /* normally only be overridden by formats with overviews. */
4823 : /* */
4824 : /* To keep things relatively simple, this method does not */
4825 : /* currently take advantage of some special cases addressed in */
4826 : /* GDALRasterBand::IRasterIO(), so it is likely best to only */
4827 : /* call it when you know it will help. That is in cases where */
4828 : /* data is at 1:1 to the buffer, and you know the driver is */
4829 : /* implementing interleaved IO efficiently on a block by block */
4830 : /* basis. Overviews will be used when possible. */
4831 : /************************************************************************/
4832 :
4833 65946 : CPLErr GDALDataset::BlockBasedRasterIO(
4834 : GDALRWFlag eRWFlag, int nXOff, int nYOff, int nXSize, int nYSize,
4835 : void *pData, int nBufXSize, int nBufYSize, GDALDataType eBufType,
4836 : int nBandCount, const int *panBandMap, GSpacing nPixelSpace,
4837 : GSpacing nLineSpace, GSpacing nBandSpace, GDALRasterIOExtraArg *psExtraArg)
4838 :
4839 : {
4840 65946 : CPLAssert(nullptr != pData);
4841 :
4842 65946 : GByte **papabySrcBlock = nullptr;
4843 65946 : GDALRasterBlock *poBlock = nullptr;
4844 65946 : GDALRasterBlock **papoBlocks = nullptr;
4845 65946 : int nLBlockX = -1;
4846 65946 : int nLBlockY = -1;
4847 : int iBufYOff;
4848 : int iBufXOff;
4849 65946 : int nBlockXSize = 1;
4850 65946 : int nBlockYSize = 1;
4851 65946 : CPLErr eErr = CE_None;
4852 65946 : GDALDataType eDataType = GDT_UInt8;
4853 :
4854 65946 : const bool bUseIntegerRequestCoords =
4855 65989 : (!psExtraArg->bFloatingPointWindowValidity ||
4856 43 : (nXOff == psExtraArg->dfXOff && nYOff == psExtraArg->dfYOff &&
4857 41 : nXSize == psExtraArg->dfXSize && nYSize == psExtraArg->dfYSize));
4858 :
4859 : /* -------------------------------------------------------------------- */
4860 : /* Ensure that all bands share a common block size and data type. */
4861 : /* -------------------------------------------------------------------- */
4862 312044 : for (int iBand = 0; iBand < nBandCount; iBand++)
4863 : {
4864 246098 : GDALRasterBand *poBand = GetRasterBand(panBandMap[iBand]);
4865 :
4866 246098 : if (iBand == 0)
4867 : {
4868 65946 : poBand->GetBlockSize(&nBlockXSize, &nBlockYSize);
4869 65946 : eDataType = poBand->GetRasterDataType();
4870 : }
4871 : else
4872 : {
4873 180152 : int nThisBlockXSize = 0;
4874 180152 : int nThisBlockYSize = 0;
4875 180152 : poBand->GetBlockSize(&nThisBlockXSize, &nThisBlockYSize);
4876 180152 : if (nThisBlockXSize != nBlockXSize ||
4877 180152 : nThisBlockYSize != nBlockYSize)
4878 : {
4879 0 : CPLDebug("GDAL", "GDALDataset::BlockBasedRasterIO() ... "
4880 : "mismatched block sizes, use std method.");
4881 0 : return BandBasedRasterIO(eRWFlag, nXOff, nYOff, nXSize, nYSize,
4882 : pData, nBufXSize, nBufYSize, eBufType,
4883 : nBandCount, panBandMap, nPixelSpace,
4884 0 : nLineSpace, nBandSpace, psExtraArg);
4885 : }
4886 :
4887 180152 : if (eDataType != poBand->GetRasterDataType() &&
4888 0 : (nXSize != nBufXSize || nYSize != nBufYSize))
4889 : {
4890 0 : CPLDebug("GDAL", "GDALDataset::BlockBasedRasterIO() ... "
4891 : "mismatched band data types, use std method.");
4892 0 : return BandBasedRasterIO(eRWFlag, nXOff, nYOff, nXSize, nYSize,
4893 : pData, nBufXSize, nBufYSize, eBufType,
4894 : nBandCount, panBandMap, nPixelSpace,
4895 0 : nLineSpace, nBandSpace, psExtraArg);
4896 : }
4897 : }
4898 : }
4899 :
4900 : /* ==================================================================== */
4901 : /* In this special case at full resolution we step through in */
4902 : /* blocks, turning the request over to the per-band */
4903 : /* IRasterIO(), but ensuring that all bands of one block are */
4904 : /* called before proceeding to the next. */
4905 : /* ==================================================================== */
4906 :
4907 65946 : if (nXSize == nBufXSize && nYSize == nBufYSize && bUseIntegerRequestCoords)
4908 : {
4909 : GDALRasterIOExtraArg sDummyExtraArg;
4910 65942 : INIT_RASTERIO_EXTRA_ARG(sDummyExtraArg);
4911 :
4912 65942 : int nChunkYSize = 0;
4913 65942 : int nChunkXSize = 0;
4914 :
4915 215387 : for (iBufYOff = 0; iBufYOff < nBufYSize; iBufYOff += nChunkYSize)
4916 : {
4917 150461 : const int nChunkYOff = iBufYOff + nYOff;
4918 150461 : nChunkYSize = nBlockYSize - (nChunkYOff % nBlockYSize);
4919 150461 : if (nChunkYOff + nChunkYSize > nYOff + nYSize)
4920 60937 : nChunkYSize = (nYOff + nYSize) - nChunkYOff;
4921 :
4922 825895 : for (iBufXOff = 0; iBufXOff < nBufXSize; iBufXOff += nChunkXSize)
4923 : {
4924 676449 : const int nChunkXOff = iBufXOff + nXOff;
4925 676449 : nChunkXSize = nBlockXSize - (nChunkXOff % nBlockXSize);
4926 676449 : if (nChunkXOff + nChunkXSize > nXOff + nXSize)
4927 71009 : nChunkXSize = (nXOff + nXSize) - nChunkXOff;
4928 :
4929 676449 : GByte *pabyChunkData =
4930 676449 : static_cast<GByte *>(pData) + iBufXOff * nPixelSpace +
4931 676449 : static_cast<GPtrDiff_t>(iBufYOff) * nLineSpace;
4932 :
4933 3291110 : for (int iBand = 0; iBand < nBandCount; iBand++)
4934 : {
4935 2615670 : GDALRasterBand *poBand = GetRasterBand(panBandMap[iBand]);
4936 :
4937 5231350 : eErr = poBand->IRasterIO(
4938 : eRWFlag, nChunkXOff, nChunkYOff, nChunkXSize,
4939 : nChunkYSize,
4940 2615670 : pabyChunkData +
4941 2615670 : static_cast<GPtrDiff_t>(iBand) * nBandSpace,
4942 : nChunkXSize, nChunkYSize, eBufType, nPixelSpace,
4943 2615670 : nLineSpace, &sDummyExtraArg);
4944 2615670 : if (eErr != CE_None)
4945 1015 : return eErr;
4946 : }
4947 : }
4948 :
4949 168360 : if (psExtraArg->pfnProgress != nullptr &&
4950 18914 : !psExtraArg->pfnProgress(
4951 168360 : 1.0 * std::min(nBufYSize, iBufYOff + nChunkYSize) /
4952 : nBufYSize,
4953 : "", psExtraArg->pProgressData))
4954 : {
4955 1 : return CE_Failure;
4956 : }
4957 : }
4958 :
4959 64926 : return CE_None;
4960 : }
4961 :
4962 : /* Below code is not compatible with that case. It would need a complete */
4963 : /* separate code like done in GDALRasterBand::IRasterIO. */
4964 4 : if (eRWFlag == GF_Write && (nBufXSize < nXSize || nBufYSize < nYSize))
4965 : {
4966 0 : return BandBasedRasterIO(eRWFlag, nXOff, nYOff, nXSize, nYSize, pData,
4967 : nBufXSize, nBufYSize, eBufType, nBandCount,
4968 : panBandMap, nPixelSpace, nLineSpace,
4969 0 : nBandSpace, psExtraArg);
4970 : }
4971 :
4972 : /* We could have a smarter implementation, but that will do for now */
4973 4 : if (psExtraArg->eResampleAlg != GRIORA_NearestNeighbour &&
4974 0 : (nBufXSize != nXSize || nBufYSize != nYSize))
4975 : {
4976 0 : return BandBasedRasterIO(eRWFlag, nXOff, nYOff, nXSize, nYSize, pData,
4977 : nBufXSize, nBufYSize, eBufType, nBandCount,
4978 : panBandMap, nPixelSpace, nLineSpace,
4979 0 : nBandSpace, psExtraArg);
4980 : }
4981 :
4982 : /* ==================================================================== */
4983 : /* Loop reading required source blocks to satisfy output */
4984 : /* request. This is the most general implementation. */
4985 : /* ==================================================================== */
4986 :
4987 4 : const int nBandDataSize = GDALGetDataTypeSizeBytes(eDataType);
4988 :
4989 : papabySrcBlock =
4990 4 : static_cast<GByte **>(CPLCalloc(sizeof(GByte *), nBandCount));
4991 : papoBlocks =
4992 4 : static_cast<GDALRasterBlock **>(CPLCalloc(sizeof(void *), nBandCount));
4993 :
4994 : /* -------------------------------------------------------------------- */
4995 : /* Select an overview level if appropriate. */
4996 : /* -------------------------------------------------------------------- */
4997 :
4998 : GDALRasterIOExtraArg sExtraArg;
4999 4 : GDALCopyRasterIOExtraArg(&sExtraArg, psExtraArg);
5000 4 : const int nOverviewLevel = GDALDatasetGetBestOverviewLevel(
5001 : this, nXOff, nYOff, nXSize, nYSize, nBufXSize, nBufYSize, nBandCount,
5002 : panBandMap, &sExtraArg);
5003 4 : if (nOverviewLevel >= 0)
5004 : {
5005 2 : GetRasterBand(panBandMap[0])
5006 2 : ->GetOverview(nOverviewLevel)
5007 2 : ->GetBlockSize(&nBlockXSize, &nBlockYSize);
5008 : }
5009 :
5010 4 : double dfXOff = nXOff;
5011 4 : double dfYOff = nYOff;
5012 4 : double dfXSize = nXSize;
5013 4 : double dfYSize = nYSize;
5014 4 : if (sExtraArg.bFloatingPointWindowValidity)
5015 : {
5016 2 : dfXOff = sExtraArg.dfXOff;
5017 2 : dfYOff = sExtraArg.dfYOff;
5018 2 : dfXSize = sExtraArg.dfXSize;
5019 2 : dfYSize = sExtraArg.dfYSize;
5020 : }
5021 :
5022 : /* -------------------------------------------------------------------- */
5023 : /* Compute stepping increment. */
5024 : /* -------------------------------------------------------------------- */
5025 4 : const double dfSrcXInc = dfXSize / static_cast<double>(nBufXSize);
5026 4 : const double dfSrcYInc = dfYSize / static_cast<double>(nBufYSize);
5027 :
5028 4 : constexpr double EPS = 1e-10;
5029 : /* -------------------------------------------------------------------- */
5030 : /* Loop over buffer computing source locations. */
5031 : /* -------------------------------------------------------------------- */
5032 36 : for (iBufYOff = 0; iBufYOff < nBufYSize; iBufYOff++)
5033 : {
5034 : GPtrDiff_t iSrcOffset;
5035 :
5036 : // Add small epsilon to avoid some numeric precision issues.
5037 32 : const double dfSrcY = (iBufYOff + 0.5) * dfSrcYInc + dfYOff + EPS;
5038 32 : const int iSrcY = static_cast<int>(std::min(
5039 32 : std::max(0.0, dfSrcY), static_cast<double>(nRasterYSize - 1)));
5040 :
5041 32 : GPtrDiff_t iBufOffset = static_cast<GPtrDiff_t>(iBufYOff) *
5042 : static_cast<GPtrDiff_t>(nLineSpace);
5043 :
5044 302 : for (iBufXOff = 0; iBufXOff < nBufXSize; iBufXOff++)
5045 : {
5046 270 : const double dfSrcX = (iBufXOff + 0.5) * dfSrcXInc + dfXOff + EPS;
5047 270 : const int iSrcX = static_cast<int>(std::min(
5048 270 : std::max(0.0, dfSrcX), static_cast<double>(nRasterXSize - 1)));
5049 :
5050 : // FIXME: this code likely doesn't work if the dirty block gets
5051 : // flushed to disk before being completely written. In the meantime,
5052 : // bJustInitialize should probably be set to FALSE even if it is not
5053 : // ideal performance wise, and for lossy compression
5054 :
5055 : /* --------------------------------------------------------------------
5056 : */
5057 : /* Ensure we have the appropriate block loaded. */
5058 : /* --------------------------------------------------------------------
5059 : */
5060 270 : if (iSrcX < nLBlockX * nBlockXSize ||
5061 270 : iSrcX - nBlockXSize >= nLBlockX * nBlockXSize ||
5062 266 : iSrcY < nLBlockY * nBlockYSize ||
5063 266 : iSrcY - nBlockYSize >= nLBlockY * nBlockYSize)
5064 : {
5065 4 : nLBlockX = iSrcX / nBlockXSize;
5066 4 : nLBlockY = iSrcY / nBlockYSize;
5067 :
5068 4 : const bool bJustInitialize =
5069 0 : eRWFlag == GF_Write && nYOff <= nLBlockY * nBlockYSize &&
5070 0 : nYOff + nYSize - nBlockYSize >= nLBlockY * nBlockYSize &&
5071 4 : nXOff <= nLBlockX * nBlockXSize &&
5072 0 : nXOff + nXSize - nBlockXSize >= nLBlockX * nBlockXSize;
5073 : /*bool bMemZeroBuffer = FALSE;
5074 : if( eRWFlag == GF_Write && !bJustInitialize &&
5075 : nXOff <= nLBlockX * nBlockXSize &&
5076 : nYOff <= nLBlockY * nBlockYSize &&
5077 : (nXOff + nXSize >= (nLBlockX+1) * nBlockXSize ||
5078 : (nXOff + nXSize == GetRasterXSize() &&
5079 : (nLBlockX+1) * nBlockXSize > GetRasterXSize())) &&
5080 : (nYOff + nYSize >= (nLBlockY+1) * nBlockYSize ||
5081 : (nYOff + nYSize == GetRasterYSize() &&
5082 : (nLBlockY+1) * nBlockYSize > GetRasterYSize())) )
5083 : {
5084 : bJustInitialize = TRUE;
5085 : bMemZeroBuffer = TRUE;
5086 : }*/
5087 12 : for (int iBand = 0; iBand < nBandCount; iBand++)
5088 : {
5089 8 : GDALRasterBand *poBand = GetRasterBand(panBandMap[iBand]);
5090 8 : if (nOverviewLevel >= 0)
5091 2 : poBand = poBand->GetOverview(nOverviewLevel);
5092 16 : poBlock = poBand->GetLockedBlockRef(nLBlockX, nLBlockY,
5093 8 : bJustInitialize);
5094 8 : if (poBlock == nullptr)
5095 : {
5096 0 : eErr = CE_Failure;
5097 0 : goto CleanupAndReturn;
5098 : }
5099 :
5100 8 : if (eRWFlag == GF_Write)
5101 0 : poBlock->MarkDirty();
5102 :
5103 8 : if (papoBlocks[iBand] != nullptr)
5104 0 : papoBlocks[iBand]->DropLock();
5105 :
5106 8 : papoBlocks[iBand] = poBlock;
5107 :
5108 8 : papabySrcBlock[iBand] =
5109 8 : static_cast<GByte *>(poBlock->GetDataRef());
5110 : /*if( bMemZeroBuffer )
5111 : {
5112 : memset(papabySrcBlock[iBand], 0,
5113 : static_cast<GPtrDiff_t>(nBandDataSize) * nBlockXSize
5114 : * nBlockYSize);
5115 : }*/
5116 : }
5117 : }
5118 :
5119 : /* --------------------------------------------------------------------
5120 : */
5121 : /* Copy over this pixel of data. */
5122 : /* --------------------------------------------------------------------
5123 : */
5124 270 : iSrcOffset = (static_cast<GPtrDiff_t>(iSrcX) -
5125 270 : static_cast<GPtrDiff_t>(nLBlockX) * nBlockXSize +
5126 270 : (static_cast<GPtrDiff_t>(iSrcY) -
5127 270 : static_cast<GPtrDiff_t>(nLBlockY) * nBlockYSize) *
5128 270 : nBlockXSize) *
5129 270 : nBandDataSize;
5130 :
5131 980 : for (int iBand = 0; iBand < nBandCount; iBand++)
5132 : {
5133 710 : GByte *pabySrcBlock = papabySrcBlock[iBand];
5134 710 : GPtrDiff_t iBandBufOffset =
5135 710 : iBufOffset + static_cast<GPtrDiff_t>(iBand) *
5136 : static_cast<GPtrDiff_t>(nBandSpace);
5137 :
5138 710 : if (eDataType == eBufType)
5139 : {
5140 710 : if (eRWFlag == GF_Read)
5141 710 : memcpy(static_cast<GByte *>(pData) + iBandBufOffset,
5142 710 : pabySrcBlock + iSrcOffset, nBandDataSize);
5143 : else
5144 0 : memcpy(pabySrcBlock + iSrcOffset,
5145 : static_cast<const GByte *>(pData) +
5146 0 : iBandBufOffset,
5147 : nBandDataSize);
5148 : }
5149 : else
5150 : {
5151 : /* type to type conversion ... ouch, this is expensive way
5152 : of handling single words */
5153 :
5154 0 : if (eRWFlag == GF_Read)
5155 0 : GDALCopyWords64(pabySrcBlock + iSrcOffset, eDataType, 0,
5156 : static_cast<GByte *>(pData) +
5157 0 : iBandBufOffset,
5158 : eBufType, 0, 1);
5159 : else
5160 0 : GDALCopyWords64(static_cast<const GByte *>(pData) +
5161 0 : iBandBufOffset,
5162 0 : eBufType, 0, pabySrcBlock + iSrcOffset,
5163 : eDataType, 0, 1);
5164 : }
5165 : }
5166 :
5167 270 : iBufOffset += static_cast<int>(nPixelSpace);
5168 : }
5169 : }
5170 :
5171 : /* -------------------------------------------------------------------- */
5172 : /* CleanupAndReturn. */
5173 : /* -------------------------------------------------------------------- */
5174 4 : CleanupAndReturn:
5175 4 : CPLFree(papabySrcBlock);
5176 4 : if (papoBlocks != nullptr)
5177 : {
5178 12 : for (int iBand = 0; iBand < nBandCount; iBand++)
5179 : {
5180 8 : if (papoBlocks[iBand] != nullptr)
5181 8 : papoBlocks[iBand]->DropLock();
5182 : }
5183 4 : CPLFree(papoBlocks);
5184 : }
5185 :
5186 4 : return eErr;
5187 : }
5188 :
5189 : //! @endcond
5190 :
5191 : /************************************************************************/
5192 : /* GDALCopyWholeRasterGetSwathSize() */
5193 : /************************************************************************/
5194 :
5195 3377 : static void GDALCopyWholeRasterGetSwathSize(GDALRasterBand *poSrcPrototypeBand,
5196 : GDALRasterBand *poDstPrototypeBand,
5197 : int nBandCount,
5198 : int bDstIsCompressed,
5199 : int bInterleave, int *pnSwathCols,
5200 : int *pnSwathLines)
5201 : {
5202 3377 : GDALDataType eDT = poDstPrototypeBand->GetRasterDataType();
5203 3377 : int nSrcBlockXSize = 0;
5204 3377 : int nSrcBlockYSize = 0;
5205 3377 : int nBlockXSize = 0;
5206 3377 : int nBlockYSize = 0;
5207 :
5208 3377 : int nXSize = poSrcPrototypeBand->GetXSize();
5209 3377 : int nYSize = poSrcPrototypeBand->GetYSize();
5210 :
5211 3377 : poSrcPrototypeBand->GetBlockSize(&nSrcBlockXSize, &nSrcBlockYSize);
5212 3377 : poDstPrototypeBand->GetBlockSize(&nBlockXSize, &nBlockYSize);
5213 :
5214 3377 : const int nMaxBlockXSize = std::max(nBlockXSize, nSrcBlockXSize);
5215 3377 : const int nMaxBlockYSize = std::max(nBlockYSize, nSrcBlockYSize);
5216 :
5217 3377 : int nPixelSize = GDALGetDataTypeSizeBytes(eDT);
5218 3377 : if (bInterleave)
5219 583 : nPixelSize *= nBandCount;
5220 :
5221 : // aim for one row of blocks. Do not settle for less.
5222 3377 : int nSwathCols = nXSize;
5223 3377 : int nSwathLines = nMaxBlockYSize;
5224 :
5225 : const char *pszSrcCompression =
5226 3377 : poSrcPrototypeBand->GetMetadataItem("COMPRESSION", "IMAGE_STRUCTURE");
5227 3377 : if (pszSrcCompression == nullptr)
5228 : {
5229 3357 : auto poSrcDS = poSrcPrototypeBand->GetDataset();
5230 3357 : if (poSrcDS)
5231 : pszSrcCompression =
5232 3351 : poSrcDS->GetMetadataItem("COMPRESSION", "IMAGE_STRUCTURE");
5233 : }
5234 :
5235 : /* -------------------------------------------------------------------- */
5236 : /* What will our swath size be? */
5237 : /* -------------------------------------------------------------------- */
5238 : // When writing interleaved data in a compressed format, we want to be sure
5239 : // that each block will only be written once, so the swath size must not be
5240 : // greater than the block cache.
5241 3377 : const char *pszSwathSize = CPLGetConfigOption("GDAL_SWATH_SIZE", nullptr);
5242 : int nTargetSwathSize;
5243 3377 : if (pszSwathSize != nullptr)
5244 0 : nTargetSwathSize = static_cast<int>(
5245 0 : std::min(GIntBig(INT_MAX), CPLAtoGIntBig(pszSwathSize)));
5246 : else
5247 : {
5248 : // As a default, take one 1/4 of the cache size.
5249 3377 : nTargetSwathSize = static_cast<int>(
5250 3377 : std::min(GIntBig(INT_MAX), GDALGetCacheMax64() / 4));
5251 :
5252 : // but if the minimum idal swath buf size is less, then go for it to
5253 : // avoid unnecessarily abusing RAM usage.
5254 : // but try to use 10 MB at least.
5255 3377 : GIntBig nIdealSwathBufSize =
5256 3377 : static_cast<GIntBig>(nSwathCols) * nSwathLines * nPixelSize;
5257 3377 : int nMinTargetSwathSize = 10 * 1000 * 1000;
5258 :
5259 3377 : if ((poSrcPrototypeBand->GetSuggestedBlockAccessPattern() &
5260 3377 : GSBAP_LARGEST_CHUNK_POSSIBLE) != 0)
5261 : {
5262 1 : nMinTargetSwathSize = nTargetSwathSize;
5263 : }
5264 :
5265 3377 : if (nIdealSwathBufSize < nTargetSwathSize &&
5266 3367 : nIdealSwathBufSize < nMinTargetSwathSize)
5267 : {
5268 3364 : nIdealSwathBufSize = nMinTargetSwathSize;
5269 : }
5270 :
5271 3377 : if (pszSrcCompression != nullptr &&
5272 181 : EQUAL(pszSrcCompression, "JPEG2000") &&
5273 0 : (!bDstIsCompressed || ((nSrcBlockXSize % nBlockXSize) == 0 &&
5274 0 : (nSrcBlockYSize % nBlockYSize) == 0)))
5275 : {
5276 2 : nIdealSwathBufSize =
5277 4 : std::max(nIdealSwathBufSize, static_cast<GIntBig>(nSwathCols) *
5278 2 : nSrcBlockYSize * nPixelSize);
5279 : }
5280 3377 : if (nTargetSwathSize > nIdealSwathBufSize)
5281 3364 : nTargetSwathSize = static_cast<int>(
5282 3364 : std::min(GIntBig(INT_MAX), nIdealSwathBufSize));
5283 : }
5284 :
5285 3377 : if (nTargetSwathSize < 1000000)
5286 8 : nTargetSwathSize = 1000000;
5287 :
5288 : /* But let's check that */
5289 3598 : if (bDstIsCompressed && bInterleave &&
5290 221 : nTargetSwathSize > GDALGetCacheMax64())
5291 : {
5292 0 : CPLError(CE_Warning, CPLE_AppDefined,
5293 : "When translating into a compressed interleave format, "
5294 : "the block cache size (" CPL_FRMT_GIB ") "
5295 : "should be at least the size of the swath (%d) "
5296 : "(GDAL_SWATH_SIZE config. option)",
5297 : GDALGetCacheMax64(), nTargetSwathSize);
5298 : }
5299 :
5300 : #define IS_DIVIDER_OF(x, y) ((y) % (x) == 0)
5301 : #define ROUND_TO(x, y) (((x) / (y)) * (y))
5302 :
5303 : // if both input and output datasets are tiled, that the tile dimensions
5304 : // are "compatible", try to stick to a swath dimension that is a multiple
5305 : // of input and output block dimensions.
5306 3377 : if (nBlockXSize != nXSize && nSrcBlockXSize != nXSize &&
5307 47 : IS_DIVIDER_OF(nBlockXSize, nMaxBlockXSize) &&
5308 47 : IS_DIVIDER_OF(nSrcBlockXSize, nMaxBlockXSize) &&
5309 47 : IS_DIVIDER_OF(nBlockYSize, nMaxBlockYSize) &&
5310 47 : IS_DIVIDER_OF(nSrcBlockYSize, nMaxBlockYSize))
5311 : {
5312 47 : if (static_cast<GIntBig>(nMaxBlockXSize) * nMaxBlockYSize *
5313 47 : nPixelSize <=
5314 47 : static_cast<GIntBig>(nTargetSwathSize))
5315 : {
5316 47 : nSwathCols = nTargetSwathSize / (nMaxBlockYSize * nPixelSize);
5317 47 : nSwathCols = ROUND_TO(nSwathCols, nMaxBlockXSize);
5318 47 : if (nSwathCols == 0)
5319 0 : nSwathCols = nMaxBlockXSize;
5320 47 : if (nSwathCols > nXSize)
5321 45 : nSwathCols = nXSize;
5322 47 : nSwathLines = nMaxBlockYSize;
5323 :
5324 47 : if (static_cast<GIntBig>(nSwathCols) * nSwathLines * nPixelSize >
5325 47 : static_cast<GIntBig>(nTargetSwathSize))
5326 : {
5327 0 : nSwathCols = nXSize;
5328 0 : nSwathLines = nBlockYSize;
5329 : }
5330 : }
5331 : }
5332 :
5333 3377 : const GIntBig nMemoryPerCol = static_cast<GIntBig>(nSwathCols) * nPixelSize;
5334 3377 : const GIntBig nSwathBufSize = nMemoryPerCol * nSwathLines;
5335 3377 : if (nSwathBufSize > static_cast<GIntBig>(nTargetSwathSize))
5336 : {
5337 1 : nSwathLines = static_cast<int>(nTargetSwathSize / nMemoryPerCol);
5338 1 : if (nSwathLines == 0)
5339 1 : nSwathLines = 1;
5340 :
5341 1 : CPLDebug(
5342 : "GDAL",
5343 : "GDALCopyWholeRasterGetSwathSize(): adjusting to %d line swath "
5344 : "since requirement (" CPL_FRMT_GIB " bytes) exceed target swath "
5345 : "size (%d bytes) (GDAL_SWATH_SIZE config. option)",
5346 1 : nSwathLines, nBlockYSize * nMemoryPerCol, nTargetSwathSize);
5347 : }
5348 : // If we are processing single scans, try to handle several at once.
5349 : // If we are handling swaths already, only grow the swath if a row
5350 : // of blocks is substantially less than our target buffer size.
5351 3376 : else if (nSwathLines == 1 ||
5352 2824 : nMemoryPerCol * nSwathLines <
5353 2824 : static_cast<GIntBig>(nTargetSwathSize) / 10)
5354 : {
5355 3348 : nSwathLines = std::min(
5356 : nYSize,
5357 3348 : std::max(1, static_cast<int>(nTargetSwathSize / nMemoryPerCol)));
5358 :
5359 : /* If possible try to align to source and target block height */
5360 3348 : if ((nSwathLines % nMaxBlockYSize) != 0 &&
5361 273 : nSwathLines > nMaxBlockYSize &&
5362 273 : IS_DIVIDER_OF(nBlockYSize, nMaxBlockYSize) &&
5363 244 : IS_DIVIDER_OF(nSrcBlockYSize, nMaxBlockYSize))
5364 217 : nSwathLines = ROUND_TO(nSwathLines, nMaxBlockYSize);
5365 : }
5366 :
5367 3377 : if (pszSrcCompression != nullptr && EQUAL(pszSrcCompression, "JPEG2000") &&
5368 0 : (!bDstIsCompressed || (IS_DIVIDER_OF(nBlockXSize, nSrcBlockXSize) &&
5369 0 : IS_DIVIDER_OF(nBlockYSize, nSrcBlockYSize))))
5370 : {
5371 : // Typical use case: converting from Pleaiades that is 2048x2048 tiled.
5372 2 : if (nSwathLines < nSrcBlockYSize)
5373 : {
5374 0 : nSwathLines = nSrcBlockYSize;
5375 :
5376 : // Number of pixels that can be read/write simultaneously.
5377 0 : nSwathCols = nTargetSwathSize / (nSrcBlockXSize * nPixelSize);
5378 0 : nSwathCols = ROUND_TO(nSwathCols, nSrcBlockXSize);
5379 0 : if (nSwathCols == 0)
5380 0 : nSwathCols = nSrcBlockXSize;
5381 0 : if (nSwathCols > nXSize)
5382 0 : nSwathCols = nXSize;
5383 :
5384 0 : CPLDebug(
5385 : "GDAL",
5386 : "GDALCopyWholeRasterGetSwathSize(): because of compression and "
5387 : "too high block, "
5388 : "use partial width at one time");
5389 : }
5390 2 : else if ((nSwathLines % nSrcBlockYSize) != 0)
5391 : {
5392 : /* Round on a multiple of nSrcBlockYSize */
5393 0 : nSwathLines = ROUND_TO(nSwathLines, nSrcBlockYSize);
5394 0 : CPLDebug(
5395 : "GDAL",
5396 : "GDALCopyWholeRasterGetSwathSize(): because of compression, "
5397 : "round nSwathLines to block height : %d",
5398 : nSwathLines);
5399 : }
5400 : }
5401 3375 : else if (bDstIsCompressed)
5402 : {
5403 419 : if (nSwathLines < nBlockYSize)
5404 : {
5405 146 : nSwathLines = nBlockYSize;
5406 :
5407 : // Number of pixels that can be read/write simultaneously.
5408 146 : nSwathCols = nTargetSwathSize / (nSwathLines * nPixelSize);
5409 146 : nSwathCols = ROUND_TO(nSwathCols, nBlockXSize);
5410 146 : if (nSwathCols == 0)
5411 0 : nSwathCols = nBlockXSize;
5412 146 : if (nSwathCols > nXSize)
5413 146 : nSwathCols = nXSize;
5414 :
5415 146 : CPLDebug(
5416 : "GDAL",
5417 : "GDALCopyWholeRasterGetSwathSize(): because of compression and "
5418 : "too high block, "
5419 : "use partial width at one time");
5420 : }
5421 273 : else if ((nSwathLines % nBlockYSize) != 0)
5422 : {
5423 : // Round on a multiple of nBlockYSize.
5424 9 : nSwathLines = ROUND_TO(nSwathLines, nBlockYSize);
5425 9 : CPLDebug(
5426 : "GDAL",
5427 : "GDALCopyWholeRasterGetSwathSize(): because of compression, "
5428 : "round nSwathLines to block height : %d",
5429 : nSwathLines);
5430 : }
5431 : }
5432 :
5433 3377 : *pnSwathCols = nSwathCols;
5434 3377 : *pnSwathLines = nSwathLines;
5435 3377 : }
5436 :
5437 : /************************************************************************/
5438 : /* GDALDatasetCopyWholeRaster() */
5439 : /************************************************************************/
5440 :
5441 : /**
5442 : * \brief Copy all dataset raster data.
5443 : *
5444 : * This function copies the complete raster contents of one dataset to
5445 : * another similarly configured dataset. The source and destination
5446 : * dataset must have the same number of bands, and the same width
5447 : * and height. The bands do not have to have the same data type.
5448 : *
5449 : * This function is primarily intended to support implementation of
5450 : * driver specific CreateCopy() functions. It implements efficient copying,
5451 : * in particular "chunking" the copy in substantial blocks and, if appropriate,
5452 : * performing the transfer in a pixel interleaved fashion.
5453 : *
5454 : * Currently the only papszOptions value supported are :
5455 : * <ul>
5456 : * <li>"INTERLEAVE=PIXEL/BAND" to force pixel (resp. band) interleaved read and
5457 : * write access pattern (this does not modify the layout of the destination
5458 : * data)</li>
5459 : * <li>"COMPRESSED=YES" to force alignment on target dataset block
5460 : * sizes to achieve best compression.</li>
5461 : * <li>"SKIP_HOLES=YES" to skip chunks
5462 : * for which GDALGetDataCoverageStatus() returns GDAL_DATA_COVERAGE_STATUS_EMPTY
5463 : * (GDAL >= 2.2)</li>
5464 : * </ul>
5465 : * More options may be supported in the future.
5466 : *
5467 : * @param hSrcDS the source dataset
5468 : * @param hDstDS the destination dataset
5469 : * @param papszOptions transfer hints in "StringList" Name=Value format.
5470 : * @param pfnProgress progress reporting function.
5471 : * @param pProgressData callback data for progress function.
5472 : *
5473 : * @return CE_None on success, or CE_Failure on failure.
5474 : */
5475 :
5476 3349 : CPLErr CPL_STDCALL GDALDatasetCopyWholeRaster(GDALDatasetH hSrcDS,
5477 : GDALDatasetH hDstDS,
5478 : CSLConstList papszOptions,
5479 : GDALProgressFunc pfnProgress,
5480 : void *pProgressData)
5481 :
5482 : {
5483 3349 : VALIDATE_POINTER1(hSrcDS, "GDALDatasetCopyWholeRaster", CE_Failure);
5484 3349 : VALIDATE_POINTER1(hDstDS, "GDALDatasetCopyWholeRaster", CE_Failure);
5485 :
5486 3349 : GDALDataset *poSrcDS = GDALDataset::FromHandle(hSrcDS);
5487 3349 : GDALDataset *poDstDS = GDALDataset::FromHandle(hDstDS);
5488 :
5489 3349 : if (pfnProgress == nullptr)
5490 0 : pfnProgress = GDALDummyProgress;
5491 :
5492 : /* -------------------------------------------------------------------- */
5493 : /* Confirm the datasets match in size and band counts. */
5494 : /* -------------------------------------------------------------------- */
5495 3349 : const int nXSize = poDstDS->GetRasterXSize();
5496 3349 : const int nYSize = poDstDS->GetRasterYSize();
5497 3349 : const int nBandCount = poDstDS->GetRasterCount();
5498 :
5499 3349 : if (poSrcDS->GetRasterXSize() != nXSize ||
5500 6698 : poSrcDS->GetRasterYSize() != nYSize ||
5501 3349 : poSrcDS->GetRasterCount() != nBandCount)
5502 : {
5503 0 : CPLError(CE_Failure, CPLE_AppDefined,
5504 : "Input and output dataset sizes or band counts do not\n"
5505 : "match in GDALDatasetCopyWholeRaster()");
5506 0 : return CE_Failure;
5507 : }
5508 :
5509 : /* -------------------------------------------------------------------- */
5510 : /* Report preliminary (0) progress. */
5511 : /* -------------------------------------------------------------------- */
5512 3349 : if (!pfnProgress(0.0, nullptr, pProgressData))
5513 : {
5514 1 : CPLError(CE_Failure, CPLE_UserInterrupt,
5515 : "User terminated CreateCopy()");
5516 1 : return CE_Failure;
5517 : }
5518 :
5519 : /* -------------------------------------------------------------------- */
5520 : /* Get our prototype band, and assume the others are similarly */
5521 : /* configured. */
5522 : /* -------------------------------------------------------------------- */
5523 3348 : if (nBandCount == 0)
5524 0 : return CE_None;
5525 :
5526 3348 : GDALRasterBand *poSrcPrototypeBand = poSrcDS->GetRasterBand(1);
5527 3348 : GDALRasterBand *poDstPrototypeBand = poDstDS->GetRasterBand(1);
5528 3348 : GDALDataType eDT = poDstPrototypeBand->GetRasterDataType();
5529 :
5530 : /* -------------------------------------------------------------------- */
5531 : /* Do we want to try and do the operation in a pixel */
5532 : /* interleaved fashion? */
5533 : /* -------------------------------------------------------------------- */
5534 3348 : bool bInterleave = false;
5535 : const char *pszInterleave =
5536 3348 : poSrcDS->GetMetadataItem("INTERLEAVE", "IMAGE_STRUCTURE");
5537 3348 : if (pszInterleave != nullptr &&
5538 2943 : (EQUAL(pszInterleave, "PIXEL") || EQUAL(pszInterleave, "LINE")))
5539 209 : bInterleave = true;
5540 :
5541 3348 : pszInterleave = poDstDS->GetMetadataItem("INTERLEAVE", "IMAGE_STRUCTURE");
5542 3348 : if (pszInterleave != nullptr &&
5543 2882 : (EQUAL(pszInterleave, "PIXEL") || EQUAL(pszInterleave, "LINE")))
5544 528 : bInterleave = true;
5545 :
5546 3348 : pszInterleave = CSLFetchNameValue(papszOptions, "INTERLEAVE");
5547 3348 : if (pszInterleave != nullptr && EQUAL(pszInterleave, "PIXEL"))
5548 5 : bInterleave = true;
5549 3343 : else if (pszInterleave != nullptr && EQUAL(pszInterleave, "BAND"))
5550 13 : bInterleave = false;
5551 : // attributes is specific to the TileDB driver
5552 3330 : else if (pszInterleave != nullptr && EQUAL(pszInterleave, "ATTRIBUTES"))
5553 4 : bInterleave = true;
5554 3326 : else if (pszInterleave != nullptr)
5555 : {
5556 0 : CPLError(CE_Warning, CPLE_NotSupported,
5557 : "Unsupported value for option INTERLEAVE");
5558 : }
5559 :
5560 : // If the destination is compressed, we must try to write blocks just once,
5561 : // to save disk space (GTiff case for example), and to avoid data loss
5562 : // (JPEG compression for example).
5563 3348 : bool bDstIsCompressed = false;
5564 : const char *pszDstCompressed =
5565 3348 : CSLFetchNameValue(papszOptions, "COMPRESSED");
5566 3348 : if (pszDstCompressed != nullptr && CPLTestBool(pszDstCompressed))
5567 393 : bDstIsCompressed = true;
5568 :
5569 : /* -------------------------------------------------------------------- */
5570 : /* What will our swath size be? */
5571 : /* -------------------------------------------------------------------- */
5572 :
5573 3348 : int nSwathCols = 0;
5574 3348 : int nSwathLines = 0;
5575 3348 : GDALCopyWholeRasterGetSwathSize(poSrcPrototypeBand, poDstPrototypeBand,
5576 : nBandCount, bDstIsCompressed, bInterleave,
5577 : &nSwathCols, &nSwathLines);
5578 :
5579 3348 : int nPixelSize = GDALGetDataTypeSizeBytes(eDT);
5580 3348 : if (bInterleave)
5581 583 : nPixelSize *= nBandCount;
5582 :
5583 3348 : void *pSwathBuf = VSI_MALLOC3_VERBOSE(nSwathCols, nSwathLines, nPixelSize);
5584 3348 : if (pSwathBuf == nullptr)
5585 : {
5586 0 : return CE_Failure;
5587 : }
5588 :
5589 3348 : CPLDebug("GDAL",
5590 : "GDALDatasetCopyWholeRaster(): %d*%d swaths, bInterleave=%d",
5591 : nSwathCols, nSwathLines, static_cast<int>(bInterleave));
5592 :
5593 : // Advise the source raster that we are going to read it completely
5594 : // Note: this might already have been done by GDALCreateCopy() in the
5595 : // likely case this function is indirectly called by it
5596 3348 : poSrcDS->AdviseRead(0, 0, nXSize, nYSize, nXSize, nYSize, eDT, nBandCount,
5597 3348 : nullptr, nullptr);
5598 :
5599 : /* ==================================================================== */
5600 : /* Band oriented (uninterleaved) case. */
5601 : /* ==================================================================== */
5602 3348 : CPLErr eErr = CE_None;
5603 : const bool bCheckHoles =
5604 3348 : CPLTestBool(CSLFetchNameValueDef(papszOptions, "SKIP_HOLES", "NO"));
5605 :
5606 3348 : if (!bInterleave)
5607 : {
5608 : GDALRasterIOExtraArg sExtraArg;
5609 2765 : INIT_RASTERIO_EXTRA_ARG(sExtraArg);
5610 2765 : CPL_IGNORE_RET_VAL(sExtraArg.pfnProgress); // to make cppcheck happy
5611 :
5612 8295 : const GIntBig nTotalBlocks = static_cast<GIntBig>(nBandCount) *
5613 2765 : DIV_ROUND_UP(nYSize, nSwathLines) *
5614 2765 : DIV_ROUND_UP(nXSize, nSwathCols);
5615 2765 : GIntBig nBlocksDone = 0;
5616 :
5617 7973 : for (int iBand = 0; iBand < nBandCount && eErr == CE_None; iBand++)
5618 : {
5619 5208 : int nBand = iBand + 1;
5620 :
5621 10679 : for (int iY = 0; iY < nYSize && eErr == CE_None; iY += nSwathLines)
5622 : {
5623 5471 : int nThisLines = nSwathLines;
5624 :
5625 5471 : if (iY + nThisLines > nYSize)
5626 368 : nThisLines = nYSize - iY;
5627 :
5628 10942 : for (int iX = 0; iX < nXSize && eErr == CE_None;
5629 5471 : iX += nSwathCols)
5630 : {
5631 5471 : int nThisCols = nSwathCols;
5632 :
5633 5471 : if (iX + nThisCols > nXSize)
5634 0 : nThisCols = nXSize - iX;
5635 :
5636 5471 : int nStatus = GDAL_DATA_COVERAGE_STATUS_DATA;
5637 5471 : if (bCheckHoles)
5638 : {
5639 : nStatus = poSrcDS->GetRasterBand(nBand)
5640 3757 : ->GetDataCoverageStatus(
5641 : iX, iY, nThisCols, nThisLines,
5642 : GDAL_DATA_COVERAGE_STATUS_DATA);
5643 : }
5644 5471 : if (nStatus & GDAL_DATA_COVERAGE_STATUS_DATA)
5645 : {
5646 5467 : sExtraArg.pfnProgress = GDALScaledProgress;
5647 10934 : sExtraArg.pProgressData = GDALCreateScaledProgress(
5648 5467 : nBlocksDone / static_cast<double>(nTotalBlocks),
5649 5467 : (nBlocksDone + 0.5) /
5650 5467 : static_cast<double>(nTotalBlocks),
5651 : pfnProgress, pProgressData);
5652 5467 : if (sExtraArg.pProgressData == nullptr)
5653 1684 : sExtraArg.pfnProgress = nullptr;
5654 :
5655 5467 : eErr = poSrcDS->RasterIO(GF_Read, iX, iY, nThisCols,
5656 : nThisLines, pSwathBuf,
5657 : nThisCols, nThisLines, eDT, 1,
5658 : &nBand, 0, 0, 0, &sExtraArg);
5659 :
5660 5467 : GDALDestroyScaledProgress(sExtraArg.pProgressData);
5661 :
5662 5467 : if (eErr == CE_None)
5663 5459 : eErr = poDstDS->RasterIO(
5664 : GF_Write, iX, iY, nThisCols, nThisLines,
5665 : pSwathBuf, nThisCols, nThisLines, eDT, 1,
5666 : &nBand, 0, 0, 0, nullptr);
5667 : }
5668 :
5669 5471 : nBlocksDone++;
5670 10899 : if (eErr == CE_None &&
5671 5428 : !pfnProgress(nBlocksDone /
5672 5428 : static_cast<double>(nTotalBlocks),
5673 : nullptr, pProgressData))
5674 : {
5675 2 : eErr = CE_Failure;
5676 2 : CPLError(CE_Failure, CPLE_UserInterrupt,
5677 : "User terminated CreateCopy()");
5678 : }
5679 : }
5680 : }
5681 : }
5682 : }
5683 :
5684 : /* ==================================================================== */
5685 : /* Pixel interleaved case. */
5686 : /* ==================================================================== */
5687 : else /* if( bInterleave ) */
5688 : {
5689 : GDALRasterIOExtraArg sExtraArg;
5690 583 : INIT_RASTERIO_EXTRA_ARG(sExtraArg);
5691 583 : CPL_IGNORE_RET_VAL(sExtraArg.pfnProgress); // to make cppcheck happy
5692 :
5693 583 : const GIntBig nTotalBlocks =
5694 583 : static_cast<GIntBig>(DIV_ROUND_UP(nYSize, nSwathLines)) *
5695 583 : DIV_ROUND_UP(nXSize, nSwathCols);
5696 583 : GIntBig nBlocksDone = 0;
5697 :
5698 1388 : for (int iY = 0; iY < nYSize && eErr == CE_None; iY += nSwathLines)
5699 : {
5700 805 : int nThisLines = nSwathLines;
5701 :
5702 805 : if (iY + nThisLines > nYSize)
5703 198 : nThisLines = nYSize - iY;
5704 :
5705 1615 : for (int iX = 0; iX < nXSize && eErr == CE_None; iX += nSwathCols)
5706 : {
5707 810 : int nThisCols = nSwathCols;
5708 :
5709 810 : if (iX + nThisCols > nXSize)
5710 3 : nThisCols = nXSize - iX;
5711 :
5712 810 : int nStatus = GDAL_DATA_COVERAGE_STATUS_DATA;
5713 810 : if (bCheckHoles)
5714 : {
5715 551 : nStatus = 0;
5716 604 : for (int iBand = 0; iBand < nBandCount; iBand++)
5717 : {
5718 585 : nStatus |= poSrcDS->GetRasterBand(iBand + 1)
5719 585 : ->GetDataCoverageStatus(
5720 : iX, iY, nThisCols, nThisLines,
5721 : GDAL_DATA_COVERAGE_STATUS_DATA);
5722 585 : if (nStatus & GDAL_DATA_COVERAGE_STATUS_DATA)
5723 532 : break;
5724 : }
5725 : }
5726 810 : if (nStatus & GDAL_DATA_COVERAGE_STATUS_DATA)
5727 : {
5728 791 : sExtraArg.pfnProgress = GDALScaledProgress;
5729 1582 : sExtraArg.pProgressData = GDALCreateScaledProgress(
5730 791 : nBlocksDone / static_cast<double>(nTotalBlocks),
5731 791 : (nBlocksDone + 0.5) / static_cast<double>(nTotalBlocks),
5732 : pfnProgress, pProgressData);
5733 791 : if (sExtraArg.pProgressData == nullptr)
5734 375 : sExtraArg.pfnProgress = nullptr;
5735 :
5736 791 : eErr = poSrcDS->RasterIO(GF_Read, iX, iY, nThisCols,
5737 : nThisLines, pSwathBuf, nThisCols,
5738 : nThisLines, eDT, nBandCount,
5739 : nullptr, 0, 0, 0, &sExtraArg);
5740 :
5741 791 : GDALDestroyScaledProgress(sExtraArg.pProgressData);
5742 :
5743 791 : if (eErr == CE_None)
5744 790 : eErr = poDstDS->RasterIO(
5745 : GF_Write, iX, iY, nThisCols, nThisLines, pSwathBuf,
5746 : nThisCols, nThisLines, eDT, nBandCount, nullptr, 0,
5747 : 0, 0, nullptr);
5748 : }
5749 :
5750 810 : nBlocksDone++;
5751 1615 : if (eErr == CE_None &&
5752 805 : !pfnProgress(nBlocksDone /
5753 805 : static_cast<double>(nTotalBlocks),
5754 : nullptr, pProgressData))
5755 : {
5756 1 : eErr = CE_Failure;
5757 1 : CPLError(CE_Failure, CPLE_UserInterrupt,
5758 : "User terminated CreateCopy()");
5759 : }
5760 : }
5761 : }
5762 : }
5763 :
5764 : /* -------------------------------------------------------------------- */
5765 : /* Cleanup */
5766 : /* -------------------------------------------------------------------- */
5767 3348 : CPLFree(pSwathBuf);
5768 :
5769 3348 : return eErr;
5770 : }
5771 :
5772 : /************************************************************************/
5773 : /* GDALRasterBandCopyWholeRaster() */
5774 : /************************************************************************/
5775 :
5776 : /**
5777 : * \brief Copy a whole raster band
5778 : *
5779 : * This function copies the complete raster contents of one band to
5780 : * another similarly configured band. The source and destination
5781 : * bands must have the same width and height. The bands do not have
5782 : * to have the same data type.
5783 : *
5784 : * It implements efficient copying, in particular "chunking" the copy in
5785 : * substantial blocks.
5786 : *
5787 : * Currently the only papszOptions value supported are :
5788 : * <ul>
5789 : * <li>"COMPRESSED=YES" to force alignment on target dataset block sizes to
5790 : * achieve best compression.</li>
5791 : * <li>"SKIP_HOLES=YES" to skip chunks for which GDALGetDataCoverageStatus()
5792 : * returns GDAL_DATA_COVERAGE_STATUS_EMPTY (GDAL >= 2.2)</li>
5793 : * </ul>
5794 : *
5795 : * @param hSrcBand the source band
5796 : * @param hDstBand the destination band
5797 : * @param papszOptions transfer hints in "StringList" Name=Value format.
5798 : * @param pfnProgress progress reporting function.
5799 : * @param pProgressData callback data for progress function.
5800 : *
5801 : * @return CE_None on success, or CE_Failure on failure.
5802 : */
5803 :
5804 29 : CPLErr CPL_STDCALL GDALRasterBandCopyWholeRaster(
5805 : GDALRasterBandH hSrcBand, GDALRasterBandH hDstBand,
5806 : const char *const *const papszOptions, GDALProgressFunc pfnProgress,
5807 : void *pProgressData)
5808 :
5809 : {
5810 29 : VALIDATE_POINTER1(hSrcBand, "GDALRasterBandCopyWholeRaster", CE_Failure);
5811 29 : VALIDATE_POINTER1(hDstBand, "GDALRasterBandCopyWholeRaster", CE_Failure);
5812 :
5813 29 : GDALRasterBand *poSrcBand = GDALRasterBand::FromHandle(hSrcBand);
5814 29 : GDALRasterBand *poDstBand = GDALRasterBand::FromHandle(hDstBand);
5815 29 : CPLErr eErr = CE_None;
5816 :
5817 29 : if (pfnProgress == nullptr)
5818 2 : pfnProgress = GDALDummyProgress;
5819 :
5820 : /* -------------------------------------------------------------------- */
5821 : /* Confirm the datasets match in size and band counts. */
5822 : /* -------------------------------------------------------------------- */
5823 29 : int nXSize = poSrcBand->GetXSize();
5824 29 : int nYSize = poSrcBand->GetYSize();
5825 :
5826 29 : if (poDstBand->GetXSize() != nXSize || poDstBand->GetYSize() != nYSize)
5827 : {
5828 0 : CPLError(CE_Failure, CPLE_AppDefined,
5829 : "Input and output band sizes do not\n"
5830 : "match in GDALRasterBandCopyWholeRaster()");
5831 0 : return CE_Failure;
5832 : }
5833 :
5834 : /* -------------------------------------------------------------------- */
5835 : /* Report preliminary (0) progress. */
5836 : /* -------------------------------------------------------------------- */
5837 29 : if (!pfnProgress(0.0, nullptr, pProgressData))
5838 : {
5839 0 : CPLError(CE_Failure, CPLE_UserInterrupt,
5840 : "User terminated CreateCopy()");
5841 0 : return CE_Failure;
5842 : }
5843 :
5844 29 : GDALDataType eDT = poDstBand->GetRasterDataType();
5845 :
5846 : // If the destination is compressed, we must try to write blocks just once,
5847 : // to save disk space (GTiff case for example), and to avoid data loss
5848 : // (JPEG compression for example).
5849 29 : bool bDstIsCompressed = false;
5850 : const char *pszDstCompressed =
5851 29 : CSLFetchNameValue(const_cast<char **>(papszOptions), "COMPRESSED");
5852 29 : if (pszDstCompressed != nullptr && CPLTestBool(pszDstCompressed))
5853 26 : bDstIsCompressed = true;
5854 :
5855 : /* -------------------------------------------------------------------- */
5856 : /* What will our swath size be? */
5857 : /* -------------------------------------------------------------------- */
5858 :
5859 29 : int nSwathCols = 0;
5860 29 : int nSwathLines = 0;
5861 29 : GDALCopyWholeRasterGetSwathSize(poSrcBand, poDstBand, 1, bDstIsCompressed,
5862 : FALSE, &nSwathCols, &nSwathLines);
5863 :
5864 29 : const int nPixelSize = GDALGetDataTypeSizeBytes(eDT);
5865 :
5866 29 : void *pSwathBuf = VSI_MALLOC3_VERBOSE(nSwathCols, nSwathLines, nPixelSize);
5867 29 : if (pSwathBuf == nullptr)
5868 : {
5869 0 : return CE_Failure;
5870 : }
5871 :
5872 29 : CPLDebug("GDAL", "GDALRasterBandCopyWholeRaster(): %d*%d swaths",
5873 : nSwathCols, nSwathLines);
5874 :
5875 : const bool bCheckHoles =
5876 29 : CPLTestBool(CSLFetchNameValueDef(papszOptions, "SKIP_HOLES", "NO"));
5877 :
5878 : // Advise the source raster that we are going to read it completely
5879 29 : poSrcBand->AdviseRead(0, 0, nXSize, nYSize, nXSize, nYSize, eDT, nullptr);
5880 :
5881 : /* ==================================================================== */
5882 : /* Band oriented (uninterleaved) case. */
5883 : /* ==================================================================== */
5884 :
5885 72 : for (int iY = 0; iY < nYSize && eErr == CE_None; iY += nSwathLines)
5886 : {
5887 43 : int nThisLines = nSwathLines;
5888 :
5889 43 : if (iY + nThisLines > nYSize)
5890 8 : nThisLines = nYSize - iY;
5891 :
5892 86 : for (int iX = 0; iX < nXSize && eErr == CE_None; iX += nSwathCols)
5893 : {
5894 43 : int nThisCols = nSwathCols;
5895 :
5896 43 : if (iX + nThisCols > nXSize)
5897 0 : nThisCols = nXSize - iX;
5898 :
5899 43 : int nStatus = GDAL_DATA_COVERAGE_STATUS_DATA;
5900 43 : if (bCheckHoles)
5901 : {
5902 0 : nStatus = poSrcBand->GetDataCoverageStatus(
5903 : iX, iY, nThisCols, nThisLines,
5904 : GDAL_DATA_COVERAGE_STATUS_DATA);
5905 : }
5906 43 : if (nStatus & GDAL_DATA_COVERAGE_STATUS_DATA)
5907 : {
5908 43 : eErr = poSrcBand->RasterIO(GF_Read, iX, iY, nThisCols,
5909 : nThisLines, pSwathBuf, nThisCols,
5910 : nThisLines, eDT, 0, 0, nullptr);
5911 :
5912 43 : if (eErr == CE_None)
5913 43 : eErr = poDstBand->RasterIO(GF_Write, iX, iY, nThisCols,
5914 : nThisLines, pSwathBuf, nThisCols,
5915 : nThisLines, eDT, 0, 0, nullptr);
5916 : }
5917 :
5918 86 : if (eErr == CE_None && !pfnProgress(double(iY + nThisLines) /
5919 43 : static_cast<double>(nYSize),
5920 : nullptr, pProgressData))
5921 : {
5922 0 : eErr = CE_Failure;
5923 0 : CPLError(CE_Failure, CPLE_UserInterrupt,
5924 : "User terminated CreateCopy()");
5925 : }
5926 : }
5927 : }
5928 :
5929 : /* -------------------------------------------------------------------- */
5930 : /* Cleanup */
5931 : /* -------------------------------------------------------------------- */
5932 29 : CPLFree(pSwathBuf);
5933 :
5934 29 : return eErr;
5935 : }
5936 :
5937 : /************************************************************************/
5938 : /* GDALCopyRasterIOExtraArg () */
5939 : /************************************************************************/
5940 :
5941 535023 : void GDALCopyRasterIOExtraArg(GDALRasterIOExtraArg *psDestArg,
5942 : const GDALRasterIOExtraArg *psSrcArg)
5943 : {
5944 535023 : INIT_RASTERIO_EXTRA_ARG(*psDestArg);
5945 535023 : if (psSrcArg)
5946 : {
5947 535023 : psDestArg->eResampleAlg = psSrcArg->eResampleAlg;
5948 535023 : psDestArg->pfnProgress = psSrcArg->pfnProgress;
5949 535023 : psDestArg->pProgressData = psSrcArg->pProgressData;
5950 535023 : psDestArg->bFloatingPointWindowValidity =
5951 535023 : psSrcArg->bFloatingPointWindowValidity;
5952 535023 : if (psSrcArg->bFloatingPointWindowValidity)
5953 : {
5954 212051 : psDestArg->dfXOff = psSrcArg->dfXOff;
5955 212051 : psDestArg->dfYOff = psSrcArg->dfYOff;
5956 212051 : psDestArg->dfXSize = psSrcArg->dfXSize;
5957 212051 : psDestArg->dfYSize = psSrcArg->dfYSize;
5958 : }
5959 535023 : if (psSrcArg->nVersion >= 2)
5960 : {
5961 535023 : psDestArg->bUseOnlyThisScale = psSrcArg->bUseOnlyThisScale;
5962 : }
5963 535023 : if (psSrcArg->nVersion >= 3)
5964 : {
5965 535023 : psDestArg->bOperateInBufType = psSrcArg->bOperateInBufType;
5966 : }
5967 : }
5968 535023 : }
5969 :
5970 : /************************************************************************/
5971 : /* HasOnlyNoData() */
5972 : /************************************************************************/
5973 :
5974 51285976 : template <class T> static inline bool IsEqualToNoData(T value, T noDataValue)
5975 : {
5976 51285976 : return value == noDataValue;
5977 : }
5978 :
5979 5509 : template <> bool IsEqualToNoData<GFloat16>(GFloat16 value, GFloat16 noDataValue)
5980 : {
5981 : using std::isnan;
5982 5509 : return isnan(noDataValue) ? isnan(value) : value == noDataValue;
5983 : }
5984 :
5985 251221 : template <> bool IsEqualToNoData<float>(float value, float noDataValue)
5986 : {
5987 251221 : return std::isnan(noDataValue) ? std::isnan(value) : value == noDataValue;
5988 : }
5989 :
5990 264257 : template <> bool IsEqualToNoData<double>(double value, double noDataValue)
5991 : {
5992 264257 : return std::isnan(noDataValue) ? std::isnan(value) : value == noDataValue;
5993 : }
5994 :
5995 : template <class T>
5996 12024 : static bool HasOnlyNoDataT(const T *pBuffer, T noDataValue, size_t nWidth,
5997 : size_t nHeight, size_t nLineStride,
5998 : size_t nComponents)
5999 : {
6000 : // Fast test: check the 4 corners and the middle pixel.
6001 23297 : for (size_t iBand = 0; iBand < nComponents; iBand++)
6002 : {
6003 24095 : if (!(IsEqualToNoData(pBuffer[iBand], noDataValue) &&
6004 11880 : IsEqualToNoData(pBuffer[(nWidth - 1) * nComponents + iBand],
6005 11750 : noDataValue) &&
6006 11750 : IsEqualToNoData(
6007 11750 : pBuffer[((nHeight - 1) / 2 * nLineStride + (nWidth - 1) / 2) *
6008 11750 : nComponents +
6009 : iBand],
6010 11276 : noDataValue) &&
6011 11276 : IsEqualToNoData(
6012 11276 : pBuffer[(nHeight - 1) * nLineStride * nComponents + iBand],
6013 : noDataValue) &&
6014 11276 : IsEqualToNoData(
6015 11276 : pBuffer[((nHeight - 1) * nLineStride + nWidth - 1) *
6016 11276 : nComponents +
6017 : iBand],
6018 : noDataValue)))
6019 : {
6020 942 : return false;
6021 : }
6022 : }
6023 :
6024 : // Test all pixels.
6025 52954 : for (size_t iY = 0; iY < nHeight; iY++)
6026 : {
6027 41993 : const T *pBufferLine = pBuffer + iY * nLineStride * nComponents;
6028 51790448 : for (size_t iX = 0; iX < nWidth * nComponents; iX++)
6029 : {
6030 51748615 : if (!IsEqualToNoData(pBufferLine[iX], noDataValue))
6031 : {
6032 121 : return false;
6033 : }
6034 : }
6035 : }
6036 10961 : return true;
6037 : }
6038 :
6039 : /************************************************************************/
6040 : /* GDALBufferHasOnlyNoData() */
6041 : /************************************************************************/
6042 :
6043 43911 : bool GDALBufferHasOnlyNoData(const void *pBuffer, double dfNoDataValue,
6044 : size_t nWidth, size_t nHeight, size_t nLineStride,
6045 : size_t nComponents, int nBitsPerSample,
6046 : GDALBufferSampleFormat nSampleFormat)
6047 : {
6048 : // In the case where the nodata is 0, we can compare several bytes at
6049 : // once. Select the largest natural integer type for the architecture.
6050 43911 : if (dfNoDataValue == 0.0 && nWidth == nLineStride &&
6051 : // Do not use this optimized code path for floating point numbers,
6052 : // as it can't detect negative zero.
6053 : nSampleFormat != GSF_FLOATING_POINT)
6054 : {
6055 27265 : const GByte *pabyBuffer = static_cast<const GByte *>(pBuffer);
6056 27265 : const size_t nSize =
6057 27265 : static_cast<size_t>((static_cast<uint64_t>(nWidth) * nHeight *
6058 27265 : nComponents * nBitsPerSample +
6059 : 7) /
6060 : 8);
6061 : #ifdef HAVE_SSE2
6062 27265 : size_t n = nSize;
6063 : // Align to 16 bytes
6064 27328 : while ((reinterpret_cast<uintptr_t>(pabyBuffer) & 15) != 0 && n > 0)
6065 : {
6066 73 : --n;
6067 73 : if (*pabyBuffer)
6068 10 : return false;
6069 63 : pabyBuffer++;
6070 : }
6071 :
6072 27255 : const auto zero = _mm_setzero_si128();
6073 27255 : constexpr int UNROLLING = 4;
6074 2223230 : while (n >= UNROLLING * sizeof(zero))
6075 : {
6076 2207980 : const auto v0 = _mm_load_si128(reinterpret_cast<const __m128i *>(
6077 : pabyBuffer + 0 * sizeof(zero)));
6078 2207980 : const auto v1 = _mm_load_si128(reinterpret_cast<const __m128i *>(
6079 2207980 : pabyBuffer + 1 * sizeof(zero)));
6080 2207980 : const auto v2 = _mm_load_si128(reinterpret_cast<const __m128i *>(
6081 2207980 : pabyBuffer + 2 * sizeof(zero)));
6082 2207980 : const auto v3 = _mm_load_si128(reinterpret_cast<const __m128i *>(
6083 2207980 : pabyBuffer + 3 * sizeof(zero)));
6084 : const auto v =
6085 6623930 : _mm_or_si128(_mm_or_si128(v0, v1), _mm_or_si128(v2, v3));
6086 : #if defined(__SSE4_1__) || defined(USE_NEON_OPTIMIZATIONS)
6087 : if (!_mm_test_all_zeros(v, v))
6088 : #else
6089 4415960 : if (_mm_movemask_epi8(_mm_cmpeq_epi8(v, zero)) != 0xFFFF)
6090 : #endif
6091 : {
6092 12001 : return false;
6093 : }
6094 2195980 : pabyBuffer += UNROLLING * sizeof(zero);
6095 2195980 : n -= UNROLLING * sizeof(zero);
6096 : }
6097 :
6098 233639 : while (n > 0)
6099 : {
6100 218489 : --n;
6101 218489 : if (*pabyBuffer)
6102 104 : return false;
6103 218385 : pabyBuffer++;
6104 : }
6105 : #else
6106 : #if SIZEOF_VOIDP >= 8 || defined(__x86_64__)
6107 : // We test __x86_64__ for x32 arch where SIZEOF_VOIDP == 4
6108 : typedef std::uint64_t WordType;
6109 : #else
6110 : typedef std::uint32_t WordType;
6111 : #endif
6112 :
6113 : const size_t nInitialIters =
6114 : std::min(sizeof(WordType) -
6115 : static_cast<size_t>(
6116 : reinterpret_cast<std::uintptr_t>(pabyBuffer) %
6117 : sizeof(WordType)),
6118 : nSize);
6119 : size_t i = 0;
6120 : for (; i < nInitialIters; i++)
6121 : {
6122 : if (pabyBuffer[i])
6123 : return false;
6124 : }
6125 : for (; i + sizeof(WordType) - 1 < nSize; i += sizeof(WordType))
6126 : {
6127 : if (*(reinterpret_cast<const WordType *>(pabyBuffer + i)))
6128 : return false;
6129 : }
6130 : for (; i < nSize; i++)
6131 : {
6132 : if (pabyBuffer[i])
6133 : return false;
6134 : }
6135 : #endif
6136 15150 : return true;
6137 : }
6138 :
6139 : #ifdef HAVE_SSE2
6140 16646 : else if (dfNoDataValue == 0.0 && nWidth == nLineStride &&
6141 710 : nBitsPerSample == 32 && nSampleFormat == GSF_FLOATING_POINT)
6142 : {
6143 710 : const auto signMask = _mm_set1_epi32(0x7FFFFFFF);
6144 710 : const auto zero = _mm_setzero_si128();
6145 710 : const GByte *pabyBuffer = static_cast<const GByte *>(pBuffer);
6146 710 : const size_t n = nWidth * nHeight * nComponents;
6147 :
6148 710 : size_t i = 0;
6149 710 : constexpr int UNROLLING = 4;
6150 710 : constexpr size_t VALUES_PER_ITER =
6151 : UNROLLING * sizeof(zero) / sizeof(float);
6152 24985 : for (; i + VALUES_PER_ITER <= n; i += VALUES_PER_ITER)
6153 : {
6154 24936 : const auto v0 = _mm_loadu_si128(reinterpret_cast<const __m128i *>(
6155 : pabyBuffer + 0 * sizeof(zero)));
6156 24936 : const auto v1 = _mm_loadu_si128(reinterpret_cast<const __m128i *>(
6157 24936 : pabyBuffer + 1 * sizeof(zero)));
6158 24936 : const auto v2 = _mm_loadu_si128(reinterpret_cast<const __m128i *>(
6159 24936 : pabyBuffer + 2 * sizeof(zero)));
6160 24936 : const auto v3 = _mm_loadu_si128(reinterpret_cast<const __m128i *>(
6161 24936 : pabyBuffer + 3 * sizeof(zero)));
6162 74808 : auto v = _mm_or_si128(_mm_or_si128(v0, v1), _mm_or_si128(v2, v3));
6163 : // Clear the sign bit (makes -0.0 become +0.0)
6164 24936 : v = _mm_and_si128(v, signMask);
6165 : #if defined(__SSE4_1__) || defined(USE_NEON_OPTIMIZATIONS)
6166 : if (!_mm_test_all_zeros(v, v))
6167 : #else
6168 49872 : if (_mm_movemask_epi8(_mm_cmpeq_epi8(v, zero)) != 0xFFFF)
6169 : #endif
6170 : {
6171 661 : return false;
6172 : }
6173 24275 : pabyBuffer += UNROLLING * sizeof(zero);
6174 : }
6175 :
6176 304 : for (; i < n; i++)
6177 : {
6178 : uint32_t bits;
6179 272 : memcpy(&bits, pabyBuffer, sizeof(bits));
6180 272 : pabyBuffer += sizeof(bits);
6181 272 : if ((bits & 0x7FFFFFFF) != 0)
6182 17 : return false;
6183 : }
6184 :
6185 32 : return true;
6186 : }
6187 :
6188 15936 : else if (dfNoDataValue == 0.0 && nWidth == nLineStride &&
6189 3905 : nBitsPerSample == 64 && nSampleFormat == GSF_FLOATING_POINT)
6190 : {
6191 3905 : const auto signMask = _mm_set1_epi64x(0x7FFFFFFFFFFFFFFFLL);
6192 3905 : const auto zero = _mm_setzero_si128();
6193 3905 : const GByte *pabyBuffer = static_cast<const GByte *>(pBuffer);
6194 3905 : const size_t n = nWidth * nHeight * nComponents;
6195 :
6196 3905 : size_t i = 0;
6197 3905 : constexpr int UNROLLING = 4;
6198 3905 : constexpr size_t VALUES_PER_ITER =
6199 : UNROLLING * sizeof(zero) / sizeof(double);
6200 1664570 : for (; i + VALUES_PER_ITER <= n; i += VALUES_PER_ITER)
6201 : {
6202 1660950 : const auto v0 = _mm_loadu_si128(reinterpret_cast<const __m128i *>(
6203 : pabyBuffer + 0 * sizeof(zero)));
6204 1660950 : const auto v1 = _mm_loadu_si128(reinterpret_cast<const __m128i *>(
6205 1660950 : pabyBuffer + 1 * sizeof(zero)));
6206 1660950 : const auto v2 = _mm_loadu_si128(reinterpret_cast<const __m128i *>(
6207 1660950 : pabyBuffer + 2 * sizeof(zero)));
6208 1660950 : const auto v3 = _mm_loadu_si128(reinterpret_cast<const __m128i *>(
6209 1660950 : pabyBuffer + 3 * sizeof(zero)));
6210 4982850 : auto v = _mm_or_si128(_mm_or_si128(v0, v1), _mm_or_si128(v2, v3));
6211 : // Clear the sign bit (makes -0.0 become +0.0)
6212 1660950 : v = _mm_and_si128(v, signMask);
6213 : #if defined(__SSE4_1__) || defined(USE_NEON_OPTIMIZATIONS)
6214 : if (!_mm_test_all_zeros(v, v))
6215 : #else
6216 3321900 : if (_mm_movemask_epi8(_mm_cmpeq_epi8(v, zero)) != 0xFFFF)
6217 : #endif
6218 : {
6219 289 : return false;
6220 : }
6221 1660660 : pabyBuffer += UNROLLING * sizeof(zero);
6222 : }
6223 :
6224 3643 : for (; i < n; i++)
6225 : {
6226 : uint64_t bits;
6227 34 : memcpy(&bits, pabyBuffer, sizeof(bits));
6228 34 : pabyBuffer += sizeof(bits);
6229 34 : if ((bits & 0x7FFFFFFFFFFFFFFFULL) != 0)
6230 7 : return false;
6231 : }
6232 :
6233 3609 : return true;
6234 : }
6235 : #endif
6236 :
6237 12031 : if (nBitsPerSample == 8 && nSampleFormat == GSF_UNSIGNED_INT)
6238 : {
6239 22424 : return GDALIsValueInRange<uint8_t>(dfNoDataValue) &&
6240 11212 : HasOnlyNoDataT(static_cast<const uint8_t *>(pBuffer),
6241 11212 : static_cast<uint8_t>(dfNoDataValue), nWidth,
6242 11212 : nHeight, nLineStride, nComponents);
6243 : }
6244 819 : if (nBitsPerSample == 8 && nSampleFormat == GSF_SIGNED_INT)
6245 : {
6246 : // Use unsigned implementation by converting the nodatavalue to
6247 : // unsigned
6248 119 : return GDALIsValueInRange<int8_t>(dfNoDataValue) &&
6249 59 : HasOnlyNoDataT(
6250 : static_cast<const uint8_t *>(pBuffer),
6251 59 : static_cast<uint8_t>(static_cast<int8_t>(dfNoDataValue)),
6252 60 : nWidth, nHeight, nLineStride, nComponents);
6253 : }
6254 759 : if (nBitsPerSample == 16 && nSampleFormat == GSF_UNSIGNED_INT)
6255 : {
6256 23 : return GDALIsValueInRange<uint16_t>(dfNoDataValue) &&
6257 11 : HasOnlyNoDataT(static_cast<const uint16_t *>(pBuffer),
6258 11 : static_cast<uint16_t>(dfNoDataValue), nWidth,
6259 12 : nHeight, nLineStride, nComponents);
6260 : }
6261 747 : if (nBitsPerSample == 16 && nSampleFormat == GSF_SIGNED_INT)
6262 : {
6263 : // Use unsigned implementation by converting the nodatavalue to
6264 : // unsigned
6265 111 : return GDALIsValueInRange<int16_t>(dfNoDataValue) &&
6266 55 : HasOnlyNoDataT(
6267 : static_cast<const uint16_t *>(pBuffer),
6268 55 : static_cast<uint16_t>(static_cast<int16_t>(dfNoDataValue)),
6269 56 : nWidth, nHeight, nLineStride, nComponents);
6270 : }
6271 691 : if (nBitsPerSample == 32 && nSampleFormat == GSF_UNSIGNED_INT)
6272 : {
6273 129 : return GDALIsValueInRange<uint32_t>(dfNoDataValue) &&
6274 64 : HasOnlyNoDataT(static_cast<const uint32_t *>(pBuffer),
6275 : static_cast<uint32_t>(dfNoDataValue), nWidth,
6276 65 : nHeight, nLineStride, nComponents);
6277 : }
6278 626 : if (nBitsPerSample == 32 && nSampleFormat == GSF_SIGNED_INT)
6279 : {
6280 : // Use unsigned implementation by converting the nodatavalue to
6281 : // unsigned
6282 23 : return GDALIsValueInRange<int32_t>(dfNoDataValue) &&
6283 11 : HasOnlyNoDataT(
6284 : static_cast<const uint32_t *>(pBuffer),
6285 11 : static_cast<uint32_t>(static_cast<int32_t>(dfNoDataValue)),
6286 12 : nWidth, nHeight, nLineStride, nComponents);
6287 : }
6288 614 : if (nBitsPerSample == 64 && nSampleFormat == GSF_UNSIGNED_INT)
6289 : {
6290 112 : return GDALIsValueInRange<uint64_t>(dfNoDataValue) &&
6291 56 : HasOnlyNoDataT(static_cast<const uint64_t *>(pBuffer),
6292 : static_cast<uint64_t>(dfNoDataValue), nWidth,
6293 56 : nHeight, nLineStride, nComponents);
6294 : }
6295 558 : if (nBitsPerSample == 64 && nSampleFormat == GSF_SIGNED_INT)
6296 : {
6297 : // Use unsigned implementation by converting the nodatavalue to
6298 : // unsigned
6299 0 : return GDALIsValueInRange<int64_t>(dfNoDataValue) &&
6300 0 : HasOnlyNoDataT(
6301 : static_cast<const uint64_t *>(pBuffer),
6302 0 : static_cast<uint64_t>(static_cast<int64_t>(dfNoDataValue)),
6303 0 : nWidth, nHeight, nLineStride, nComponents);
6304 : }
6305 558 : if (nBitsPerSample == 16 && nSampleFormat == GSF_FLOATING_POINT)
6306 : {
6307 106 : return (std::isnan(dfNoDataValue) ||
6308 211 : GDALIsValueInRange<GFloat16>(dfNoDataValue)) &&
6309 105 : HasOnlyNoDataT(static_cast<const GFloat16 *>(pBuffer),
6310 : static_cast<GFloat16>(dfNoDataValue), nWidth,
6311 106 : nHeight, nLineStride, nComponents);
6312 : }
6313 452 : if (nBitsPerSample == 32 && nSampleFormat == GSF_FLOATING_POINT)
6314 : {
6315 268 : return (std::isnan(dfNoDataValue) ||
6316 535 : GDALIsValueInRange<float>(dfNoDataValue)) &&
6317 267 : HasOnlyNoDataT(static_cast<const float *>(pBuffer),
6318 : static_cast<float>(dfNoDataValue), nWidth,
6319 268 : nHeight, nLineStride, nComponents);
6320 : }
6321 184 : if (nBitsPerSample == 64 && nSampleFormat == GSF_FLOATING_POINT)
6322 : {
6323 184 : return HasOnlyNoDataT(static_cast<const double *>(pBuffer),
6324 : dfNoDataValue, nWidth, nHeight, nLineStride,
6325 184 : nComponents);
6326 : }
6327 0 : return false;
6328 : }
6329 :
6330 : #ifdef HAVE_SSE2
6331 :
6332 : /************************************************************************/
6333 : /* GDALDeinterleave3Byte() */
6334 : /************************************************************************/
6335 :
6336 : #if defined(__GNUC__) && !defined(__clang__)
6337 : __attribute__((optimize("no-tree-vectorize")))
6338 : #endif
6339 382254 : static void GDALDeinterleave3Byte(const GByte *CPL_RESTRICT pabySrc,
6340 : GByte *CPL_RESTRICT pabyDest0,
6341 : GByte *CPL_RESTRICT pabyDest1,
6342 : GByte *CPL_RESTRICT pabyDest2, size_t nIters)
6343 : #ifdef USE_NEON_OPTIMIZATIONS
6344 : {
6345 : return GDALDeinterleave3Byte_SSSE3(pabySrc, pabyDest0, pabyDest1, pabyDest2,
6346 : nIters);
6347 : }
6348 : #else
6349 : {
6350 : #ifdef HAVE_SSSE3_AT_COMPILE_TIME
6351 382254 : if (CPLHaveRuntimeSSSE3())
6352 : {
6353 382252 : return GDALDeinterleave3Byte_SSSE3(pabySrc, pabyDest0, pabyDest1,
6354 382252 : pabyDest2, nIters);
6355 : }
6356 : #endif
6357 :
6358 2 : size_t i = 0;
6359 2 : if (((reinterpret_cast<uintptr_t>(pabySrc) |
6360 2 : reinterpret_cast<uintptr_t>(pabyDest0) |
6361 2 : reinterpret_cast<uintptr_t>(pabyDest1) |
6362 2 : reinterpret_cast<uintptr_t>(pabyDest2)) %
6363 : sizeof(unsigned int)) == 0)
6364 : {
6365 : // Slightly better than GCC autovectorizer
6366 17 : for (size_t j = 0; i + 3 < nIters; i += 4, ++j)
6367 : {
6368 15 : unsigned int word0 =
6369 15 : *reinterpret_cast<const unsigned int *>(pabySrc + 3 * i);
6370 15 : unsigned int word1 =
6371 15 : *reinterpret_cast<const unsigned int *>(pabySrc + 3 * i + 4);
6372 15 : unsigned int word2 =
6373 15 : *reinterpret_cast<const unsigned int *>(pabySrc + 3 * i + 8);
6374 15 : reinterpret_cast<unsigned int *>(pabyDest0)[j] =
6375 15 : (word0 & 0xff) | ((word0 >> 24) << 8) | (word1 & 0x00ff0000) |
6376 15 : ((word2 >> 8) << 24);
6377 15 : reinterpret_cast<unsigned int *>(pabyDest1)[j] =
6378 15 : ((word0 >> 8) & 0xff) | ((word1 & 0xff) << 8) |
6379 15 : (((word1 >> 24)) << 16) | ((word2 >> 16) << 24);
6380 15 : pabyDest2[j * 4] = static_cast<GByte>(word0 >> 16);
6381 15 : pabyDest2[j * 4 + 1] = static_cast<GByte>(word1 >> 8);
6382 15 : pabyDest2[j * 4 + 2] = static_cast<GByte>(word2);
6383 15 : pabyDest2[j * 4 + 3] = static_cast<GByte>(word2 >> 24);
6384 : }
6385 : }
6386 : #if defined(__clang__)
6387 : #pragma clang loop vectorize(disable)
6388 : #endif
6389 3 : for (; i < nIters; ++i)
6390 : {
6391 1 : pabyDest0[i] = pabySrc[3 * i + 0];
6392 1 : pabyDest1[i] = pabySrc[3 * i + 1];
6393 1 : pabyDest2[i] = pabySrc[3 * i + 2];
6394 : }
6395 : }
6396 : #endif
6397 :
6398 : /************************************************************************/
6399 : /* GDALDeinterleave4Byte() */
6400 : /************************************************************************/
6401 :
6402 : #if !defined(__GNUC__) || defined(__clang__)
6403 :
6404 : /************************************************************************/
6405 : /* deinterleave() */
6406 : /************************************************************************/
6407 :
6408 : template <bool SHIFT, bool MASK>
6409 : inline __m128i deinterleave(__m128i &xmm0_ori, __m128i &xmm1_ori,
6410 : __m128i &xmm2_ori, __m128i &xmm3_ori)
6411 : {
6412 : // Set higher 24bit of each int32 packed word to 0
6413 : if (SHIFT)
6414 : {
6415 : xmm0_ori = _mm_srli_epi32(xmm0_ori, 8);
6416 : xmm1_ori = _mm_srli_epi32(xmm1_ori, 8);
6417 : xmm2_ori = _mm_srli_epi32(xmm2_ori, 8);
6418 : xmm3_ori = _mm_srli_epi32(xmm3_ori, 8);
6419 : }
6420 : __m128i xmm0;
6421 : __m128i xmm1;
6422 : __m128i xmm2;
6423 : __m128i xmm3;
6424 : if (MASK)
6425 : {
6426 : const __m128i xmm_mask = _mm_set1_epi32(0xff);
6427 : xmm0 = _mm_and_si128(xmm0_ori, xmm_mask);
6428 : xmm1 = _mm_and_si128(xmm1_ori, xmm_mask);
6429 : xmm2 = _mm_and_si128(xmm2_ori, xmm_mask);
6430 : xmm3 = _mm_and_si128(xmm3_ori, xmm_mask);
6431 : }
6432 : else
6433 : {
6434 : xmm0 = xmm0_ori;
6435 : xmm1 = xmm1_ori;
6436 : xmm2 = xmm2_ori;
6437 : xmm3 = xmm3_ori;
6438 : }
6439 : // Pack int32 to int16
6440 : xmm0 = _mm_packs_epi32(xmm0, xmm1);
6441 : xmm2 = _mm_packs_epi32(xmm2, xmm3);
6442 : // Pack int16 to uint8
6443 : xmm0 = _mm_packus_epi16(xmm0, xmm2);
6444 : return xmm0;
6445 : }
6446 :
6447 : static void GDALDeinterleave4Byte(const GByte *CPL_RESTRICT pabySrc,
6448 : GByte *CPL_RESTRICT pabyDest0,
6449 : GByte *CPL_RESTRICT pabyDest1,
6450 : GByte *CPL_RESTRICT pabyDest2,
6451 : GByte *CPL_RESTRICT pabyDest3, size_t nIters)
6452 : #ifdef USE_NEON_OPTIMIZATIONS
6453 : {
6454 : return GDALDeinterleave4Byte_SSSE3(pabySrc, pabyDest0, pabyDest1, pabyDest2,
6455 : pabyDest3, nIters);
6456 : }
6457 : #else
6458 : {
6459 : #ifdef HAVE_SSSE3_AT_COMPILE_TIME
6460 : if (CPLHaveRuntimeSSSE3())
6461 : {
6462 : return GDALDeinterleave4Byte_SSSE3(pabySrc, pabyDest0, pabyDest1,
6463 : pabyDest2, pabyDest3, nIters);
6464 : }
6465 : #endif
6466 :
6467 : // Not the optimal SSE2-only code, as gcc auto-vectorizer manages to
6468 : // do something slightly better.
6469 : size_t i = 0;
6470 : for (; i + 15 < nIters; i += 16)
6471 : {
6472 : __m128i xmm0_ori = _mm_loadu_si128(
6473 : reinterpret_cast<__m128i const *>(pabySrc + 4 * i + 0));
6474 : __m128i xmm1_ori = _mm_loadu_si128(
6475 : reinterpret_cast<__m128i const *>(pabySrc + 4 * i + 16));
6476 : __m128i xmm2_ori = _mm_loadu_si128(
6477 : reinterpret_cast<__m128i const *>(pabySrc + 4 * i + 32));
6478 : __m128i xmm3_ori = _mm_loadu_si128(
6479 : reinterpret_cast<__m128i const *>(pabySrc + 4 * i + 48));
6480 :
6481 : _mm_storeu_si128(
6482 : reinterpret_cast<__m128i *>(pabyDest0 + i),
6483 : deinterleave<false, true>(xmm0_ori, xmm1_ori, xmm2_ori, xmm3_ori));
6484 : _mm_storeu_si128(
6485 : reinterpret_cast<__m128i *>(pabyDest1 + i),
6486 : deinterleave<true, true>(xmm0_ori, xmm1_ori, xmm2_ori, xmm3_ori));
6487 : _mm_storeu_si128(
6488 : reinterpret_cast<__m128i *>(pabyDest2 + i),
6489 : deinterleave<true, true>(xmm0_ori, xmm1_ori, xmm2_ori, xmm3_ori));
6490 : _mm_storeu_si128(
6491 : reinterpret_cast<__m128i *>(pabyDest3 + i),
6492 : deinterleave<true, false>(xmm0_ori, xmm1_ori, xmm2_ori, xmm3_ori));
6493 : }
6494 :
6495 : #if defined(__clang__)
6496 : #pragma clang loop vectorize(disable)
6497 : #endif
6498 : for (; i < nIters; ++i)
6499 : {
6500 : pabyDest0[i] = pabySrc[4 * i + 0];
6501 : pabyDest1[i] = pabySrc[4 * i + 1];
6502 : pabyDest2[i] = pabySrc[4 * i + 2];
6503 : pabyDest3[i] = pabySrc[4 * i + 3];
6504 : }
6505 : }
6506 : #endif
6507 : #else
6508 : // GCC autovectorizer does an excellent job
6509 97794 : __attribute__((optimize("tree-vectorize"))) static void GDALDeinterleave4Byte(
6510 : const GByte *CPL_RESTRICT pabySrc, GByte *CPL_RESTRICT pabyDest0,
6511 : GByte *CPL_RESTRICT pabyDest1, GByte *CPL_RESTRICT pabyDest2,
6512 : GByte *CPL_RESTRICT pabyDest3, size_t nIters)
6513 : {
6514 545702000 : for (size_t i = 0; i < nIters; ++i)
6515 : {
6516 545604000 : pabyDest0[i] = pabySrc[4 * i + 0];
6517 545604000 : pabyDest1[i] = pabySrc[4 * i + 1];
6518 545604000 : pabyDest2[i] = pabySrc[4 * i + 2];
6519 545604000 : pabyDest3[i] = pabySrc[4 * i + 3];
6520 : }
6521 97794 : }
6522 : #endif
6523 :
6524 : #else
6525 :
6526 : /************************************************************************/
6527 : /* GDALDeinterleave3Byte() */
6528 : /************************************************************************/
6529 :
6530 : // TODO: Enabling below could help on non-Intel architectures where GCC knows
6531 : // how to auto-vectorize
6532 : // #if defined(__GNUC__)
6533 : //__attribute__((optimize("tree-vectorize")))
6534 : // #endif
6535 : static void GDALDeinterleave3Byte(const GByte *CPL_RESTRICT pabySrc,
6536 : GByte *CPL_RESTRICT pabyDest0,
6537 : GByte *CPL_RESTRICT pabyDest1,
6538 : GByte *CPL_RESTRICT pabyDest2, size_t nIters)
6539 : {
6540 : for (size_t i = 0; i < nIters; ++i)
6541 : {
6542 : pabyDest0[i] = pabySrc[3 * i + 0];
6543 : pabyDest1[i] = pabySrc[3 * i + 1];
6544 : pabyDest2[i] = pabySrc[3 * i + 2];
6545 : }
6546 : }
6547 :
6548 : /************************************************************************/
6549 : /* GDALDeinterleave4Byte() */
6550 : /************************************************************************/
6551 :
6552 : // TODO: Enabling below could help on non-Intel architectures where gcc knows
6553 : // how to auto-vectorize
6554 : // #if defined(__GNUC__)
6555 : //__attribute__((optimize("tree-vectorize")))
6556 : // #endif
6557 : static void GDALDeinterleave4Byte(const GByte *CPL_RESTRICT pabySrc,
6558 : GByte *CPL_RESTRICT pabyDest0,
6559 : GByte *CPL_RESTRICT pabyDest1,
6560 : GByte *CPL_RESTRICT pabyDest2,
6561 : GByte *CPL_RESTRICT pabyDest3, size_t nIters)
6562 : {
6563 : for (size_t i = 0; i < nIters; ++i)
6564 : {
6565 : pabyDest0[i] = pabySrc[4 * i + 0];
6566 : pabyDest1[i] = pabySrc[4 * i + 1];
6567 : pabyDest2[i] = pabySrc[4 * i + 2];
6568 : pabyDest3[i] = pabySrc[4 * i + 3];
6569 : }
6570 : }
6571 :
6572 : #endif
6573 :
6574 : /************************************************************************/
6575 : /* GDALDeinterleave() */
6576 : /************************************************************************/
6577 :
6578 : /*! Copy values from a pixel-interleave buffer to multiple per-component
6579 : buffers.
6580 :
6581 : In pseudo-code
6582 : \verbatim
6583 : for(size_t i = 0; i < nIters; ++i)
6584 : for(int iComp = 0; iComp < nComponents; iComp++ )
6585 : ppDestBuffer[iComp][i] = pSourceBuffer[nComponents * i + iComp]
6586 : \endverbatim
6587 :
6588 : The implementation is optimized for a few cases, like de-interleaving
6589 : of 3 or 4-components Byte buffers.
6590 :
6591 : \since GDAL 3.6
6592 : */
6593 480398 : void GDALDeinterleave(const void *pSourceBuffer, GDALDataType eSourceDT,
6594 : int nComponents, void **ppDestBuffer,
6595 : GDALDataType eDestDT, size_t nIters)
6596 : {
6597 480398 : if (eSourceDT == eDestDT)
6598 : {
6599 480376 : if (eSourceDT == GDT_UInt8 || eSourceDT == GDT_Int8)
6600 : {
6601 480055 : if (nComponents == 3)
6602 : {
6603 382254 : const GByte *CPL_RESTRICT pabySrc =
6604 : static_cast<const GByte *>(pSourceBuffer);
6605 382254 : GByte *CPL_RESTRICT pabyDest0 =
6606 : static_cast<GByte *>(ppDestBuffer[0]);
6607 382254 : GByte *CPL_RESTRICT pabyDest1 =
6608 : static_cast<GByte *>(ppDestBuffer[1]);
6609 382254 : GByte *CPL_RESTRICT pabyDest2 =
6610 : static_cast<GByte *>(ppDestBuffer[2]);
6611 382254 : GDALDeinterleave3Byte(pabySrc, pabyDest0, pabyDest1, pabyDest2,
6612 : nIters);
6613 382254 : return;
6614 : }
6615 97801 : else if (nComponents == 4)
6616 : {
6617 97794 : const GByte *CPL_RESTRICT pabySrc =
6618 : static_cast<const GByte *>(pSourceBuffer);
6619 97794 : GByte *CPL_RESTRICT pabyDest0 =
6620 : static_cast<GByte *>(ppDestBuffer[0]);
6621 97794 : GByte *CPL_RESTRICT pabyDest1 =
6622 : static_cast<GByte *>(ppDestBuffer[1]);
6623 97794 : GByte *CPL_RESTRICT pabyDest2 =
6624 : static_cast<GByte *>(ppDestBuffer[2]);
6625 97794 : GByte *CPL_RESTRICT pabyDest3 =
6626 : static_cast<GByte *>(ppDestBuffer[3]);
6627 97794 : GDALDeinterleave4Byte(pabySrc, pabyDest0, pabyDest1, pabyDest2,
6628 : pabyDest3, nIters);
6629 97794 : return;
6630 7 : }
6631 : }
6632 : #if ((defined(__GNUC__) && !defined(__clang__)) || \
6633 : defined(__INTEL_CLANG_COMPILER)) && \
6634 : defined(HAVE_SSE2) && defined(HAVE_SSSE3_AT_COMPILE_TIME)
6635 642 : else if ((eSourceDT == GDT_Int16 || eSourceDT == GDT_UInt16) &&
6636 321 : CPLHaveRuntimeSSSE3())
6637 : {
6638 321 : if (nComponents == 3)
6639 : {
6640 126 : const GUInt16 *CPL_RESTRICT panSrc =
6641 : static_cast<const GUInt16 *>(pSourceBuffer);
6642 126 : GUInt16 *CPL_RESTRICT panDest0 =
6643 : static_cast<GUInt16 *>(ppDestBuffer[0]);
6644 126 : GUInt16 *CPL_RESTRICT panDest1 =
6645 : static_cast<GUInt16 *>(ppDestBuffer[1]);
6646 126 : GUInt16 *CPL_RESTRICT panDest2 =
6647 : static_cast<GUInt16 *>(ppDestBuffer[2]);
6648 126 : GDALDeinterleave3UInt16_SSSE3(panSrc, panDest0, panDest1,
6649 : panDest2, nIters);
6650 126 : return;
6651 : }
6652 : #if !defined(__INTEL_CLANG_COMPILER)
6653 : // ICC autovectorizer doesn't do a good job, at least with icx
6654 : // 2022.1.0.20220316
6655 195 : else if (nComponents == 4)
6656 : {
6657 195 : const GUInt16 *CPL_RESTRICT panSrc =
6658 : static_cast<const GUInt16 *>(pSourceBuffer);
6659 195 : GUInt16 *CPL_RESTRICT panDest0 =
6660 : static_cast<GUInt16 *>(ppDestBuffer[0]);
6661 195 : GUInt16 *CPL_RESTRICT panDest1 =
6662 : static_cast<GUInt16 *>(ppDestBuffer[1]);
6663 195 : GUInt16 *CPL_RESTRICT panDest2 =
6664 : static_cast<GUInt16 *>(ppDestBuffer[2]);
6665 195 : GUInt16 *CPL_RESTRICT panDest3 =
6666 : static_cast<GUInt16 *>(ppDestBuffer[3]);
6667 195 : GDALDeinterleave4UInt16_SSSE3(panSrc, panDest0, panDest1,
6668 : panDest2, panDest3, nIters);
6669 195 : return;
6670 : }
6671 : #endif
6672 : }
6673 : #endif
6674 : }
6675 :
6676 29 : const int nSourceDTSize = GDALGetDataTypeSizeBytes(eSourceDT);
6677 29 : const int nDestDTSize = GDALGetDataTypeSizeBytes(eDestDT);
6678 108 : for (int iComp = 0; iComp < nComponents; iComp++)
6679 : {
6680 79 : GDALCopyWords64(static_cast<const GByte *>(pSourceBuffer) +
6681 79 : iComp * nSourceDTSize,
6682 : eSourceDT, nComponents * nSourceDTSize,
6683 79 : ppDestBuffer[iComp], eDestDT, nDestDTSize, nIters);
6684 : }
6685 : }
6686 :
6687 : /************************************************************************/
6688 : /* GDALTranspose2DSingleToSingle() */
6689 : /************************************************************************/
6690 : /**
6691 : * Transpose a 2D array of non-complex values, in a efficient (cache-oblivious) way.
6692 : *
6693 : * @param pSrc Source array of height = nSrcHeight and width = nSrcWidth.
6694 : * @param pDst Destination transposed array of height = nSrcWidth and width = nSrcHeight.
6695 : * @param nSrcWidth Width of pSrc array.
6696 : * @param nSrcHeight Height of pSrc array.
6697 : */
6698 :
6699 : template <class DST, class SRC>
6700 160 : void GDALTranspose2DSingleToSingle(const SRC *CPL_RESTRICT pSrc,
6701 : DST *CPL_RESTRICT pDst, size_t nSrcWidth,
6702 : size_t nSrcHeight)
6703 : {
6704 160 : constexpr size_t blocksize = 32;
6705 345 : for (size_t i = 0; i < nSrcHeight; i += blocksize)
6706 : {
6707 185 : const size_t max_k = std::min(i + blocksize, nSrcHeight);
6708 5016 : for (size_t j = 0; j < nSrcWidth; j += blocksize)
6709 : {
6710 : // transpose the block beginning at [i,j]
6711 4831 : const size_t max_l = std::min(j + blocksize, nSrcWidth);
6712 26185 : for (size_t k = i; k < max_k; ++k)
6713 : {
6714 669282 : for (size_t l = j; l < max_l; ++l)
6715 : {
6716 647928 : GDALCopyWord(pSrc[l + k * nSrcWidth],
6717 647928 : pDst[k + l * nSrcHeight]);
6718 : }
6719 : }
6720 : }
6721 : }
6722 160 : }
6723 :
6724 : /************************************************************************/
6725 : /* GDALTranspose2DComplexToComplex() */
6726 : /************************************************************************/
6727 : /**
6728 : * Transpose a 2D array of complex values into an array of complex values,
6729 : * in a efficient (cache-oblivious) way.
6730 : *
6731 : * @param pSrc Source array of height = nSrcHeight and width = nSrcWidth.
6732 : * @param pDst Destination transposed array of height = nSrcWidth and width = nSrcHeight.
6733 : * @param nSrcWidth Width of pSrc array.
6734 : * @param nSrcHeight Height of pSrc array.
6735 : */
6736 : template <class DST, class SRC>
6737 25 : void GDALTranspose2DComplexToComplex(const SRC *CPL_RESTRICT pSrc,
6738 : DST *CPL_RESTRICT pDst, size_t nSrcWidth,
6739 : size_t nSrcHeight)
6740 : {
6741 25 : constexpr size_t blocksize = 32;
6742 50 : for (size_t i = 0; i < nSrcHeight; i += blocksize)
6743 : {
6744 25 : const size_t max_k = std::min(i + blocksize, nSrcHeight);
6745 50 : for (size_t j = 0; j < nSrcWidth; j += blocksize)
6746 : {
6747 : // transpose the block beginning at [i,j]
6748 25 : const size_t max_l = std::min(j + blocksize, nSrcWidth);
6749 75 : for (size_t k = i; k < max_k; ++k)
6750 : {
6751 200 : for (size_t l = j; l < max_l; ++l)
6752 : {
6753 150 : GDALCopyWord(pSrc[2 * (l + k * nSrcWidth) + 0],
6754 150 : pDst[2 * (k + l * nSrcHeight) + 0]);
6755 150 : GDALCopyWord(pSrc[2 * (l + k * nSrcWidth) + 1],
6756 150 : pDst[2 * (k + l * nSrcHeight) + 1]);
6757 : }
6758 : }
6759 : }
6760 : }
6761 25 : }
6762 :
6763 : /************************************************************************/
6764 : /* GDALTranspose2DComplexToSingle() */
6765 : /************************************************************************/
6766 : /**
6767 : * Transpose a 2D array of complex values into an array of non-complex values,
6768 : * in a efficient (cache-oblivious) way.
6769 : *
6770 : * @param pSrc Source array of height = nSrcHeight and width = nSrcWidth.
6771 : * @param pDst Destination transposed array of height = nSrcWidth and width = nSrcHeight.
6772 : * @param nSrcWidth Width of pSrc array.
6773 : * @param nSrcHeight Height of pSrc array.
6774 : */
6775 : template <class DST, class SRC>
6776 55 : void GDALTranspose2DComplexToSingle(const SRC *CPL_RESTRICT pSrc,
6777 : DST *CPL_RESTRICT pDst, size_t nSrcWidth,
6778 : size_t nSrcHeight)
6779 : {
6780 55 : constexpr size_t blocksize = 32;
6781 110 : for (size_t i = 0; i < nSrcHeight; i += blocksize)
6782 : {
6783 55 : const size_t max_k = std::min(i + blocksize, nSrcHeight);
6784 110 : for (size_t j = 0; j < nSrcWidth; j += blocksize)
6785 : {
6786 : // transpose the block beginning at [i,j]
6787 55 : const size_t max_l = std::min(j + blocksize, nSrcWidth);
6788 165 : for (size_t k = i; k < max_k; ++k)
6789 : {
6790 440 : for (size_t l = j; l < max_l; ++l)
6791 : {
6792 330 : GDALCopyWord(pSrc[2 * (l + k * nSrcWidth) + 0],
6793 330 : pDst[k + l * nSrcHeight]);
6794 : }
6795 : }
6796 : }
6797 : }
6798 55 : }
6799 :
6800 : /************************************************************************/
6801 : /* GDALTranspose2DSingleToComplex() */
6802 : /************************************************************************/
6803 : /**
6804 : * Transpose a 2D array of non-complex values into an array of complex values,
6805 : * in a efficient (cache-oblivious) way.
6806 : *
6807 : * @param pSrc Source array of height = nSrcHeight and width = nSrcWidth.
6808 : * @param pDst Destination transposed array of height = nSrcWidth and width = nSrcHeight.
6809 : * @param nSrcWidth Width of pSrc array.
6810 : * @param nSrcHeight Height of pSrc array.
6811 : */
6812 : template <class DST, class SRC>
6813 55 : void GDALTranspose2DSingleToComplex(const SRC *CPL_RESTRICT pSrc,
6814 : DST *CPL_RESTRICT pDst, size_t nSrcWidth,
6815 : size_t nSrcHeight)
6816 : {
6817 55 : constexpr size_t blocksize = 32;
6818 110 : for (size_t i = 0; i < nSrcHeight; i += blocksize)
6819 : {
6820 55 : const size_t max_k = std::min(i + blocksize, nSrcHeight);
6821 110 : for (size_t j = 0; j < nSrcWidth; j += blocksize)
6822 : {
6823 : // transpose the block beginning at [i,j]
6824 55 : const size_t max_l = std::min(j + blocksize, nSrcWidth);
6825 165 : for (size_t k = i; k < max_k; ++k)
6826 : {
6827 440 : for (size_t l = j; l < max_l; ++l)
6828 : {
6829 330 : GDALCopyWord(pSrc[l + k * nSrcWidth],
6830 330 : pDst[2 * (k + l * nSrcHeight) + 0]);
6831 330 : pDst[2 * (k + l * nSrcHeight) + 1] = 0;
6832 : }
6833 : }
6834 : }
6835 : }
6836 55 : }
6837 :
6838 : /************************************************************************/
6839 : /* GDALTranspose2D() */
6840 : /************************************************************************/
6841 :
6842 : template <class DST, bool DST_IS_COMPLEX>
6843 295 : static void GDALTranspose2D(const void *pSrc, GDALDataType eSrcType, DST *pDst,
6844 : size_t nSrcWidth, size_t nSrcHeight)
6845 : {
6846 : #define CALL_GDALTranspose2D_internal(SRC_TYPE) \
6847 : do \
6848 : { \
6849 : if constexpr (DST_IS_COMPLEX) \
6850 : { \
6851 : GDALTranspose2DSingleToComplex( \
6852 : static_cast<const SRC_TYPE *>(pSrc), pDst, nSrcWidth, \
6853 : nSrcHeight); \
6854 : } \
6855 : else \
6856 : { \
6857 : GDALTranspose2DSingleToSingle(static_cast<const SRC_TYPE *>(pSrc), \
6858 : pDst, nSrcWidth, nSrcHeight); \
6859 : } \
6860 : } while (0)
6861 :
6862 : #define CALL_GDALTranspose2DComplex_internal(SRC_TYPE) \
6863 : do \
6864 : { \
6865 : if constexpr (DST_IS_COMPLEX) \
6866 : { \
6867 : GDALTranspose2DComplexToComplex( \
6868 : static_cast<const SRC_TYPE *>(pSrc), pDst, nSrcWidth, \
6869 : nSrcHeight); \
6870 : } \
6871 : else \
6872 : { \
6873 : GDALTranspose2DComplexToSingle( \
6874 : static_cast<const SRC_TYPE *>(pSrc), pDst, nSrcWidth, \
6875 : nSrcHeight); \
6876 : } \
6877 : } while (0)
6878 :
6879 : // clang-format off
6880 295 : switch (eSrcType)
6881 : {
6882 16 : case GDT_UInt8: CALL_GDALTranspose2D_internal(uint8_t); break;
6883 15 : case GDT_Int8: CALL_GDALTranspose2D_internal(int8_t); break;
6884 33 : case GDT_UInt16: CALL_GDALTranspose2D_internal(uint16_t); break;
6885 20 : case GDT_Int16: CALL_GDALTranspose2D_internal(int16_t); break;
6886 24 : case GDT_UInt32: CALL_GDALTranspose2D_internal(uint32_t); break;
6887 16 : case GDT_Int32: CALL_GDALTranspose2D_internal(int32_t); break;
6888 16 : case GDT_UInt64: CALL_GDALTranspose2D_internal(uint64_t); break;
6889 16 : case GDT_Int64: CALL_GDALTranspose2D_internal(int64_t); break;
6890 16 : case GDT_Float16: CALL_GDALTranspose2D_internal(GFloat16); break;
6891 19 : case GDT_Float32: CALL_GDALTranspose2D_internal(float); break;
6892 24 : case GDT_Float64: CALL_GDALTranspose2D_internal(double); break;
6893 16 : case GDT_CInt16: CALL_GDALTranspose2DComplex_internal(int16_t); break;
6894 16 : case GDT_CInt32: CALL_GDALTranspose2DComplex_internal(int32_t); break;
6895 16 : case GDT_CFloat16: CALL_GDALTranspose2DComplex_internal(GFloat16); break;
6896 16 : case GDT_CFloat32: CALL_GDALTranspose2DComplex_internal(float); break;
6897 16 : case GDT_CFloat64: CALL_GDALTranspose2DComplex_internal(double); break;
6898 0 : case GDT_Unknown:
6899 : case GDT_TypeCount:
6900 0 : break;
6901 : }
6902 : // clang-format on
6903 :
6904 : #undef CALL_GDALTranspose2D_internal
6905 : #undef CALL_GDALTranspose2DComplex_internal
6906 295 : }
6907 :
6908 : /************************************************************************/
6909 : /* GDALInterleave2Byte() */
6910 : /************************************************************************/
6911 :
6912 : #if defined(HAVE_SSE2) && \
6913 : (!defined(__GNUC__) || defined(__INTEL_CLANG_COMPILER))
6914 :
6915 : // ICC autovectorizer doesn't do a good job at generating good SSE code,
6916 : // at least with icx 2024.0.2.20231213, but it nicely unrolls the below loop.
6917 : #if defined(__GNUC__)
6918 : __attribute__((noinline))
6919 : #endif
6920 : static void GDALInterleave2Byte(const uint8_t *CPL_RESTRICT pSrc,
6921 : uint8_t *CPL_RESTRICT pDst, size_t nIters)
6922 : {
6923 : size_t i = 0;
6924 : constexpr size_t VALS_PER_ITER = 16;
6925 : for (i = 0; i + VALS_PER_ITER <= nIters; i += VALS_PER_ITER)
6926 : {
6927 : __m128i xmm0 =
6928 : _mm_loadu_si128(reinterpret_cast<__m128i const *>(pSrc + i));
6929 : __m128i xmm1 = _mm_loadu_si128(
6930 : reinterpret_cast<__m128i const *>(pSrc + i + nIters));
6931 : _mm_storeu_si128(reinterpret_cast<__m128i *>(pDst + 2 * i),
6932 : _mm_unpacklo_epi8(xmm0, xmm1));
6933 : _mm_storeu_si128(
6934 : reinterpret_cast<__m128i *>(pDst + 2 * i + VALS_PER_ITER),
6935 : _mm_unpackhi_epi8(xmm0, xmm1));
6936 : }
6937 : #if defined(__clang__)
6938 : #pragma clang loop vectorize(disable)
6939 : #endif
6940 : for (; i < nIters; ++i)
6941 : {
6942 : pDst[2 * i + 0] = pSrc[i + 0 * nIters];
6943 : pDst[2 * i + 1] = pSrc[i + 1 * nIters];
6944 : }
6945 : }
6946 :
6947 : #else
6948 :
6949 : #if defined(__GNUC__) && !defined(__clang__)
6950 : __attribute__((optimize("tree-vectorize")))
6951 : #endif
6952 : #if defined(__GNUC__)
6953 : __attribute__((noinline))
6954 : #endif
6955 : #if defined(__clang__) && !defined(__INTEL_CLANG_COMPILER)
6956 : // clang++ -O2 -fsanitize=undefined fails to vectorize, ignore that warning
6957 : #pragma clang diagnostic push
6958 : #pragma clang diagnostic ignored "-Wpass-failed"
6959 : #endif
6960 9 : static void GDALInterleave2Byte(const uint8_t *CPL_RESTRICT pSrc,
6961 : uint8_t *CPL_RESTRICT pDst, size_t nIters)
6962 : {
6963 : #if defined(__clang__) && !defined(__INTEL_CLANG_COMPILER)
6964 : #pragma clang loop vectorize(enable)
6965 : #endif
6966 355429 : for (size_t i = 0; i < nIters; ++i)
6967 : {
6968 355420 : pDst[2 * i + 0] = pSrc[i + 0 * nIters];
6969 355420 : pDst[2 * i + 1] = pSrc[i + 1 * nIters];
6970 : }
6971 9 : }
6972 : #if defined(__clang__) && !defined(__INTEL_CLANG_COMPILER)
6973 : #pragma clang diagnostic pop
6974 : #endif
6975 :
6976 : #endif
6977 :
6978 : /************************************************************************/
6979 : /* GDALInterleave4Byte() */
6980 : /************************************************************************/
6981 :
6982 : #if defined(HAVE_SSE2) && \
6983 : (!defined(__GNUC__) || defined(__INTEL_CLANG_COMPILER))
6984 :
6985 : // ICC autovectorizer doesn't do a good job at generating good SSE code,
6986 : // at least with icx 2024.0.2.20231213, but it nicely unrolls the below loop.
6987 : #if defined(__GNUC__)
6988 : __attribute__((noinline))
6989 : #endif
6990 : static void GDALInterleave4Byte(const uint8_t *CPL_RESTRICT pSrc,
6991 : uint8_t *CPL_RESTRICT pDst, size_t nIters)
6992 : {
6993 : size_t i = 0;
6994 : constexpr size_t VALS_PER_ITER = 16;
6995 : for (i = 0; i + VALS_PER_ITER <= nIters; i += VALS_PER_ITER)
6996 : {
6997 : __m128i xmm0 = _mm_loadu_si128(
6998 : reinterpret_cast<__m128i const *>(pSrc + i + 0 * nIters));
6999 : __m128i xmm1 = _mm_loadu_si128(
7000 : reinterpret_cast<__m128i const *>(pSrc + i + 1 * nIters));
7001 : __m128i xmm2 = _mm_loadu_si128(
7002 : reinterpret_cast<__m128i const *>(pSrc + i + 2 * nIters));
7003 : __m128i xmm3 = _mm_loadu_si128(
7004 : reinterpret_cast<__m128i const *>(pSrc + i + 3 * nIters));
7005 : auto tmp0 = _mm_unpacklo_epi8(
7006 : xmm0,
7007 : xmm1); // (xmm0_0, xmm1_0, xmm0_1, xmm1_1, xmm0_2, xmm1_2, ...)
7008 : auto tmp1 = _mm_unpackhi_epi8(
7009 : xmm0,
7010 : xmm1); // (xmm0_8, xmm1_8, xmm0_9, xmm1_9, xmm0_10, xmm1_10, ...)
7011 : auto tmp2 = _mm_unpacklo_epi8(
7012 : xmm2,
7013 : xmm3); // (xmm2_0, xmm3_0, xmm2_1, xmm3_1, xmm2_2, xmm3_2, ...)
7014 : auto tmp3 = _mm_unpackhi_epi8(
7015 : xmm2,
7016 : xmm3); // (xmm2_8, xmm3_8, xmm2_9, xmm3_9, xmm2_10, xmm3_10, ...)
7017 : auto tmp2_0 = _mm_unpacklo_epi16(
7018 : tmp0,
7019 : tmp2); // (xmm0_0, xmm1_0, xmm2_0, xmm3_0, xmm0_1, xmm1_1, xmm2_1, xmm3_1, ...)
7020 : auto tmp2_1 = _mm_unpackhi_epi16(tmp0, tmp2);
7021 : auto tmp2_2 = _mm_unpacklo_epi16(tmp1, tmp3);
7022 : auto tmp2_3 = _mm_unpackhi_epi16(tmp1, tmp3);
7023 : _mm_storeu_si128(
7024 : reinterpret_cast<__m128i *>(pDst + 4 * i + 0 * VALS_PER_ITER),
7025 : tmp2_0);
7026 : _mm_storeu_si128(
7027 : reinterpret_cast<__m128i *>(pDst + 4 * i + 1 * VALS_PER_ITER),
7028 : tmp2_1);
7029 : _mm_storeu_si128(
7030 : reinterpret_cast<__m128i *>(pDst + 4 * i + 2 * VALS_PER_ITER),
7031 : tmp2_2);
7032 : _mm_storeu_si128(
7033 : reinterpret_cast<__m128i *>(pDst + 4 * i + 3 * VALS_PER_ITER),
7034 : tmp2_3);
7035 : }
7036 : #if defined(__clang__)
7037 : #pragma clang loop vectorize(disable)
7038 : #endif
7039 : for (; i < nIters; ++i)
7040 : {
7041 : pDst[4 * i + 0] = pSrc[i + 0 * nIters];
7042 : pDst[4 * i + 1] = pSrc[i + 1 * nIters];
7043 : pDst[4 * i + 2] = pSrc[i + 2 * nIters];
7044 : pDst[4 * i + 3] = pSrc[i + 3 * nIters];
7045 : }
7046 : }
7047 :
7048 : #else
7049 :
7050 : #if defined(__GNUC__) && !defined(__clang__)
7051 : __attribute__((optimize("tree-vectorize")))
7052 : #endif
7053 : #if defined(__GNUC__)
7054 : __attribute__((noinline))
7055 : #endif
7056 : #if defined(__clang__) && !defined(__INTEL_CLANG_COMPILER)
7057 : // clang++ -O2 -fsanitize=undefined fails to vectorize, ignore that warning
7058 : #pragma clang diagnostic push
7059 : #pragma clang diagnostic ignored "-Wpass-failed"
7060 : #endif
7061 30 : static void GDALInterleave4Byte(const uint8_t *CPL_RESTRICT pSrc,
7062 : uint8_t *CPL_RESTRICT pDst, size_t nIters)
7063 : {
7064 : #if defined(__clang__) && !defined(__INTEL_CLANG_COMPILER)
7065 : #pragma clang loop vectorize(enable)
7066 : #endif
7067 49620700 : for (size_t i = 0; i < nIters; ++i)
7068 : {
7069 49620600 : pDst[4 * i + 0] = pSrc[i + 0 * nIters];
7070 49620600 : pDst[4 * i + 1] = pSrc[i + 1 * nIters];
7071 49620600 : pDst[4 * i + 2] = pSrc[i + 2 * nIters];
7072 49620600 : pDst[4 * i + 3] = pSrc[i + 3 * nIters];
7073 : }
7074 30 : }
7075 : #if defined(__clang__) && !defined(__INTEL_CLANG_COMPILER)
7076 : #pragma clang diagnostic pop
7077 : #endif
7078 :
7079 : #endif
7080 :
7081 : /************************************************************************/
7082 : /* GDALTranspose2D() */
7083 : /************************************************************************/
7084 :
7085 : /**
7086 : * Transpose a 2D array in a efficient (cache-oblivious) way.
7087 : *
7088 : * @param pSrc Source array of width = nSrcWidth and height = nSrcHeight.
7089 : * @param eSrcType Data type of pSrc.
7090 : * @param pDst Destination transposed array of width = nSrcHeight and height = nSrcWidth.
7091 : * @param eDstType Data type of pDst.
7092 : * @param nSrcWidth Width of pSrc array.
7093 : * @param nSrcHeight Height of pSrc array.
7094 : * @since GDAL 3.11
7095 : */
7096 :
7097 365 : void GDALTranspose2D(const void *pSrc, GDALDataType eSrcType, void *pDst,
7098 : GDALDataType eDstType, size_t nSrcWidth, size_t nSrcHeight)
7099 : {
7100 365 : if (eSrcType == eDstType && (eSrcType == GDT_UInt8 || eSrcType == GDT_Int8))
7101 : {
7102 70 : if (nSrcHeight == 2)
7103 : {
7104 9 : GDALInterleave2Byte(static_cast<const uint8_t *>(pSrc),
7105 : static_cast<uint8_t *>(pDst), nSrcWidth);
7106 9 : return;
7107 : }
7108 61 : if (nSrcHeight == 4)
7109 : {
7110 30 : GDALInterleave4Byte(static_cast<const uint8_t *>(pSrc),
7111 : static_cast<uint8_t *>(pDst), nSrcWidth);
7112 30 : return;
7113 : }
7114 : #if (defined(HAVE_SSSE3_AT_COMPILE_TIME) && \
7115 : (defined(__x86_64) || defined(_M_X64)))
7116 31 : if (CPLHaveRuntimeSSSE3())
7117 : {
7118 31 : GDALTranspose2D_Byte_SSSE3(static_cast<const uint8_t *>(pSrc),
7119 : static_cast<uint8_t *>(pDst), nSrcWidth,
7120 : nSrcHeight);
7121 31 : return;
7122 : }
7123 : #elif defined(USE_NEON_OPTIMIZATIONS)
7124 : {
7125 : GDALTranspose2D_Byte_SSSE3(static_cast<const uint8_t *>(pSrc),
7126 : static_cast<uint8_t *>(pDst), nSrcWidth,
7127 : nSrcHeight);
7128 : return;
7129 : }
7130 : #endif
7131 : }
7132 :
7133 : #define CALL_GDALTranspose2D_internal(DST_TYPE, DST_IS_COMPLEX) \
7134 : GDALTranspose2D<DST_TYPE, DST_IS_COMPLEX>( \
7135 : pSrc, eSrcType, static_cast<DST_TYPE *>(pDst), nSrcWidth, nSrcHeight)
7136 :
7137 : // clang-format off
7138 295 : switch (eDstType)
7139 : {
7140 15 : case GDT_UInt8: CALL_GDALTranspose2D_internal(uint8_t, false); break;
7141 15 : case GDT_Int8: CALL_GDALTranspose2D_internal(int8_t, false); break;
7142 33 : case GDT_UInt16: CALL_GDALTranspose2D_internal(uint16_t, false); break;
7143 20 : case GDT_Int16: CALL_GDALTranspose2D_internal(int16_t, false); break;
7144 24 : case GDT_UInt32: CALL_GDALTranspose2D_internal(uint32_t, false); break;
7145 16 : case GDT_Int32: CALL_GDALTranspose2D_internal(int32_t, false); break;
7146 16 : case GDT_UInt64: CALL_GDALTranspose2D_internal(uint64_t, false); break;
7147 16 : case GDT_Int64: CALL_GDALTranspose2D_internal(int64_t, false); break;
7148 16 : case GDT_Float16: CALL_GDALTranspose2D_internal(GFloat16, false); break;
7149 19 : case GDT_Float32: CALL_GDALTranspose2D_internal(float, false); break;
7150 25 : case GDT_Float64: CALL_GDALTranspose2D_internal(double, false); break;
7151 16 : case GDT_CInt16: CALL_GDALTranspose2D_internal(int16_t, true); break;
7152 16 : case GDT_CInt32: CALL_GDALTranspose2D_internal(int32_t, true); break;
7153 16 : case GDT_CFloat16: CALL_GDALTranspose2D_internal(GFloat16, true); break;
7154 16 : case GDT_CFloat32: CALL_GDALTranspose2D_internal(float, true); break;
7155 16 : case GDT_CFloat64: CALL_GDALTranspose2D_internal(double, true); break;
7156 0 : case GDT_Unknown:
7157 : case GDT_TypeCount:
7158 0 : break;
7159 : }
7160 : // clang-format on
7161 :
7162 : #undef CALL_GDALTranspose2D_internal
7163 : }
7164 :
7165 : /************************************************************************/
7166 : /* ExtractBitAndConvertTo255() */
7167 : /************************************************************************/
7168 :
7169 : #if defined(__GNUC__) || defined(_MSC_VER)
7170 : // Signedness of char implementation dependent, so be explicit.
7171 : // Assumes 2-complement integer types and sign extension of right shifting
7172 : // GCC guarantees such:
7173 : // https://gcc.gnu.org/onlinedocs/gcc/Integers-implementation.html#Integers-implementation
7174 143590 : static inline GByte ExtractBitAndConvertTo255(GByte byVal, int nBit)
7175 : {
7176 143590 : return static_cast<GByte>(static_cast<signed char>(byVal << (7 - nBit)) >>
7177 143590 : 7);
7178 : }
7179 : #else
7180 : // Portable way
7181 : static inline GByte ExtractBitAndConvertTo255(GByte byVal, int nBit)
7182 : {
7183 : return (byVal & (1 << nBit)) ? 255 : 0;
7184 : }
7185 : #endif
7186 :
7187 : /************************************************************************/
7188 : /* ExpandEightPackedBitsToByteAt255() */
7189 : /************************************************************************/
7190 :
7191 17813 : static inline void ExpandEightPackedBitsToByteAt255(GByte byVal,
7192 : GByte abyOutput[8])
7193 : {
7194 17813 : abyOutput[0] = ExtractBitAndConvertTo255(byVal, 7);
7195 17813 : abyOutput[1] = ExtractBitAndConvertTo255(byVal, 6);
7196 17813 : abyOutput[2] = ExtractBitAndConvertTo255(byVal, 5);
7197 17813 : abyOutput[3] = ExtractBitAndConvertTo255(byVal, 4);
7198 17813 : abyOutput[4] = ExtractBitAndConvertTo255(byVal, 3);
7199 17813 : abyOutput[5] = ExtractBitAndConvertTo255(byVal, 2);
7200 17813 : abyOutput[6] = ExtractBitAndConvertTo255(byVal, 1);
7201 17813 : abyOutput[7] = ExtractBitAndConvertTo255(byVal, 0);
7202 17813 : }
7203 :
7204 : /************************************************************************/
7205 : /* GDALExpandPackedBitsToByteAt0Or255() */
7206 : /************************************************************************/
7207 :
7208 : /** Expand packed-bits (ordered from most-significant bit to least one)
7209 : into a byte each, where a bit at 0 is expanded to a byte at 0, and a bit
7210 : at 1 to a byte at 255.
7211 :
7212 : The function does (in a possibly more optimized way) the following:
7213 : \code{.cpp}
7214 : for (size_t i = 0; i < nInputBits; ++i )
7215 : {
7216 : pabyOutput[i] = (pabyInput[i / 8] & (1 << (7 - (i % 8)))) ? 255 : 0;
7217 : }
7218 : \endcode
7219 :
7220 : @param pabyInput Input array of (nInputBits + 7) / 8 bytes.
7221 : @param pabyOutput Output array of nInputBits bytes.
7222 : @param nInputBits Number of valid bits in pabyInput.
7223 :
7224 : @since 3.11
7225 : */
7226 :
7227 45357 : void GDALExpandPackedBitsToByteAt0Or255(const GByte *CPL_RESTRICT pabyInput,
7228 : GByte *CPL_RESTRICT pabyOutput,
7229 : size_t nInputBits)
7230 : {
7231 45357 : const size_t nInputWholeBytes = nInputBits / 8;
7232 45357 : size_t iByte = 0;
7233 :
7234 : #ifdef HAVE_SSE2
7235 : // Mask to isolate each bit
7236 45357 : const __m128i bit_mask = _mm_set_epi8(1, 2, 4, 8, 16, 32, 64, -128, 1, 2, 4,
7237 : 8, 16, 32, 64, -128);
7238 45357 : const __m128i zero = _mm_setzero_si128();
7239 45357 : const __m128i all_ones = _mm_set1_epi8(-1);
7240 : #ifdef __SSSE3__
7241 : const __m128i dispatch_two_bytes =
7242 : _mm_set_epi8(1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0);
7243 : #endif
7244 45357 : constexpr size_t SSE_REG_SIZE = sizeof(bit_mask);
7245 135866 : for (; iByte + SSE_REG_SIZE <= nInputWholeBytes; iByte += SSE_REG_SIZE)
7246 : {
7247 90509 : __m128i reg_ori = _mm_loadu_si128(
7248 90509 : reinterpret_cast<const __m128i *>(pabyInput + iByte));
7249 :
7250 90509 : constexpr int NUM_PROCESSED_BYTES_PER_REG = 2;
7251 814581 : for (size_t k = 0; k < SSE_REG_SIZE / NUM_PROCESSED_BYTES_PER_REG; ++k)
7252 : {
7253 : // Given reg_ori = (A, B, ... 14 other bytes ...),
7254 : // expand to (A, A, A, A, A, A, A, A, B, B, B, B, B, B, B, B)
7255 : #ifdef __SSSE3__
7256 : __m128i reg = _mm_shuffle_epi8(reg_ori, dispatch_two_bytes);
7257 : #else
7258 724072 : __m128i reg = _mm_unpacklo_epi8(reg_ori, reg_ori);
7259 724072 : reg = _mm_unpacklo_epi16(reg, reg);
7260 724072 : reg = _mm_unpacklo_epi32(reg, reg);
7261 : #endif
7262 :
7263 : // Test if bits of interest are set
7264 724072 : reg = _mm_and_si128(reg, bit_mask);
7265 :
7266 : // Now test if those bits are set, by comparing to zero. So the
7267 : // result will be that bytes where bits are set will be at 0, and
7268 : // ones where they are cleared will be at 0xFF. So the inverse of
7269 : // the end result we want!
7270 724072 : reg = _mm_cmpeq_epi8(reg, zero);
7271 :
7272 : // Invert the result
7273 724072 : reg = _mm_andnot_si128(reg, all_ones);
7274 :
7275 : _mm_storeu_si128(reinterpret_cast<__m128i *>(pabyOutput), reg);
7276 :
7277 724072 : pabyOutput += SSE_REG_SIZE;
7278 :
7279 : // Right-shift of 2 bytes
7280 724072 : reg_ori = _mm_bsrli_si128(reg_ori, NUM_PROCESSED_BYTES_PER_REG);
7281 : }
7282 : }
7283 :
7284 : #endif // HAVE_SSE2
7285 :
7286 63170 : for (; iByte < nInputWholeBytes; ++iByte)
7287 : {
7288 17813 : ExpandEightPackedBitsToByteAt255(pabyInput[iByte], pabyOutput);
7289 17813 : pabyOutput += 8;
7290 : }
7291 46443 : for (int iBit = 0; iBit < static_cast<int>(nInputBits % 8); ++iBit)
7292 : {
7293 1086 : *pabyOutput = ExtractBitAndConvertTo255(pabyInput[iByte], 7 - iBit);
7294 1086 : ++pabyOutput;
7295 : }
7296 45357 : }
7297 :
7298 : /************************************************************************/
7299 : /* ExpandEightPackedBitsToByteAt1() */
7300 : /************************************************************************/
7301 :
7302 136113 : static inline void ExpandEightPackedBitsToByteAt1(GByte byVal,
7303 : GByte abyOutput[8])
7304 : {
7305 136113 : abyOutput[0] = (byVal >> 7) & 0x1;
7306 136113 : abyOutput[1] = (byVal >> 6) & 0x1;
7307 136113 : abyOutput[2] = (byVal >> 5) & 0x1;
7308 136113 : abyOutput[3] = (byVal >> 4) & 0x1;
7309 136113 : abyOutput[4] = (byVal >> 3) & 0x1;
7310 136113 : abyOutput[5] = (byVal >> 2) & 0x1;
7311 136113 : abyOutput[6] = (byVal >> 1) & 0x1;
7312 136113 : abyOutput[7] = (byVal >> 0) & 0x1;
7313 136113 : }
7314 :
7315 : /************************************************************************/
7316 : /* GDALExpandPackedBitsToByteAt0Or1() */
7317 : /************************************************************************/
7318 :
7319 : /** Expand packed-bits (ordered from most-significant bit to least one)
7320 : into a byte each, where a bit at 0 is expanded to a byte at 0, and a bit
7321 : at 1 to a byte at 1.
7322 :
7323 : The function does (in a possibly more optimized way) the following:
7324 : \code{.cpp}
7325 : for (size_t i = 0; i < nInputBits; ++i )
7326 : {
7327 : pabyOutput[i] = (pabyInput[i / 8] & (1 << (7 - (i % 8)))) ? 1 : 0;
7328 : }
7329 : \endcode
7330 :
7331 : @param pabyInput Input array of (nInputBits + 7) / 8 bytes.
7332 : @param pabyOutput Output array of nInputBits bytes.
7333 : @param nInputBits Number of valid bits in pabyInput.
7334 :
7335 : @since 3.11
7336 : */
7337 :
7338 7033 : void GDALExpandPackedBitsToByteAt0Or1(const GByte *CPL_RESTRICT pabyInput,
7339 : GByte *CPL_RESTRICT pabyOutput,
7340 : size_t nInputBits)
7341 : {
7342 7033 : const size_t nInputWholeBytes = nInputBits / 8;
7343 7033 : size_t iByte = 0;
7344 143146 : for (; iByte < nInputWholeBytes; ++iByte)
7345 : {
7346 136113 : ExpandEightPackedBitsToByteAt1(pabyInput[iByte], pabyOutput);
7347 136113 : pabyOutput += 8;
7348 : }
7349 18886 : for (int iBit = 0; iBit < static_cast<int>(nInputBits % 8); ++iBit)
7350 : {
7351 11853 : *pabyOutput = (pabyInput[iByte] >> (7 - iBit)) & 0x1;
7352 11853 : ++pabyOutput;
7353 : }
7354 7033 : }
|
