Line data Source code
1 : /*
2 : * Copyright (c) 2002-2012, California Institute of Technology.
3 : * All rights reserved. Based on Government Sponsored Research under contracts
4 : * NAS7-1407 and/or NAS7-03001.
5 : *
6 : * Redistribution and use in source and binary forms, with or without
7 : * modification, are permitted provided that the following conditions are met:
8 : * 1. Redistributions of source code must retain the above copyright notice,
9 : * this list of conditions and the following disclaimer.
10 : * 2. Redistributions in binary form must reproduce the above copyright
11 : * notice, this list of conditions and the following disclaimer in the
12 : * documentation and/or other materials provided with the distribution.
13 : * 3. Neither the name of the California Institute of Technology (Caltech),
14 : * its operating division the Jet Propulsion Laboratory (JPL), the National
15 : * Aeronautics and Space Administration (NASA), nor the names of its
16 : * contributors may be used to endorse or promote products derived from this
17 : * software without specific prior written permission.
18 : *
19 : * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20 : * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 : * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 : * ARE DISCLAIMED. IN NO EVENT SHALL THE CALIFORNIA INSTITUTE OF TECHNOLOGY BE
23 : * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 : * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 : * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 : * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 : * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 : * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 : * POSSIBILITY OF SUCH DAMAGE.
30 : *
31 : * Portions copyright 2014-2021 Esri
32 : *
33 : * Licensed under the Apache License, Version 2.0 (the "License");
34 : * you may not use this file except in compliance with the License.
35 : * You may obtain a copy of the License at
36 : *
37 : * http://www.apache.org/licenses/LICENSE-2.0
38 : *
39 : * Unless required by applicable law or agreed to in writing, software
40 : * distributed under the License is distributed on an "AS IS" BASIS,
41 : * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
42 : * See the License for the specific language governing permissions and
43 : * limitations under the License.
44 : */
45 :
46 : /******************************************************************************
47 : *
48 : * Project: Meta Raster File Format Driver Implementation, Dataset
49 : * Purpose: Implementation of GDAL dataset
50 : *
51 : * Author: Lucian Plesea, Lucian.Plesea jpl.nasa.gov, lplesea esri.com
52 : *
53 : ******************************************************************************
54 : *
55 : * The MRF dataset and the band are closely tied together, they should be
56 : * considered a single class, or a class (dataset) with extensions (bands).
57 : *
58 : *
59 : ****************************************************************************/
60 :
61 : #include "marfa.h"
62 : #include "mrfdrivercore.h"
63 : #include "cpl_multiproc.h" /* for CPLSleep() */
64 : #include "gdal_priv.h"
65 : #include <assert.h>
66 :
67 : #include <algorithm>
68 : #include <limits>
69 : #include <vector>
70 : #if defined(ZSTD_SUPPORT)
71 : #include <zstd.h>
72 : #endif
73 : using std::string;
74 : using std::vector;
75 :
76 : NAMESPACE_MRF_START
77 :
78 : // Initialize as invalid
79 372 : MRFDataset::MRFDataset()
80 : : zslice(0), idxSize(0), clonedSource(FALSE), nocopy(FALSE),
81 : bypass_cache(
82 372 : CPLTestBool(CPLGetConfigOption("MRF_BYPASSCACHING", "FALSE"))),
83 : mp_safe(FALSE), hasVersions(FALSE), verCount(0),
84 : bCrystalized(TRUE), // Assume not in create mode
85 : spacing(0), no_errors(0), missing(0), poSrcDS(nullptr), level(-1),
86 : cds(nullptr), scale(0.0), pbuffer(nullptr), pbsize(0), tile(ILSize()),
87 : bdirty(0), bGeoTransformValid(TRUE), poColorTable(nullptr), Quality(0),
88 744 : pzscctx(nullptr), pzsdctx(nullptr), read_timer(), write_timer(0)
89 : {
90 372 : m_oSRS.SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER);
91 372 : ifp.FP = dfp.FP = nullptr;
92 372 : dfp.acc = GF_Read;
93 372 : ifp.acc = GF_Read;
94 372 : }
95 :
96 181 : bool MRFDataset::SetPBuffer(unsigned int sz)
97 : {
98 181 : if (sz == 0)
99 : {
100 0 : CPLFree(pbuffer);
101 0 : pbuffer = nullptr;
102 : }
103 181 : void *pbufferNew = VSIRealloc(pbuffer, sz);
104 181 : if (pbufferNew == nullptr)
105 : {
106 0 : CPLError(CE_Failure, CPLE_OutOfMemory, "Cannot allocate %u bytes", sz);
107 0 : return false;
108 : }
109 181 : pbuffer = pbufferNew;
110 181 : pbsize = sz;
111 181 : return true;
112 : }
113 :
114 256 : static GDALColorEntry GetXMLColorEntry(CPLXMLNode *p)
115 : {
116 : GDALColorEntry ce;
117 256 : ce.c1 = static_cast<short>(getXMLNum(p, "c1", 0));
118 256 : ce.c2 = static_cast<short>(getXMLNum(p, "c2", 0));
119 256 : ce.c3 = static_cast<short>(getXMLNum(p, "c3", 0));
120 256 : ce.c4 = static_cast<short>(getXMLNum(p, "c4", 255));
121 256 : return ce;
122 : }
123 :
124 : //
125 : // Called by dataset destructor or at GDAL termination, to avoid
126 : // closing datasets whose drivers have already been unloaded
127 : //
128 372 : int MRFDataset::CloseDependentDatasets()
129 : {
130 372 : int bHasDroppedRef = GDALPamDataset::CloseDependentDatasets();
131 :
132 372 : if (poSrcDS)
133 : {
134 3 : bHasDroppedRef = TRUE;
135 3 : GDALClose(GDALDataset::ToHandle(poSrcDS));
136 3 : poSrcDS = nullptr;
137 : }
138 :
139 372 : if (cds)
140 : {
141 2 : bHasDroppedRef = TRUE;
142 2 : GDALClose(GDALDataset::ToHandle(cds));
143 2 : cds = nullptr;
144 : }
145 :
146 372 : return bHasDroppedRef;
147 : }
148 :
149 744 : MRFDataset::~MRFDataset()
150 : { // Make sure everything gets written
151 372 : if (eAccess != GA_ReadOnly && !bCrystalized)
152 50 : if (!MRFDataset::Crystalize())
153 : {
154 : // Can't return error code from a destructor, just emit the error
155 10 : CPLError(CE_Failure, CPLE_FileIO, "Error creating files");
156 : }
157 :
158 372 : MRFDataset::FlushCache(true);
159 372 : MRFDataset::CloseDependentDatasets();
160 :
161 372 : if (ifp.FP)
162 263 : VSIFCloseL(ifp.FP);
163 372 : if (dfp.FP)
164 267 : VSIFCloseL(dfp.FP);
165 :
166 372 : delete poColorTable;
167 :
168 : // CPLFree ignores being called with NULL
169 372 : CPLFree(pbuffer);
170 372 : pbsize = 0;
171 : #if defined(ZSTD_SUPPORT)
172 372 : ZSTD_freeCCtx(static_cast<ZSTD_CCtx *>(pzscctx));
173 372 : ZSTD_freeDCtx(static_cast<ZSTD_DCtx *>(pzsdctx));
174 : #endif
175 : // total time spend doing compression and decompression
176 372 : if (0 != write_timer.count())
177 112 : CPLDebug("MRF_Timing", "Compression took %fms",
178 112 : 1e-6 * write_timer.count());
179 :
180 372 : if (0 != read_timer.count())
181 143 : CPLDebug("MRF_Timing", "Decompression took %fms",
182 143 : 1e-6 * read_timer.count());
183 744 : }
184 :
185 : /*
186 : *\brief Format specific RasterIO, may be bypassed by BlockBasedRasterIO by
187 : *setting GDAL_FORCE_CACHING to Yes, in which case the band ReadBlock and
188 : *WriteBLock are called directly
189 : */
190 140 : CPLErr MRFDataset::IRasterIO(GDALRWFlag eRWFlag, int nXOff, int nYOff,
191 : int nXSize, int nYSize, void *pData, int nBufXSize,
192 : int nBufYSize, GDALDataType eBufType,
193 : int nBandCount, BANDMAP_TYPE panBandMap,
194 : GSpacing nPixelSpace, GSpacing nLineSpace,
195 : GSpacing nBandSpace,
196 : GDALRasterIOExtraArg *psExtraArgs)
197 : {
198 140 : CPLDebug("MRF_IO",
199 : "IRasterIO %s, %d, %d, %d, %d, bufsz %d,%d,%d strides P %d, L %d, "
200 : "B %d \n",
201 : eRWFlag == GF_Write ? "Write" : "Read", nXOff, nYOff, nXSize,
202 : nYSize, nBufXSize, nBufYSize, nBandCount,
203 : static_cast<int>(nPixelSpace), static_cast<int>(nLineSpace),
204 : static_cast<int>(nBandSpace));
205 :
206 140 : if (eRWFlag == GF_Write && !bCrystalized && !Crystalize())
207 : {
208 0 : CPLError(CE_Failure, CPLE_FileIO, "MRF: Error creating files");
209 0 : return CE_Failure;
210 : }
211 :
212 : //
213 : // Call the parent implementation, which splits it into bands and calls
214 : // their IRasterIO
215 : //
216 140 : return GDALPamDataset::IRasterIO(eRWFlag, nXOff, nYOff, nXSize, nYSize,
217 : pData, nBufXSize, nBufYSize, eBufType,
218 : nBandCount, panBandMap, nPixelSpace,
219 140 : nLineSpace, nBandSpace, psExtraArgs);
220 : }
221 :
222 : /**
223 : *\brief Build some overviews
224 : *
225 : * if nOverviews is 0, erase the overviews (reduce to base image only)
226 : */
227 :
228 31 : CPLErr MRFDataset::IBuildOverviews(const char *pszResampling, int nOverviews,
229 : const int *panOverviewList, int nBandsIn,
230 : const int *panBandList,
231 : GDALProgressFunc pfnProgress,
232 : void *pProgressData,
233 : CSLConstList papszOptions)
234 :
235 : {
236 31 : if (pfnProgress == nullptr)
237 0 : pfnProgress = GDALDummyProgress;
238 :
239 31 : CPLErr eErr = CE_None;
240 31 : CPLDebug("MRF_OVERLAY", "IBuildOverviews %d, bands %d\n", nOverviews,
241 : nBandsIn);
242 :
243 31 : if (nBands != nBandsIn)
244 : {
245 0 : CPLError(CE_Failure, CPLE_NotSupported, "nBands = %d not supported",
246 : nBandsIn);
247 0 : return CE_Failure;
248 : }
249 :
250 : // If we don't have write access, then create external overviews
251 31 : if (GetAccess() != GA_Update)
252 : {
253 1 : CPLDebug("MRF", "File open read-only, creating overviews externally.");
254 1 : return GDALDataset::IBuildOverviews(
255 : pszResampling, nOverviews, panOverviewList, nBands, panBandList,
256 1 : pfnProgress, pProgressData, papszOptions);
257 : }
258 :
259 30 : if (nOverviews == 0)
260 : {
261 : // If there are none, nothing to do
262 0 : if (GetRasterBand(1)->GetOverviewCount() == 0)
263 0 : return CE_None;
264 :
265 0 : auto *b = static_cast<MRFRasterBand *>(GetRasterBand(1));
266 : // If the first band internal overviews don't exist, they are external
267 0 : if (b->overviews.empty())
268 0 : return GDALDataset::IBuildOverviews(
269 : pszResampling, nOverviews, panOverviewList, nBands, panBandList,
270 0 : pfnProgress, pProgressData, papszOptions);
271 :
272 : // We should clean overviews, but this is not allowed in an MRF
273 0 : CPLError(CE_Warning, CPLE_NotSupported,
274 : "MRF: Internal overviews cannot be removed, "
275 : "but they can be rebuilt");
276 0 : return CE_None;
277 : }
278 :
279 30 : std::vector<int> panOverviewListNew(nOverviews);
280 61 : for (int i = 0; i < nOverviews; i++)
281 31 : panOverviewListNew[i] = panOverviewList[i];
282 : try
283 : { // Throw an error code, to make sure memory gets freed properly
284 : // Modify the metadata file if it doesn't already have the Rset model
285 : // set
286 30 : if (0.0 == scale)
287 : {
288 30 : CPLXMLNode *config = ReadConfig();
289 : try
290 : {
291 : const char *model =
292 30 : CPLGetXMLValue(config, "Rsets.model", "uniform");
293 30 : if (!EQUAL(model, "uniform"))
294 : {
295 0 : CPLError(CE_Failure, CPLE_AppDefined,
296 : "MRF:IBuildOverviews, Overviews not implemented "
297 : "for model %s",
298 : model);
299 0 : throw CE_Failure;
300 : }
301 :
302 : // The scale value is the same as first overview
303 30 : scale =
304 30 : strtod(CPLGetXMLValue(
305 : config, "Rsets.scale",
306 60 : CPLOPrintf("%d", panOverviewList[0]).c_str()),
307 : nullptr);
308 30 : if (scale == 0.0)
309 : {
310 0 : CPLError(CE_Failure, CPLE_IllegalArg,
311 : "Invalid Rsets.scale value");
312 0 : throw CE_Failure;
313 : }
314 :
315 30 : if (static_cast<int>(scale) != 2 &&
316 0 : (EQUALN("Avg", pszResampling, 3) ||
317 0 : EQUALN("Nnb", pszResampling, 3)))
318 : {
319 0 : CPLError(CE_Failure, CPLE_IllegalArg,
320 : "MRF internal resampling requires a scale factor "
321 : "of two");
322 0 : throw CE_Failure;
323 : }
324 :
325 : // Initialize the empty overlays, all of them for a given scale
326 : // They could already exist, in which case they are not erased
327 30 : idxSize = AddOverviews(int(scale));
328 :
329 : // If we don't have overviews, don't try to generate them
330 30 : if (GetRasterBand(1)->GetOverviewCount() == 0)
331 0 : throw CE_None;
332 :
333 30 : if (!CheckFileSize(current.idxfname, idxSize, GA_Update))
334 : {
335 0 : CPLError(CE_Failure, CPLE_AppDefined,
336 : "MRF: Can't extend index file");
337 0 : throw CE_Failure;
338 : }
339 :
340 : // Set the uniform node, in case it was not set before, and
341 : // save the new configuration
342 30 : CPLSetXMLValue(config, "Rsets.#model", "uniform");
343 30 : CPLSetXMLValue(config, "Rsets.#scale", PrintDouble(scale));
344 :
345 30 : if (!WriteConfig(config))
346 : {
347 0 : CPLError(CE_Failure, CPLE_AppDefined,
348 : "MRF: Can't rewrite the metadata file");
349 0 : throw CE_Failure;
350 : }
351 30 : CPLDestroyXMLNode(config);
352 30 : config = nullptr;
353 : }
354 0 : catch (const CPLErr &)
355 : {
356 0 : CPLDestroyXMLNode(config);
357 0 : throw; // Rethrow
358 : }
359 :
360 : // To avoid issues with blacks overviews, generate all of them
361 : // if the user asked for a couple of overviews in the correct
362 : // sequence and starting with the lowest one
363 90 : if (!EQUAL(pszResampling, "NONE") &&
364 32 : nOverviews != GetRasterBand(1)->GetOverviewCount() &&
365 2 : CPLTestBool(
366 : CPLGetConfigOption("MRF_ALL_OVERVIEW_LEVELS", "YES")))
367 : {
368 2 : bool bIncreasingPowers =
369 2 : (panOverviewList[0] == static_cast<int>(scale));
370 3 : for (int i = 1; i < nOverviews; i++)
371 1 : bIncreasingPowers =
372 2 : bIncreasingPowers &&
373 1 : (panOverviewList[i] ==
374 1 : static_cast<int>(scale * panOverviewList[i - 1]));
375 :
376 2 : int ovrcount = GetRasterBand(1)->GetOverviewCount();
377 2 : if (bIncreasingPowers && nOverviews != ovrcount)
378 : {
379 2 : CPLDebug("MRF",
380 : "Generating %d levels instead of the %d requested",
381 : ovrcount, nOverviews);
382 2 : nOverviews = ovrcount;
383 2 : panOverviewListNew.resize(ovrcount);
384 7 : for (int i = 0; i < ovrcount; i++)
385 5 : panOverviewListNew[i] = int(std::pow(scale, i + 1));
386 : }
387 : }
388 : }
389 :
390 30 : if (static_cast<int>(scale) != 2 && (EQUALN("Avg", pszResampling, 3) ||
391 0 : EQUALN("Nnb", pszResampling, 3)))
392 : {
393 0 : CPLError(CE_Failure, CPLE_IllegalArg,
394 : "MRF internal resampling requires a scale factor of two");
395 0 : throw CE_Failure;
396 : }
397 :
398 : // First pass, count the pixels to be processed, mark invalid levels
399 : // Smallest positive value to avoid Coverity Scan complaining about
400 : // division by zero
401 30 : double dfTotalPixels = std::numeric_limits<double>::min();
402 30 : double dfPixels = double(nRasterXSize) * nRasterYSize;
403 63 : for (int i = 0; i < nOverviews; i++)
404 : {
405 : // Verify that scales are reasonable, val/scale has to be an integer
406 33 : if (!IsPower(panOverviewListNew[i], scale))
407 : {
408 0 : CPLError(CE_Warning, CPLE_AppDefined,
409 : "MRF:IBuildOverviews, skipping overview factor %d,"
410 : " it is not a power of %f",
411 0 : panOverviewListNew[i], scale);
412 0 : panOverviewListNew[i] = -1;
413 0 : continue;
414 : };
415 :
416 33 : int srclevel = int(logbase(panOverviewListNew[i], scale) - 0.5);
417 33 : MRFRasterBand *b = static_cast<MRFRasterBand *>(GetRasterBand(1));
418 :
419 : // Warn for requests for invalid levels
420 33 : if (srclevel >= b->GetOverviewCount())
421 : {
422 0 : CPLError(CE_Warning, CPLE_AppDefined,
423 : "MRF:IBuildOverviews, overview factor %d is not valid "
424 : "for this dataset",
425 0 : panOverviewListNew[i]);
426 0 : panOverviewListNew[i] = -1;
427 0 : continue;
428 : }
429 33 : dfTotalPixels += dfPixels * std::pow(scale, -srclevel);
430 : }
431 :
432 30 : dfPixels = 0;
433 63 : for (int i = 0; i < nOverviews; i++)
434 : {
435 : // Skip the invalid levels
436 33 : if (panOverviewListNew[i] < 0)
437 0 : continue;
438 :
439 33 : int srclevel = int(logbase(panOverviewListNew[i], scale) - 0.5);
440 : auto dfLevelPixels =
441 33 : std::pow(scale, -srclevel) * nRasterXSize * nRasterYSize;
442 : // Use "avg" flag to trigger the internal average sampling
443 33 : if (EQUALN("Avg", pszResampling, 3) ||
444 19 : EQUALN("Nnb", pszResampling, 3))
445 : {
446 26 : int sampling = EQUALN("Avg", pszResampling, 3) ? SAMPLING_Avg
447 : : SAMPLING_Near;
448 26 : auto b = static_cast<MRFRasterBand *>(GetRasterBand(1));
449 : // Internal, using PatchOverview
450 26 : if (srclevel > 0)
451 : b = static_cast<MRFRasterBand *>(
452 3 : b->GetOverview(srclevel - 1));
453 :
454 : eErr =
455 26 : PatchOverview(0, 0, b->nBlocksPerRow, b->nBlocksPerColumn,
456 : srclevel, 0, sampling);
457 26 : if (eErr == CE_Failure)
458 26 : throw eErr;
459 : }
460 : else
461 : {
462 : // Use the GDAL method
463 14 : std::vector<GDALRasterBand *> apoSrcBandList(nBands);
464 : std::vector<std::vector<GDALRasterBand *>> aapoOverviewBandList(
465 14 : nBands);
466 14 : for (int iBand = 0; iBand < nBands; iBand++)
467 : {
468 : // This is the base level
469 7 : apoSrcBandList[iBand] = GetRasterBand(panBandList[iBand]);
470 : // Set up the destination
471 14 : aapoOverviewBandList[iBand] = std::vector<GDALRasterBand *>{
472 14 : apoSrcBandList[iBand]->GetOverview(srclevel)};
473 : // Use the previous level as the source
474 7 : if (srclevel > 0)
475 0 : apoSrcBandList[iBand] =
476 0 : apoSrcBandList[iBand]->GetOverview(srclevel - 1);
477 : }
478 :
479 14 : auto pScaledProgress = GDALCreateScaledProgress(
480 : dfPixels / dfTotalPixels,
481 7 : (dfPixels + dfLevelPixels) / dfTotalPixels, pfnProgress,
482 : pProgressData);
483 7 : eErr = GDALRegenerateOverviewsMultiBand(
484 : apoSrcBandList, aapoOverviewBandList, pszResampling,
485 : GDALScaledProgress, pScaledProgress, papszOptions);
486 7 : GDALDestroyScaledProgress(pScaledProgress);
487 : }
488 33 : dfPixels += dfLevelPixels;
489 33 : pfnProgress(dfPixels / dfTotalPixels, "", pProgressData);
490 33 : if (eErr == CE_Failure)
491 0 : throw eErr;
492 : }
493 :
494 30 : pfnProgress(1.0, "", pProgressData);
495 : }
496 0 : catch (const CPLErr &e)
497 : {
498 0 : eErr = e;
499 : }
500 :
501 30 : return eErr;
502 : }
503 :
504 : /*
505 : *\brief blank separated list to vector of doubles
506 : */
507 34 : static void list2vec(std::vector<double> &v, const char *pszList)
508 : {
509 34 : if ((pszList == nullptr) || (pszList[0] == 0))
510 0 : return;
511 34 : char **papszTokens = CSLTokenizeString2(
512 : pszList, " \t\n\r", CSLT_STRIPLEADSPACES | CSLT_STRIPENDSPACES);
513 34 : v.clear();
514 69 : for (int i = 0; i < CSLCount(papszTokens); i++)
515 35 : v.push_back(CPLStrtod(papszTokens[i], nullptr));
516 34 : CSLDestroy(papszTokens);
517 : }
518 :
519 20 : void MRFDataset::SetNoDataValue(const char *pszVal)
520 : {
521 20 : list2vec(vNoData, pszVal);
522 20 : }
523 :
524 7 : void MRFDataset::SetMinValue(const char *pszVal)
525 : {
526 7 : list2vec(vMin, pszVal);
527 7 : }
528 :
529 7 : void MRFDataset::SetMaxValue(const char *pszVal)
530 : {
531 7 : list2vec(vMax, pszVal);
532 7 : }
533 :
534 : /**
535 : *
536 : *\brief Read the XML config tree, from file
537 : * Caller is responsible for freeing the memory
538 : *
539 : * @return NULL on failure, or the document tree on success.
540 : *
541 : */
542 30 : CPLXMLNode *MRFDataset::ReadConfig() const
543 : {
544 30 : if (fname[0] == '<')
545 0 : return CPLParseXMLString(fname);
546 30 : return CPLParseXMLFile(fname);
547 : }
548 :
549 : /**
550 : *\brief Write the XML config tree
551 : * Caller is responsible for correctness of data
552 : * and for freeing the memory
553 : *
554 : * @param config The document tree to write
555 : * @return TRUE on success, FALSE otherwise
556 : */
557 202 : int MRFDataset::WriteConfig(CPLXMLNode *config)
558 : {
559 202 : if (fname[0] == '<')
560 0 : return FALSE;
561 202 : return CPLSerializeXMLTreeToFile(config, fname);
562 : }
563 :
564 : static void
565 5 : stringSplit(vector<string> &theStringVector, // Altered/returned value
566 : const string &theString, size_t start = 0,
567 : const char theDelimiter = ' ')
568 : {
569 : while (true)
570 : {
571 5 : size_t end = theString.find(theDelimiter, start);
572 5 : if (string::npos == end)
573 : {
574 5 : theStringVector.push_back(theString.substr(start));
575 5 : return;
576 : }
577 0 : theStringVector.push_back(theString.substr(start, end - start));
578 0 : start = end + 1;
579 0 : }
580 : }
581 :
582 : // Returns the number following the prefix if it exists in one of the vector
583 : // strings Otherwise it returns the default
584 15 : static int getnum(const vector<string> &theStringVector, const char prefix,
585 : int def)
586 : {
587 25 : for (unsigned int i = 0; i < theStringVector.size(); i++)
588 15 : if (theStringVector[i][0] == prefix)
589 5 : return atoi(theStringVector[i].c_str() + 1);
590 10 : return def;
591 : }
592 :
593 : /**
594 : *\brief Open a MRF file
595 : *
596 : */
597 205 : GDALDataset *MRFDataset::Open(GDALOpenInfo *poOpenInfo)
598 : {
599 205 : if (!MRFDriverIdentify(poOpenInfo))
600 0 : return nullptr;
601 :
602 205 : CPLXMLNode *config = nullptr;
603 205 : CPLErr ret = CE_None;
604 205 : const char *pszFileName = poOpenInfo->pszFilename;
605 :
606 205 : int level = -1; // All levels
607 205 : int version = 0; // Current
608 205 : int zslice = 0;
609 410 : string fn; // Used to parse and adjust the file name
610 410 : string insidefn; // inside tar file name
611 :
612 : // Different ways to open an MRF
613 205 : if (poOpenInfo->nHeaderBytes >= 10)
614 : {
615 200 : const char *pszHeader =
616 : reinterpret_cast<char *>(poOpenInfo->pabyHeader);
617 200 : fn.assign(pszHeader, poOpenInfo->nHeaderBytes);
618 200 : if (STARTS_WITH(pszHeader, "<MRF_META>")) // Regular file name
619 192 : config = CPLParseXMLFile(pszFileName);
620 8 : else if (poOpenInfo->eAccess == GA_ReadOnly && fn.size() > 600 &&
621 8 : (fn[262] == 0 || fn[262] == 32) &&
622 1 : STARTS_WITH(fn.c_str() + 257, "ustar") &&
623 17 : strlen(CPLGetPathSafe(fn.c_str()).c_str()) == 0 &&
624 1 : STARTS_WITH(fn.c_str() + 512, "<MRF_META>"))
625 : { // An MRF inside a tar
626 1 : insidefn = string("/vsitar/") + pszFileName + "/" + pszHeader;
627 1 : config = CPLParseXMLFile(insidefn.c_str());
628 : }
629 : #if defined(LERC)
630 : else
631 7 : config = LERC_Band::GetMRFConfig(poOpenInfo);
632 : #endif
633 : }
634 : else
635 : {
636 5 : if (EQUALN(pszFileName, "<MRF_META>", 10)) // Content as file name
637 0 : config = CPLParseXMLString(pszFileName);
638 : else
639 : { // Try Ornate file name
640 5 : fn = pszFileName;
641 5 : size_t pos = fn.find(":MRF:");
642 5 : if (string::npos != pos)
643 : { // Tokenize and pick known options
644 5 : vector<string> tokens;
645 5 : stringSplit(tokens, fn, pos + 5, ':');
646 5 : level = getnum(tokens, 'L', -1);
647 5 : version = getnum(tokens, 'V', 0);
648 5 : zslice = getnum(tokens, 'Z', 0);
649 5 : fn.resize(pos); // Cut the ornamentations
650 5 : pszFileName = fn.c_str();
651 5 : config = CPLParseXMLFile(pszFileName);
652 : }
653 : }
654 : }
655 :
656 205 : if (!config)
657 0 : return nullptr;
658 :
659 205 : MRFDataset *ds = new MRFDataset();
660 205 : ds->fname = pszFileName;
661 205 : if (!insidefn.empty())
662 : {
663 1 : ds->publicname = pszFileName;
664 1 : ds->fname = insidefn;
665 : }
666 205 : ds->eAccess = poOpenInfo->eAccess;
667 205 : ds->level = level;
668 205 : ds->zslice = zslice;
669 :
670 : // OpenOptions can override file name arguments
671 205 : ds->ProcessOpenOptions(poOpenInfo->papszOpenOptions);
672 :
673 205 : if (level == -1)
674 203 : ret = ds->Initialize(config);
675 : else
676 : {
677 : // Open the whole dataset, then pick one level
678 2 : ds->cds = new MRFDataset();
679 2 : ds->cds->fname = ds->fname;
680 2 : ds->cds->eAccess = ds->eAccess;
681 2 : ds->zslice = zslice;
682 2 : ret = ds->cds->Initialize(config);
683 2 : if (ret == CE_None)
684 2 : ret = ds->LevelInit(level);
685 : }
686 205 : CPLDestroyXMLNode(config);
687 :
688 205 : if (ret != CE_None)
689 : {
690 23 : delete ds;
691 23 : return nullptr;
692 : }
693 :
694 : // Open a single version
695 182 : if (version != 0)
696 2 : ret = ds->SetVersion(version);
697 :
698 182 : if (ret != CE_None)
699 : {
700 1 : delete ds;
701 1 : return nullptr;
702 : }
703 :
704 : // Tell PAM what our real file name is, to help it find the aux.xml
705 181 : ds->SetPhysicalFilename(ds->fname);
706 : // Don't mess with metadata after this, otherwise PAM will re-write the
707 : // aux.xml
708 181 : ds->TryLoadXML();
709 :
710 : /* -------------------------------------------------------------------- */
711 : /* Open external overviews. */
712 : /* -------------------------------------------------------------------- */
713 181 : ds->oOvManager.Initialize(ds, ds->fname);
714 :
715 181 : return ds;
716 : }
717 :
718 : // Adjust the band images with the right offset, then adjust the sizes
719 2 : CPLErr MRFDataset::SetVersion(int version)
720 : {
721 2 : if (!hasVersions || version > verCount)
722 : {
723 1 : CPLError(CE_Failure, CPLE_AppDefined,
724 : "GDAL MRF: Version number error!");
725 1 : return CE_Failure;
726 : }
727 : // Size of one version index
728 2 : for (int bcount = 1; bcount <= nBands; bcount++)
729 : {
730 : MRFRasterBand *srcband =
731 1 : cpl::down_cast<MRFRasterBand *>(GetRasterBand(bcount));
732 1 : srcband->img.idxoffset += idxSize * verCount;
733 1 : for (int l = 0; l < srcband->GetOverviewCount(); l++)
734 : {
735 : MRFRasterBand *band =
736 0 : cpl::down_cast<MRFRasterBand *>(srcband->GetOverview(l));
737 0 : if (band != nullptr)
738 0 : band->img.idxoffset += idxSize * verCount;
739 : }
740 : }
741 1 : hasVersions = 0;
742 1 : return CE_None;
743 : }
744 :
745 2 : CPLErr MRFDataset::LevelInit(const int l)
746 : {
747 : // Test that this level does exist
748 2 : if (l < 0 || l >= cds->GetRasterBand(1)->GetOverviewCount())
749 : {
750 1 : CPLError(CE_Failure, CPLE_AppDefined,
751 : "GDAL MRF: Overview not present!");
752 1 : return CE_Failure;
753 : }
754 :
755 : MRFRasterBand *srcband =
756 1 : cpl::down_cast<MRFRasterBand *>(cds->GetRasterBand(1)->GetOverview(l));
757 1 : if (!srcband)
758 0 : return CE_Failure;
759 :
760 : // Copy the sizes from this level
761 1 : full = srcband->img;
762 1 : current = srcband->img;
763 1 : current.size.c = cds->current.size.c;
764 1 : scale = cds->scale;
765 1 : const auto poSRS = cds->GetSpatialRef();
766 1 : if (poSRS)
767 1 : m_oSRS = *poSRS;
768 :
769 1 : SetMetadataItem("INTERLEAVE", OrderName(current.order), "IMAGE_STRUCTURE");
770 1 : SetMetadataItem("COMPRESSION", CompName(current.comp), "IMAGE_STRUCTURE");
771 :
772 1 : bGeoTransformValid = (CE_None == cds->GetGeoTransform(m_gt));
773 4 : for (int i = 0; i < l + 1; i++)
774 : {
775 3 : m_gt.xscale *= scale;
776 3 : m_gt.yscale *= scale;
777 : }
778 :
779 1 : nRasterXSize = current.size.x;
780 1 : nRasterYSize = current.size.y;
781 1 : nBands = current.size.c;
782 :
783 : // Add the bands, copy constructor so they can be closed independently
784 2 : for (int i = 1; i <= nBands; i++)
785 1 : SetBand(i, new MRFLRasterBand(reinterpret_cast<MRFRasterBand *>(
786 1 : cds->GetRasterBand(i)->GetOverview(l))));
787 1 : return CE_None;
788 : }
789 :
790 : // Is the string positive or not
791 1282 : inline bool on(const char *pszValue)
792 : {
793 1282 : if (!pszValue || pszValue[0] == 0)
794 110 : return false;
795 2332 : return EQUAL(pszValue, "ON") || EQUAL(pszValue, "TRUE") ||
796 2332 : EQUAL(pszValue, "YES");
797 : }
798 :
799 : /**
800 : *\brief Initialize the image structure and the dataset from the XML Raster node
801 : *
802 : * @param image the structure to be initialized
803 : * @param ds the parent dataset, some things get inherited
804 : * @param defimage defimage
805 : *
806 : * The structure should be initialized with the default values as much as
807 : *possible
808 : *
809 : */
810 :
811 367 : static CPLErr Init_Raster(ILImage &image, MRFDataset *ds, CPLXMLNode *defimage)
812 : {
813 : CPLXMLNode *node; // temporary
814 367 : if (!defimage)
815 : {
816 0 : CPLError(CE_Failure, CPLE_AppDefined,
817 : "GDAL MRF: Can't find raster info");
818 0 : return CE_Failure;
819 : }
820 :
821 : // Size is mandatory
822 367 : node = CPLGetXMLNode(defimage, "Size");
823 :
824 367 : if (node)
825 : {
826 367 : image.size = ILSize(static_cast<int>(getXMLNum(node, "x", -1)),
827 367 : static_cast<int>(getXMLNum(node, "y", -1)),
828 367 : static_cast<int>(getXMLNum(node, "z", 1)),
829 367 : static_cast<int>(getXMLNum(node, "c", 1)), 0);
830 : }
831 :
832 : // Basic checks
833 367 : if (!node || image.size.x < 1 || image.size.y < 1 || image.size.z < 0 ||
834 734 : image.size.c < 0 || !GDALCheckBandCount(image.size.c, FALSE))
835 : {
836 0 : CPLError(CE_Failure, CPLE_AppDefined, "Raster size missing or invalid");
837 0 : return CE_Failure;
838 : }
839 :
840 : // Pagesize, defaults to 512,512,1,c
841 367 : image.pagesize = ILSize(std::min(512, image.size.x),
842 367 : std::min(512, image.size.y), 1, image.size.c);
843 :
844 367 : node = CPLGetXMLNode(defimage, "PageSize");
845 367 : if (node)
846 : {
847 367 : image.pagesize =
848 367 : ILSize(static_cast<int>(getXMLNum(node, "x", image.pagesize.x)),
849 367 : static_cast<int>(getXMLNum(node, "y", image.pagesize.y)),
850 : 1, // One slice at a time, forced
851 367 : static_cast<int>(getXMLNum(node, "c", image.pagesize.c)));
852 367 : if (image.pagesize.x < 1 || image.pagesize.y < 1 ||
853 367 : image.pagesize.c <= 0)
854 : {
855 0 : CPLError(CE_Failure, CPLE_IllegalArg, "Invalid PageSize");
856 0 : return CE_Failure;
857 : }
858 : }
859 :
860 : // Page Encoding, defaults to PNG
861 367 : const char *pszCompression = CPLGetXMLValue(defimage, "Compression", "PNG");
862 367 : image.comp = CompToken(pszCompression);
863 367 : if (image.comp == IL_ERR_COMP)
864 : {
865 0 : CPLError(CE_Failure, CPLE_IllegalArg,
866 : "GDAL MRF: Compression %s is unknown", pszCompression);
867 0 : return CE_Failure;
868 : }
869 :
870 : // Is there a palette?
871 : //
872 : // GDAL only supports RGB+A palette, the other modes don't work
873 : //
874 :
875 678 : if ((image.pagesize.c == 1) &&
876 311 : (nullptr != (node = CPLGetXMLNode(defimage, "Palette"))))
877 : {
878 1 : int entries = static_cast<int>(getXMLNum(node, "Size", 255));
879 1 : GDALPaletteInterp eInterp = GPI_RGB;
880 1 : if ((entries > 0) && (entries < 257))
881 : {
882 1 : GDALColorEntry ce_start = {0, 0, 0, 255}, ce_end = {0, 0, 0, 255};
883 :
884 : // Create it and initialize it to black opaque
885 1 : GDALColorTable *poColorTable = new GDALColorTable(eInterp);
886 1 : poColorTable->CreateColorRamp(0, &ce_start, entries - 1, &ce_end);
887 : // Read the values
888 1 : CPLXMLNode *p = CPLGetXMLNode(node, "Entry");
889 1 : if (p)
890 : {
891 : // Initialize the first entry
892 1 : ce_start = GetXMLColorEntry(p);
893 1 : int start_idx = static_cast<int>(getXMLNum(p, "idx", 0));
894 1 : if (start_idx < 0)
895 : {
896 0 : CPLError(CE_Failure, CPLE_IllegalArg,
897 : "GDAL MRF: Palette index %d not allowed",
898 : start_idx);
899 0 : delete poColorTable;
900 0 : return CE_Failure;
901 : }
902 1 : poColorTable->SetColorEntry(start_idx, &ce_start);
903 256 : while (nullptr != (p = SearchXMLSiblings(p, "Entry")))
904 : {
905 : // For every entry, create a ramp
906 255 : ce_end = GetXMLColorEntry(p);
907 : int end_idx =
908 255 : static_cast<int>(getXMLNum(p, "idx", start_idx + 1));
909 255 : if ((end_idx <= start_idx) || (start_idx >= entries))
910 : {
911 0 : CPLError(CE_Failure, CPLE_IllegalArg,
912 : "GDAL MRF: Index Error at index %d", end_idx);
913 0 : delete poColorTable;
914 0 : return CE_Failure;
915 : }
916 255 : poColorTable->CreateColorRamp(start_idx, &ce_start, end_idx,
917 : &ce_end);
918 255 : ce_start = ce_end;
919 255 : start_idx = end_idx;
920 : }
921 : }
922 :
923 1 : ds->SetColorTable(poColorTable);
924 : }
925 : else
926 : {
927 0 : CPLError(CE_Failure, CPLE_IllegalArg,
928 : "GDAL MRF: Palette definition error");
929 0 : return CE_Failure;
930 : }
931 : }
932 :
933 : // Order of increment
934 367 : if (image.pagesize.c != image.size.c && image.pagesize.c != 1)
935 : {
936 : // Fixes heap buffer overflow in
937 : // GDALMRFRasterBand::ReadInterleavedBlock() See
938 : // https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=2884
939 0 : CPLError(CE_Failure, CPLE_NotSupported,
940 : "GDAL MRF: image.pagesize.c = %d and image.size.c = %d",
941 : image.pagesize.c, image.size.c);
942 0 : return CE_Failure;
943 : }
944 :
945 367 : image.order = OrderToken(
946 : CPLGetXMLValue(defimage, "Order",
947 367 : (image.pagesize.c != image.size.c) ? "BAND" : "PIXEL"));
948 367 : if (image.order == IL_ERR_ORD)
949 : {
950 0 : CPLError(CE_Failure, CPLE_AppDefined, "GDAL MRF: Order %s is unknown",
951 : CPLGetXMLValue(defimage, "Order", nullptr));
952 0 : return CE_Failure;
953 : }
954 :
955 367 : const char *photo_val = CPLGetXMLValue(defimage, "Photometric", nullptr);
956 367 : if (photo_val)
957 5 : ds->SetPhotometricInterpretation(photo_val);
958 :
959 367 : image.quality = atoi(CPLGetXMLValue(defimage, "Quality", "85"));
960 367 : if (image.quality < 0 || image.quality > 99)
961 : {
962 0 : CPLError(CE_Warning, CPLE_AppDefined,
963 : "GDAL MRF: Quality setting error, using default of 85");
964 0 : image.quality = 85;
965 : }
966 :
967 : // Data Type, use GDAL Names
968 367 : image.dt = GDALGetDataTypeByName(
969 : CPLGetXMLValue(defimage, "DataType", GDALGetDataTypeName(image.dt)));
970 367 : if (image.dt == GDT_Unknown || GDALGetDataTypeSizeBytes(image.dt) == 0)
971 : {
972 0 : CPLError(CE_Failure, CPLE_AppDefined, "GDAL MRF: Unsupported type");
973 0 : return CE_Failure;
974 : }
975 :
976 : // Check the endianness if needed, assume host order
977 367 : if (is_Endianness_Dependent(image.dt, image.comp))
978 69 : image.nbo = on(CPLGetXMLValue(defimage, "NetByteOrder", "No"));
979 :
980 367 : CPLXMLNode *DataValues = CPLGetXMLNode(defimage, "DataValues");
981 367 : if (nullptr != DataValues)
982 : {
983 27 : const char *pszValue = CPLGetXMLValue(DataValues, "NoData", nullptr);
984 27 : if (pszValue)
985 20 : ds->SetNoDataValue(pszValue);
986 27 : pszValue = CPLGetXMLValue(DataValues, "min", nullptr);
987 27 : if (pszValue)
988 7 : ds->SetMinValue(pszValue);
989 27 : pszValue = CPLGetXMLValue(DataValues, "max", nullptr);
990 27 : if (pszValue)
991 7 : ds->SetMaxValue(pszValue);
992 : }
993 :
994 : // Check that pagesize.c is 64 or less so we can use bitmasks
995 367 : if (image.pagesize.c > 64)
996 : {
997 0 : CPLError(CE_Failure, CPLE_AppDefined,
998 : "MRF: Max number of pixel interleaved bands is 64");
999 0 : return CE_Failure;
1000 : }
1001 :
1002 : // Calculate the page size in bytes
1003 367 : const int nDTSize = GDALGetDataTypeSizeBytes(image.dt);
1004 367 : if (nDTSize <= 0 || image.pagesize.z <= 0 ||
1005 367 : image.pagesize.x > INT_MAX / image.pagesize.y ||
1006 367 : image.pagesize.x * image.pagesize.y > INT_MAX / image.pagesize.z ||
1007 367 : image.pagesize.x * image.pagesize.y * image.pagesize.z >
1008 367 : INT_MAX / image.pagesize.c ||
1009 367 : image.pagesize.x * image.pagesize.y * image.pagesize.z *
1010 367 : image.pagesize.c >
1011 367 : INT_MAX / nDTSize)
1012 : {
1013 0 : CPLError(CE_Failure, CPLE_AppDefined, "MRF page size is too large");
1014 0 : return CE_Failure;
1015 : }
1016 367 : image.pageSizeBytes = nDTSize * image.pagesize.x * image.pagesize.y *
1017 367 : image.pagesize.z * image.pagesize.c;
1018 :
1019 : // Calculate the page count, including the total for the level
1020 367 : image.pagecount = pcount(image.size, image.pagesize);
1021 367 : if (image.pagecount.l < 0)
1022 : {
1023 0 : return CE_Failure;
1024 : }
1025 :
1026 : // Data File Name and base offset
1027 : image.datfname =
1028 367 : getFname(defimage, "DataFile", ds->GetFname(), ILComp_Ext[image.comp]);
1029 367 : image.dataoffset = static_cast<int>(
1030 367 : getXMLNum(CPLGetXMLNode(defimage, "DataFile"), "offset", 0.0));
1031 :
1032 : // Index File Name and base offset
1033 367 : image.idxfname = getFname(defimage, "IndexFile", ds->GetFname(), ".idx");
1034 367 : image.idxoffset = static_cast<int>(
1035 367 : getXMLNum(CPLGetXMLNode(defimage, "IndexFile"), "offset", 0.0));
1036 :
1037 367 : return CE_None;
1038 : }
1039 :
1040 162 : char **MRFDataset::GetFileList()
1041 : {
1042 162 : char **papszFileList = nullptr;
1043 :
1044 162 : string usename = fname;
1045 162 : if (!publicname.empty())
1046 0 : usename = publicname;
1047 : // Add the header file name if it is real
1048 : VSIStatBufL sStat;
1049 162 : if (VSIStatExL(usename.c_str(), &sStat, VSI_STAT_EXISTS_FLAG) == 0)
1050 162 : papszFileList = CSLAddString(papszFileList, usename.c_str());
1051 :
1052 : // These two should be real
1053 : // We don't really want to add these files, since they will be erased when
1054 : // an mrf is overwritten This collides with the concept that the data file
1055 : // never shrinks. Same goes with the index, in case we just want to add
1056 : // things to it.
1057 : // papszFileList = CSLAddString( papszFileList, full.datfname);
1058 : // papszFileList = CSLAddString( papszFileList, full.idxfname);
1059 : // if (!source.empty())
1060 : // papszFileList = CSLAddString( papszFileList, source);
1061 :
1062 324 : return papszFileList;
1063 : }
1064 :
1065 : // Try to create all the folders in the path in sequence, ignore errors
1066 4 : static void mkdir_r(string const &fname)
1067 : {
1068 4 : size_t loc = fname.find_first_of("\\/");
1069 4 : if (loc == string::npos)
1070 0 : return;
1071 : while (true)
1072 : {
1073 5 : ++loc;
1074 5 : loc = fname.find_first_of("\\/", loc);
1075 5 : if (loc == string::npos)
1076 4 : break;
1077 1 : VSIMkdir(fname.substr(0, loc).c_str(), 0);
1078 : }
1079 : }
1080 :
1081 : // Returns the dataset index file or null
1082 7269 : VSILFILE *MRFDataset::IdxFP()
1083 : {
1084 7269 : if (ifp.FP != nullptr)
1085 6999 : return ifp.FP;
1086 :
1087 : // If missing is set, we already checked, there is no index
1088 270 : if (missing)
1089 0 : return nullptr;
1090 :
1091 : // If name starts with '(' it is not a real file name
1092 270 : if (current.idxfname[0] == '(')
1093 0 : return nullptr;
1094 :
1095 270 : const char *mode = "rb";
1096 270 : ifp.acc = GF_Read;
1097 :
1098 270 : if (eAccess == GA_Update || !source.empty())
1099 : {
1100 159 : mode = "r+b";
1101 159 : ifp.acc = GF_Write;
1102 : }
1103 :
1104 270 : ifp.FP = VSIFOpenL(current.idxfname, mode);
1105 :
1106 : // If file didn't open for reading and no_errors is set, just return null
1107 : // and make a note
1108 270 : if (ifp.FP == nullptr && eAccess == GA_ReadOnly && no_errors)
1109 : {
1110 0 : missing = 1;
1111 0 : return nullptr;
1112 : }
1113 :
1114 : // need to create the index file
1115 370 : if (ifp.FP == nullptr && !bCrystalized &&
1116 100 : (eAccess == GA_Update || !source.empty()))
1117 : {
1118 100 : mode = "w+b";
1119 100 : ifp.FP = VSIFOpenL(current.idxfname, mode);
1120 : }
1121 :
1122 270 : if (nullptr == ifp.FP && !source.empty())
1123 : {
1124 : // caching and cloning, try making the folder and attempt again
1125 4 : mkdir_r(current.idxfname);
1126 4 : ifp.FP = VSIFOpenL(current.idxfname, mode);
1127 : }
1128 :
1129 270 : GIntBig expected_size = idxSize;
1130 270 : if (clonedSource)
1131 2 : expected_size *= 2;
1132 :
1133 270 : if (nullptr != ifp.FP)
1134 : {
1135 409 : if (!bCrystalized &&
1136 150 : !CheckFileSize(current.idxfname, expected_size, GA_Update))
1137 : {
1138 0 : CPLError(CE_Failure, CPLE_FileIO,
1139 : "MRF: Can't extend the cache index file %s",
1140 : current.idxfname.c_str());
1141 0 : return nullptr;
1142 : }
1143 :
1144 259 : if (source.empty())
1145 255 : return ifp.FP;
1146 :
1147 : // Make sure the index is large enough before proceeding
1148 : // Timeout in .1 seconds, can't really guarantee the accuracy
1149 : // So this is about half second, should be sufficient
1150 4 : int timeout = 5;
1151 0 : do
1152 : {
1153 4 : if (CheckFileSize(current.idxfname, expected_size, GA_ReadOnly))
1154 4 : return ifp.FP;
1155 0 : CPLSleep(0.100); /* 100 ms */
1156 0 : } while (--timeout);
1157 :
1158 : // If we get here it is a time-out
1159 0 : CPLError(CE_Failure, CPLE_AppDefined,
1160 : "GDAL MRF: Timeout on fetching cloned index file %s\n",
1161 : current.idxfname.c_str());
1162 0 : return nullptr;
1163 : }
1164 :
1165 : // If single tile, and no index file, let the caller figure it out
1166 11 : if (IsSingleTile())
1167 7 : return nullptr;
1168 :
1169 : // Error if this is not a caching MRF
1170 4 : if (source.empty())
1171 : {
1172 0 : CPLError(CE_Failure, CPLE_AppDefined,
1173 : "GDAL MRF: Can't open index file %s\n",
1174 : current.idxfname.c_str());
1175 0 : return nullptr;
1176 : }
1177 :
1178 : // Caching/Cloning MRF and index could be read only
1179 : // Is this actually works, we should try again, maybe somebody else just
1180 : // created the file?
1181 4 : mode = "rb";
1182 4 : ifp.acc = GF_Read;
1183 4 : ifp.FP = VSIFOpenL(current.idxfname, mode);
1184 4 : if (nullptr != ifp.FP)
1185 0 : return ifp.FP;
1186 :
1187 : // Caching and index file absent, create it
1188 : // Due to a race, multiple processes might do this at the same time, but
1189 : // that is fine
1190 4 : ifp.FP = VSIFOpenL(current.idxfname, "wb");
1191 4 : if (nullptr == ifp.FP)
1192 : {
1193 0 : CPLError(CE_Failure, CPLE_AppDefined,
1194 : "Can't create the MRF cache index file %s",
1195 : current.idxfname.c_str());
1196 0 : return nullptr;
1197 : }
1198 4 : VSIFCloseL(ifp.FP);
1199 4 : ifp.FP = nullptr;
1200 :
1201 : // Make it large enough for caching and for cloning
1202 4 : if (!CheckFileSize(current.idxfname, expected_size, GA_Update))
1203 : {
1204 0 : CPLError(CE_Failure, CPLE_AppDefined,
1205 : "Can't extend the cache index file %s",
1206 : current.idxfname.c_str());
1207 0 : return nullptr;
1208 : }
1209 :
1210 : // Try opening it again in rw mode so we can read and write
1211 4 : mode = "r+b";
1212 4 : ifp.acc = GF_Write;
1213 4 : ifp.FP = VSIFOpenL(current.idxfname.c_str(), mode);
1214 :
1215 4 : if (nullptr == ifp.FP)
1216 : {
1217 0 : CPLError(CE_Failure, CPLE_AppDefined,
1218 : "GDAL MRF: Can't reopen cache index file %s\n",
1219 : full.idxfname.c_str());
1220 0 : return nullptr;
1221 : }
1222 4 : return ifp.FP;
1223 : }
1224 :
1225 : //
1226 : // Returns the dataset data file or null
1227 : // Data file is opened either in Read or Append mode, never in straight write
1228 : //
1229 7147 : VSILFILE *MRFDataset::DataFP()
1230 : {
1231 7147 : if (dfp.FP != nullptr)
1232 6880 : return dfp.FP;
1233 267 : const char *mode = "rb";
1234 267 : dfp.acc = GF_Read;
1235 :
1236 : // Open it for writing if updating or if caching
1237 267 : if (eAccess == GA_Update || !source.empty())
1238 : {
1239 158 : mode = "a+b";
1240 158 : dfp.acc = GF_Write;
1241 : }
1242 :
1243 267 : dfp.FP = VSIFOpenL(current.datfname, mode);
1244 267 : if (dfp.FP)
1245 267 : return dfp.FP;
1246 :
1247 : // It could be a caching MRF
1248 0 : if (source.empty())
1249 0 : goto io_error;
1250 :
1251 : // May be there but read only, remember that it was open that way
1252 0 : mode = "rb";
1253 0 : dfp.acc = GF_Read;
1254 0 : dfp.FP = VSIFOpenL(current.datfname, mode);
1255 0 : if (nullptr != dfp.FP)
1256 : {
1257 0 : CPLDebug("MRF_IO", "Opened %s RO mode %s\n", current.datfname.c_str(),
1258 : mode);
1259 0 : return dfp.FP;
1260 : }
1261 :
1262 0 : if (source.empty())
1263 0 : goto io_error;
1264 :
1265 : // caching, maybe the folder didn't exist
1266 0 : mkdir_r(current.datfname);
1267 0 : mode = "a+b";
1268 0 : dfp.acc = GF_Write;
1269 0 : dfp.FP = VSIFOpenL(current.datfname, mode);
1270 0 : if (dfp.FP)
1271 0 : return dfp.FP;
1272 :
1273 0 : io_error:
1274 0 : dfp.FP = nullptr;
1275 0 : CPLError(CE_Failure, CPLE_FileIO, "GDAL MRF: %s : %s", strerror(errno),
1276 : current.datfname.c_str());
1277 0 : return nullptr;
1278 : }
1279 :
1280 : // Builds an XML tree from the current MRF. If written to a file it becomes an
1281 : // MRF
1282 334 : CPLXMLNode *MRFDataset::BuildConfig()
1283 : {
1284 334 : CPLXMLNode *config = CPLCreateXMLNode(nullptr, CXT_Element, "MRF_META");
1285 :
1286 334 : if (!source.empty())
1287 : {
1288 : CPLXMLNode *psCachedSource =
1289 4 : CPLCreateXMLNode(config, CXT_Element, "CachedSource");
1290 : // Should wrap the string in CDATA, in case it is XML
1291 : CPLXMLNode *psSource =
1292 4 : CPLCreateXMLElementAndValue(psCachedSource, "Source", source);
1293 4 : if (clonedSource)
1294 0 : CPLSetXMLValue(psSource, "#clone", "true");
1295 : }
1296 :
1297 : // Use the full size
1298 334 : CPLXMLNode *raster = CPLCreateXMLNode(config, CXT_Element, "Raster");
1299 :
1300 : // Preserve the file names if not the default ones
1301 334 : if (full.datfname != getFname(GetFname(), ILComp_Ext[full.comp]))
1302 0 : CPLCreateXMLElementAndValue(raster, "DataFile", full.datfname.c_str());
1303 334 : if (full.idxfname != getFname(GetFname(), ".idx"))
1304 0 : CPLCreateXMLElementAndValue(raster, "IndexFile", full.idxfname.c_str());
1305 334 : if (spacing != 0)
1306 0 : XMLSetAttributeVal(raster, "Spacing", static_cast<double>(spacing),
1307 : "%.0f");
1308 :
1309 334 : XMLSetAttributeVal(raster, "Size", full.size, "%.0f");
1310 334 : XMLSetAttributeVal(raster, "PageSize", full.pagesize, "%.0f");
1311 :
1312 : #ifdef HAVE_PNG
1313 334 : if (full.comp != IL_PNG)
1314 : #endif
1315 : {
1316 194 : CPLCreateXMLElementAndValue(raster, "Compression", CompName(full.comp));
1317 : }
1318 :
1319 334 : if (full.dt != GDT_UInt8)
1320 202 : CPLCreateXMLElementAndValue(raster, "DataType",
1321 : GDALGetDataTypeName(full.dt));
1322 :
1323 : // special photometric interpretation
1324 334 : if (!photometric.empty())
1325 4 : CPLCreateXMLElementAndValue(raster, "Photometric", photometric);
1326 :
1327 334 : if (!vNoData.empty() || !vMin.empty() || !vMax.empty())
1328 : {
1329 : CPLXMLNode *values =
1330 43 : CPLCreateXMLNode(raster, CXT_Element, "DataValues");
1331 43 : XMLSetAttributeVal(values, "NoData", vNoData);
1332 43 : XMLSetAttributeVal(values, "min", vMin);
1333 43 : XMLSetAttributeVal(values, "max", vMax);
1334 : }
1335 :
1336 : // palette, if we have one
1337 334 : if (poColorTable != nullptr)
1338 : {
1339 1 : const char *pfrmt = "%.0f";
1340 1 : CPLXMLNode *pal = CPLCreateXMLNode(raster, CXT_Element, "Palette");
1341 1 : int sz = poColorTable->GetColorEntryCount();
1342 1 : if (sz != 256)
1343 0 : XMLSetAttributeVal(pal, "Size", poColorTable->GetColorEntryCount());
1344 : // RGB or RGBA for now
1345 257 : for (int i = 0; i < sz; i++)
1346 : {
1347 256 : CPLXMLNode *entry = CPLCreateXMLNode(pal, CXT_Element, "Entry");
1348 256 : const GDALColorEntry *ent = poColorTable->GetColorEntry(i);
1349 : // No need to set the index, it is always from 0 no size-1
1350 256 : XMLSetAttributeVal(entry, "c1", ent->c1, pfrmt);
1351 256 : XMLSetAttributeVal(entry, "c2", ent->c2, pfrmt);
1352 256 : XMLSetAttributeVal(entry, "c3", ent->c3, pfrmt);
1353 256 : if (ent->c4 != 255)
1354 0 : XMLSetAttributeVal(entry, "c4", ent->c4, pfrmt);
1355 : }
1356 : }
1357 :
1358 334 : if (is_Endianness_Dependent(full.dt, full.comp)) // Need to set the order
1359 62 : CPLCreateXMLElementAndValue(raster, "NetByteOrder",
1360 62 : (full.nbo || NET_ORDER) ? "TRUE" : "FALSE");
1361 :
1362 334 : if (full.quality > 0 && full.quality != 85)
1363 12 : CPLCreateXMLElementAndValue(raster, "Quality",
1364 24 : CPLOPrintf("%d", full.quality));
1365 :
1366 : // Done with the raster node
1367 :
1368 334 : if (scale != 0.0)
1369 : {
1370 0 : CPLCreateXMLNode(config, CXT_Element, "Rsets");
1371 0 : CPLSetXMLValue(config, "Rsets.#model", "uniform");
1372 0 : CPLSetXMLValue(config, "Rsets.#scale", PrintDouble(scale));
1373 : }
1374 334 : CPLXMLNode *gtags = CPLCreateXMLNode(config, CXT_Element, "GeoTags");
1375 :
1376 : // Do we have an affine transform different from identity?
1377 334 : GDALGeoTransform gt;
1378 668 : if ((MRFDataset::GetGeoTransform(gt) == CE_None) &&
1379 334 : (gt.xorig != 0 || gt.xscale != 1 || gt.xrot != 0 || gt.yorig != 0 ||
1380 197 : gt.yrot != 0 || gt.yscale != 1))
1381 : {
1382 137 : double minx = gt.xorig;
1383 137 : double maxx = gt.xscale * full.size.x + minx;
1384 137 : double maxy = gt.yorig;
1385 137 : double miny = gt.yscale * full.size.y + maxy;
1386 137 : CPLXMLNode *bbox = CPLCreateXMLNode(gtags, CXT_Element, "BoundingBox");
1387 137 : XMLSetAttributeVal(bbox, "minx", minx);
1388 137 : XMLSetAttributeVal(bbox, "miny", miny);
1389 137 : XMLSetAttributeVal(bbox, "maxx", maxx);
1390 137 : XMLSetAttributeVal(bbox, "maxy", maxy);
1391 : }
1392 :
1393 334 : const char *pszProj = GetProjectionRef();
1394 334 : if (pszProj && (!EQUAL(pszProj, "")))
1395 138 : CPLCreateXMLElementAndValue(gtags, "Projection", pszProj);
1396 :
1397 334 : if (optlist.Count() != 0)
1398 : {
1399 52 : CPLString options;
1400 54 : for (int i = 0; i < optlist.size(); i++)
1401 : {
1402 28 : options += optlist[i];
1403 28 : options += ' ';
1404 : }
1405 26 : options.pop_back();
1406 26 : CPLCreateXMLElementAndValue(config, "Options", options);
1407 : }
1408 :
1409 334 : return config;
1410 : }
1411 :
1412 : /**
1413 : * \brief Populates the dataset variables from the XML definition
1414 : *
1415 : *
1416 : */
1417 367 : CPLErr MRFDataset::Initialize(CPLXMLNode *config)
1418 : {
1419 : // We only need a basic initialization here, usually gets overwritten by the
1420 : // image params
1421 367 : full.dt = GDT_UInt8;
1422 367 : full.hasNoData = false;
1423 367 : full.NoDataValue = 0;
1424 367 : Quality = 85;
1425 :
1426 367 : CPLErr ret = Init_Raster(full, this, CPLGetXMLNode(config, "Raster"));
1427 367 : if (CE_None != ret)
1428 0 : return ret;
1429 :
1430 367 : hasVersions = on(CPLGetXMLValue(config, "Raster.versioned", "no"));
1431 367 : mp_safe = on(CPLGetXMLValue(config, "Raster.mp_safe", "no"));
1432 367 : spacing = atoi(CPLGetXMLValue(config, "Raster.Spacing", "0"));
1433 :
1434 : // The zslice defined in the file wins over the oo or the file argument
1435 367 : if (CPLGetXMLNode(config, "Raster.zslice"))
1436 0 : zslice = atoi(CPLGetXMLValue(config, "Raster.zslice", "0"));
1437 :
1438 367 : Quality = full.quality;
1439 :
1440 : // Bounding box
1441 367 : CPLXMLNode *bbox = CPLGetXMLNode(config, "GeoTags.BoundingBox");
1442 367 : if (nullptr != bbox)
1443 : {
1444 : double x0, x1, y0, y1;
1445 :
1446 161 : x0 = atof(CPLGetXMLValue(bbox, "minx", "0"));
1447 161 : x1 = atof(CPLGetXMLValue(bbox, "maxx", "1"));
1448 161 : y1 = atof(CPLGetXMLValue(bbox, "maxy", "1"));
1449 161 : y0 = atof(CPLGetXMLValue(bbox, "miny", "0"));
1450 :
1451 161 : m_gt.xorig = x0;
1452 161 : m_gt.xscale = (x1 - x0) / full.size.x;
1453 161 : m_gt.xrot = 0;
1454 161 : m_gt.yorig = y1;
1455 161 : m_gt.yrot = 0;
1456 161 : m_gt.yscale = (y0 - y1) / full.size.y;
1457 161 : bGeoTransformValid = TRUE;
1458 : }
1459 :
1460 : const char *pszRawProjFromXML =
1461 367 : CPLGetXMLValue(config, "GeoTags.Projection", "");
1462 367 : if (strlen(pszRawProjFromXML) != 0)
1463 162 : m_oSRS.SetFromUserInput(
1464 : pszRawProjFromXML,
1465 : OGRSpatialReference::SET_FROM_USER_INPUT_LIMITATIONS_get());
1466 :
1467 : // Copy the full size to current, data and index are not yet open
1468 367 : current = full;
1469 367 : if (current.size.z != 1)
1470 : {
1471 0 : SetMetadataItem("ZSIZE", CPLOPrintf("%d", current.size.z),
1472 0 : "IMAGE_STRUCTURE");
1473 0 : SetMetadataItem("ZSLICE", CPLOPrintf("%d", zslice), "IMAGE_STRUCTURE");
1474 : // Capture the zslice in pagesize.l
1475 0 : current.pagesize.l = zslice;
1476 : // Adjust offset for base image
1477 0 : if (full.size.z <= 0)
1478 : {
1479 0 : CPLError(CE_Failure, CPLE_AppDefined,
1480 : "GDAL MRF: Invalid Raster.z value");
1481 0 : return CE_Failure;
1482 : }
1483 0 : if (zslice >= full.size.z)
1484 : {
1485 0 : CPLError(CE_Failure, CPLE_AppDefined, "GDAL MRF: Invalid slice");
1486 0 : return CE_Failure;
1487 : }
1488 :
1489 0 : current.idxoffset +=
1490 0 : (current.pagecount.l / full.size.z) * zslice * sizeof(ILIdx);
1491 : }
1492 :
1493 : // Dataset metadata setup
1494 367 : SetMetadataItem("INTERLEAVE", OrderName(current.order), "IMAGE_STRUCTURE");
1495 367 : SetMetadataItem("COMPRESSION", CompName(current.comp), "IMAGE_STRUCTURE");
1496 :
1497 367 : if (is_Endianness_Dependent(current.dt, current.comp))
1498 69 : SetMetadataItem("NETBYTEORDER", current.nbo ? "TRUE" : "FALSE",
1499 69 : "IMAGE_STRUCTURE");
1500 :
1501 : // Open the files for the current image, either RW or RO
1502 367 : nRasterXSize = current.size.x;
1503 367 : nRasterYSize = current.size.y;
1504 367 : nBands = current.size.c;
1505 :
1506 367 : if (!nBands || !nRasterXSize || !nRasterYSize)
1507 : {
1508 0 : CPLError(CE_Failure, CPLE_AppDefined, "GDAL MRF: Image size missing");
1509 0 : return CE_Failure;
1510 : }
1511 :
1512 : // Pick up the source data image, if there is one
1513 367 : source = CPLGetXMLValue(config, "CachedSource.Source", "");
1514 : // Is it a clone?
1515 367 : clonedSource =
1516 367 : on(CPLGetXMLValue(config, "CachedSource.Source.clone", "no"));
1517 : // Pick up the options, if any
1518 : optlist.Assign(CSLTokenizeString2(
1519 : CPLGetXMLValue(config, "Options", nullptr), " \t\n\r",
1520 367 : CSLT_STRIPLEADSPACES | CSLT_STRIPENDSPACES));
1521 :
1522 : // Load all the options in the IMAGE_STRUCTURE metadata
1523 398 : for (int i = 0; i < optlist.Count(); i++)
1524 : {
1525 62 : CPLString s(optlist[i]);
1526 31 : size_t nSepPos = s.find_first_of(":=");
1527 31 : if (std::string::npos != nSepPos)
1528 : {
1529 31 : s.resize(nSepPos);
1530 31 : SetMetadataItem(s, optlist.FetchNameValue(s), "IMAGE_STRUCTURE");
1531 : }
1532 : }
1533 :
1534 : // We have the options, so we can call rasterband
1535 800 : for (int i = 1; i <= nBands; i++)
1536 : {
1537 : // The overviews are low resolution copies of the current one.
1538 485 : MRFRasterBand *band = newMRFRasterBand(this, current, i);
1539 485 : if (!band)
1540 52 : return CE_Failure;
1541 :
1542 433 : GDALColorInterp ci = GCI_Undefined;
1543 :
1544 : // Default color interpretation
1545 433 : switch (nBands)
1546 : {
1547 265 : case 1:
1548 : case 2:
1549 265 : ci = (i == 1) ? GCI_GrayIndex : GCI_AlphaBand;
1550 265 : break;
1551 153 : case 3:
1552 : case 4:
1553 153 : if (i < 3)
1554 100 : ci = (i == 1) ? GCI_RedBand : GCI_GreenBand;
1555 : else
1556 53 : ci = (i == 3) ? GCI_BlueBand : GCI_AlphaBand;
1557 : }
1558 :
1559 433 : if (GetColorTable())
1560 1 : ci = GCI_PaletteIndex;
1561 :
1562 : // Legacy, deprecated
1563 433 : if (optlist.FetchBoolean("MULTISPECTRAL", FALSE))
1564 0 : ci = GCI_Undefined;
1565 :
1566 : // New style
1567 433 : if (!photometric.empty())
1568 : {
1569 15 : if ("MULTISPECTRAL" == photometric)
1570 0 : ci = GCI_Undefined;
1571 : }
1572 :
1573 433 : band->SetColorInterpretation(ci);
1574 433 : SetBand(i, band);
1575 : }
1576 :
1577 315 : CPLXMLNode *rsets = CPLGetXMLNode(config, "Rsets");
1578 315 : if (nullptr != rsets && nullptr != rsets->psChild)
1579 : {
1580 : // We have rsets
1581 :
1582 : // Regular spaced overlays, until everything fits in a single tile
1583 32 : if (EQUAL("uniform", CPLGetXMLValue(rsets, "model", "uniform")))
1584 : {
1585 32 : scale = getXMLNum(rsets, "scale", 2.0);
1586 32 : if (scale <= 1)
1587 : {
1588 0 : CPLError(CE_Failure, CPLE_AppDefined,
1589 : "MRF: zoom factor less than unit not allowed");
1590 0 : return CE_Failure;
1591 : }
1592 : // Looks like there are overlays
1593 32 : AddOverviews(int(scale));
1594 : }
1595 : else
1596 : {
1597 0 : CPLError(CE_Failure, CPLE_AppDefined, "Unknown Rset definition");
1598 0 : return CE_Failure;
1599 : }
1600 : }
1601 :
1602 315 : idxSize = IdxSize(full, int(scale));
1603 315 : if (idxSize == 0)
1604 0 : return CE_Failure;
1605 :
1606 : // If not set by the bands, get a pageSizeBytes buffer
1607 315 : if (GetPBufferSize() == 0 && !SetPBuffer(current.pageSizeBytes))
1608 0 : return CE_Failure;
1609 :
1610 315 : if (hasVersions)
1611 : { // It has versions, but how many?
1612 5 : verCount = 0; // Assume it only has one
1613 : VSIStatBufL statb;
1614 : // If the file exists, compute the last version number
1615 5 : if (0 == VSIStatL(full.idxfname, &statb))
1616 5 : verCount = int(statb.st_size / idxSize - 1);
1617 : }
1618 :
1619 315 : return CE_None;
1620 : }
1621 :
1622 0 : static inline bool has_path(const CPLString &name)
1623 : {
1624 0 : return name.find_first_of("/\\") != string::npos;
1625 : }
1626 :
1627 : // Does name look like an absolute gdal file name?
1628 4 : static inline bool is_absolute(const CPLString &name)
1629 : {
1630 4 : return (name.find_first_of("/\\") == 0) // Starts with root
1631 4 : || (name.size() > 1 && name[1] == ':' &&
1632 0 : isalpha(static_cast<unsigned char>(
1633 0 : name[0]))) // Starts with drive letter
1634 8 : || (name[0] == '<'); // Maybe it is XML
1635 : }
1636 :
1637 : // Add the dirname of path to the beginning of name, if it is relative
1638 : // returns true if name was modified
1639 4 : static inline bool make_absolute(CPLString &name, const CPLString &path)
1640 : {
1641 4 : if (!is_absolute(name) && (path.find_first_of("/\\") != string::npos))
1642 : {
1643 4 : name = path.substr(0, path.find_last_of("/\\") + 1) + name;
1644 4 : return true;
1645 : }
1646 0 : return false;
1647 : }
1648 :
1649 : /**
1650 : *\brief Get the source dataset, open it if necessary
1651 : */
1652 5 : GDALDataset *MRFDataset::GetSrcDS()
1653 : {
1654 5 : if (poSrcDS)
1655 1 : return poSrcDS;
1656 4 : if (source.empty())
1657 0 : return nullptr;
1658 :
1659 : // Stub out the error handler
1660 4 : CPLPushErrorHandler(CPLQuietErrorHandler);
1661 : // Try open the source dataset as is
1662 4 : poSrcDS =
1663 4 : GDALDataset::FromHandle(GDALOpenShared(source.c_str(), GA_ReadOnly));
1664 4 : CPLPopErrorHandler();
1665 :
1666 : // It the open fails, try again with the current dataset path prepended
1667 4 : if (!poSrcDS && make_absolute(source, fname))
1668 4 : poSrcDS = GDALDataset::FromHandle(
1669 : GDALOpenShared(source.c_str(), GA_ReadOnly));
1670 :
1671 4 : if (0 == source.find("<MRF_META>") && has_path(fname))
1672 : {
1673 : // MRF XML source, might need to patch the file names with the current
1674 : // one
1675 0 : MRFDataset *poMRFDS = dynamic_cast<MRFDataset *>(poSrcDS);
1676 0 : if (!poMRFDS)
1677 : {
1678 0 : delete poSrcDS;
1679 0 : poSrcDS = nullptr;
1680 0 : return nullptr;
1681 : }
1682 0 : make_absolute(poMRFDS->current.datfname, fname);
1683 0 : make_absolute(poMRFDS->current.idxfname, fname);
1684 : }
1685 4 : mp_safe = true; // Turn on MP safety
1686 4 : return poSrcDS;
1687 : }
1688 :
1689 : /**
1690 : *\brief Add or verify that all overlays exits
1691 : *
1692 : * @return size of the index file
1693 : */
1694 :
1695 62 : GIntBig MRFDataset::AddOverviews(int scaleIn)
1696 : {
1697 : // Fit the overlays
1698 62 : ILImage img = current;
1699 134 : while (1 != img.pagecount.x * img.pagecount.y)
1700 : {
1701 : // Adjust raster data for next level
1702 : // Adjust the offsets for indices left at this level
1703 72 : img.idxoffset += sizeof(ILIdx) * img.pagecount.l / img.size.z *
1704 72 : (img.size.z - zslice);
1705 :
1706 : // Next overview size
1707 72 : img.size.x = pcount(img.size.x, scaleIn);
1708 72 : img.size.y = pcount(img.size.y, scaleIn);
1709 72 : img.size.l++; // Increment the level
1710 72 : img.pagecount = pcount(img.size, img.pagesize);
1711 :
1712 : // And adjust the offset again, within next level
1713 72 : img.idxoffset += sizeof(ILIdx) * img.pagecount.l / img.size.z * zslice;
1714 72 : int l = static_cast<int>(img.size.l);
1715 : // Create and register the overviews for each band
1716 144 : for (int i = 1; i <= nBands; i++)
1717 : {
1718 : MRFRasterBand *b =
1719 72 : reinterpret_cast<MRFRasterBand *>(GetRasterBand(i));
1720 72 : if (!(b->GetOverview(l - 1)))
1721 72 : b->AddOverview(newMRFRasterBand(this, img, i, l));
1722 : }
1723 : }
1724 :
1725 : // Last adjustment, should be a single set of c and leftover z tiles
1726 62 : return img.idxoffset +
1727 124 : sizeof(ILIdx) * img.pagecount.l / img.size.z * (img.size.z - zslice);
1728 : }
1729 :
1730 : //
1731 : // set an entry if it doesn't already exist
1732 : //
1733 135 : static char **CSLAddIfMissing(char **papszList, const char *pszName,
1734 : const char *pszValue)
1735 : {
1736 135 : if (CSLFetchNameValue(papszList, pszName))
1737 14 : return papszList;
1738 121 : return CSLSetNameValue(papszList, pszName, pszValue);
1739 : }
1740 :
1741 : // CreateCopy implemented based on Create
1742 134 : GDALDataset *MRFDataset::CreateCopy(const char *pszFilename,
1743 : GDALDataset *poSrcDS, int /*bStrict*/,
1744 : CSLConstList papszOptions,
1745 : GDALProgressFunc pfnProgress,
1746 : void *pProgressData)
1747 : {
1748 268 : ILImage img;
1749 :
1750 134 : int x = poSrcDS->GetRasterXSize();
1751 134 : int y = poSrcDS->GetRasterYSize();
1752 134 : int nBands = poSrcDS->GetRasterCount();
1753 134 : if (nBands == 0)
1754 : {
1755 1 : CPLError(CE_Failure, CPLE_NotSupported, "nBands == 0 not supported");
1756 1 : return nullptr;
1757 : }
1758 133 : GDALRasterBand *poSrcBand1 = poSrcDS->GetRasterBand(1);
1759 :
1760 133 : GDALDataType dt = poSrcBand1->GetRasterDataType();
1761 : // Have our own options, to modify as we want
1762 133 : char **options = CSLDuplicate(papszOptions);
1763 :
1764 : const char *pszValue =
1765 133 : poSrcDS->GetMetadataItem("INTERLEAVE", "IMAGE_STRUCTURE");
1766 : options =
1767 133 : CSLAddIfMissing(options, "INTERLEAVE", pszValue ? pszValue : "PIXEL");
1768 : int xb, yb;
1769 133 : poSrcBand1->GetBlockSize(&xb, &yb);
1770 :
1771 : // Keep input block size if it exists and not explicitly set
1772 134 : if (CSLFetchNameValue(options, "BLOCKSIZE") == nullptr && xb != x &&
1773 1 : yb != y)
1774 : {
1775 1 : options = CSLAddIfMissing(options, "BLOCKXSIZE",
1776 2 : PrintDouble(xb, "%d").c_str());
1777 1 : options = CSLAddIfMissing(options, "BLOCKYSIZE",
1778 2 : PrintDouble(yb, "%d").c_str());
1779 : }
1780 :
1781 133 : MRFDataset *poDS = nullptr;
1782 : try
1783 : {
1784 : poDS = reinterpret_cast<MRFDataset *>(
1785 133 : Create(pszFilename, x, y, nBands, dt, options));
1786 :
1787 133 : if (poDS == nullptr || poDS->bCrystalized)
1788 11 : throw CPLOPrintf("MRF: Can't create %s", pszFilename);
1789 :
1790 122 : img = poDS->current; // Deal with the current one here
1791 :
1792 : // Copy data values from source
1793 284 : for (int i = 0; i < poDS->nBands; i++)
1794 : {
1795 : int bHas;
1796 : double dfData;
1797 162 : GDALRasterBand *srcBand = poSrcDS->GetRasterBand(i + 1);
1798 162 : GDALRasterBand *mBand = poDS->GetRasterBand(i + 1);
1799 162 : dfData = srcBand->GetNoDataValue(&bHas);
1800 162 : if (bHas)
1801 : {
1802 16 : poDS->vNoData.push_back(dfData);
1803 16 : mBand->SetNoDataValue(dfData);
1804 : }
1805 162 : dfData = srcBand->GetMinimum(&bHas);
1806 162 : if (bHas)
1807 17 : poDS->vMin.push_back(dfData);
1808 162 : dfData = srcBand->GetMaximum(&bHas);
1809 162 : if (bHas)
1810 17 : poDS->vMax.push_back(dfData);
1811 :
1812 : // Copy the band metadata, PAM will handle it
1813 162 : CSLConstList meta = srcBand->GetMetadata("IMAGE_STRUCTURE");
1814 162 : if (CSLCount(meta))
1815 7 : mBand->SetMetadata(meta, "IMAGE_STRUCTURE");
1816 :
1817 162 : meta = srcBand->GetMetadata();
1818 162 : if (CSLCount(meta))
1819 18 : mBand->SetMetadata(meta);
1820 : }
1821 :
1822 : // Geotags
1823 122 : GDALGeoTransform gt;
1824 122 : if (CE_None == poSrcDS->GetGeoTransform(gt))
1825 119 : poDS->SetGeoTransform(gt);
1826 :
1827 122 : const auto poSRS = poSrcDS->GetSpatialRef();
1828 122 : if (poSRS)
1829 118 : poDS->m_oSRS = *poSRS;
1830 :
1831 : // Color palette if we only have one band
1832 225 : if (1 == nBands &&
1833 103 : GCI_PaletteIndex == poSrcBand1->GetColorInterpretation())
1834 1 : poDS->SetColorTable(poSrcBand1->GetColorTable()->Clone());
1835 :
1836 : // Finally write the XML in the right file name
1837 122 : if (!poDS->Crystalize())
1838 10 : throw CPLString("MRF: Error creating files");
1839 : }
1840 21 : catch (const CPLString &e)
1841 : {
1842 21 : if (nullptr != poDS)
1843 10 : delete poDS;
1844 21 : CPLError(CE_Failure, CPLE_ObjectNull, "%s", e.c_str());
1845 21 : poDS = nullptr;
1846 : }
1847 :
1848 133 : CSLDestroy(options);
1849 133 : if (nullptr == poDS)
1850 21 : return nullptr;
1851 :
1852 112 : char **papszFileList = poDS->GetFileList();
1853 112 : poDS->oOvManager.Initialize(poDS, poDS->GetPhysicalFilename(),
1854 : papszFileList);
1855 112 : CSLDestroy(papszFileList);
1856 :
1857 112 : CPLErr err = CE_None;
1858 : // Have PAM copy all, but skip the mask
1859 112 : int nCloneFlags = GCIF_PAM_DEFAULT & ~GCIF_MASK;
1860 :
1861 : // If copy is disabled, we're done, we just created an empty MRF
1862 112 : if (!on(CSLFetchNameValue(papszOptions, "NOCOPY")))
1863 : {
1864 : // Use the GDAL copy call
1865 : // Need to flag the dataset as compressed (COMPRESSED=TRUE) to force
1866 : // block writes This might not be what we want, if the input and out
1867 : // order is truly separate
1868 110 : nCloneFlags |= GCIF_MASK; // We do copy the data, so copy the mask too
1869 : // if necessary
1870 110 : char **papszCWROptions = nullptr;
1871 : papszCWROptions =
1872 110 : CSLAddNameValue(papszCWROptions, "COMPRESSED", "TRUE");
1873 :
1874 : #ifdef HAVE_JPEG
1875 : // Use the Zen version of the CopyWholeRaster if input has a dataset
1876 : // mask and JPEGs are generated
1877 112 : if (GMF_PER_DATASET == poSrcDS->GetRasterBand(1)->GetMaskFlags() &&
1878 2 : (poDS->current.comp == IL_JPEG
1879 : #ifdef HAVE_PNG
1880 0 : || poDS->current.comp == IL_JPNG
1881 : #endif
1882 : ))
1883 : {
1884 2 : err = poDS->ZenCopy(poSrcDS, pfnProgress, pProgressData);
1885 2 : nCloneFlags ^= GCIF_MASK; // Turn the external mask off
1886 : }
1887 : else
1888 : #endif
1889 : {
1890 108 : err = GDALDatasetCopyWholeRaster(
1891 : (GDALDatasetH)poSrcDS, (GDALDatasetH)poDS, papszCWROptions,
1892 : pfnProgress, pProgressData);
1893 : }
1894 :
1895 110 : CSLDestroy(papszCWROptions);
1896 : }
1897 :
1898 112 : if (CE_None == err)
1899 112 : err = poDS->CloneInfo(poSrcDS, nCloneFlags);
1900 :
1901 112 : if (CE_Failure == err)
1902 : {
1903 0 : delete poDS;
1904 0 : return nullptr;
1905 : }
1906 :
1907 112 : return poDS;
1908 : }
1909 :
1910 : // Prepares the data so it is suitable for Zen JPEG encoding, based on input
1911 : // mask If bFBO is set, only the values of the first band are set non-zero when
1912 : // needed
1913 : template <typename T>
1914 2 : static void ZenFilter(T *buffer, GByte *mask, int nPixels, int nBands,
1915 : bool bFBO)
1916 : {
1917 524290 : for (int i = 0; i < nPixels; i++)
1918 : {
1919 524288 : if (mask[i] == 0)
1920 : { // enforce zero values
1921 516096 : for (int b = 0; b < nBands; b++)
1922 387072 : buffer[nBands * i + b] = 0;
1923 : }
1924 : else
1925 : { // enforce non-zero
1926 395264 : if (bFBO)
1927 : { // First band only
1928 197632 : bool f = true;
1929 790528 : for (int b = 0; b < nBands; b++)
1930 : {
1931 592896 : if (0 == buffer[nBands * i + b])
1932 : {
1933 0 : f = false;
1934 0 : break;
1935 : }
1936 : }
1937 197632 : if (f)
1938 197632 : buffer[nBands * i] = 1;
1939 : }
1940 : else
1941 : { // Every band
1942 790528 : for (int b = 0; b < nBands; b++)
1943 592896 : if (0 == buffer[nBands * i + b])
1944 0 : buffer[nBands * i + b] = 1;
1945 : }
1946 : }
1947 : }
1948 2 : }
1949 :
1950 : // Custom CopyWholeRaster for Zen JPEG, called when the input has a PER_DATASET
1951 : // mask Works like GDALDatasetCopyWholeRaster, but it does filter the input data
1952 : // based on the mask
1953 : //
1954 2 : CPLErr MRFDataset::ZenCopy(GDALDataset *poSrc, GDALProgressFunc pfnProgress,
1955 : void *pProgressData)
1956 : {
1957 2 : VALIDATE_POINTER1(poSrc, "MRF:ZenCopy", CE_Failure);
1958 :
1959 2 : if (!pfnProgress)
1960 0 : pfnProgress = GDALDummyProgress;
1961 :
1962 : /* -------------------------------------------------------------------- */
1963 : /* Confirm the datasets match in size and band counts. */
1964 : /* -------------------------------------------------------------------- */
1965 2 : const int nXSize = GetRasterXSize();
1966 2 : const int nYSize = GetRasterYSize();
1967 2 : const int nBandCount = GetRasterCount();
1968 :
1969 2 : if (poSrc->GetRasterXSize() != nXSize ||
1970 4 : poSrc->GetRasterYSize() != nYSize ||
1971 2 : poSrc->GetRasterCount() != nBandCount)
1972 : {
1973 0 : CPLError(CE_Failure, CPLE_AppDefined,
1974 : "Input and output dataset sizes or band counts do not\n"
1975 : "match in GDALDatasetCopyWholeRaster()");
1976 0 : return CE_Failure;
1977 : }
1978 :
1979 : /* -------------------------------------------------------------------- */
1980 : /* Get our prototype band, and assume the others are similarly */
1981 : /* configured. Also get the per_dataset mask */
1982 : /* -------------------------------------------------------------------- */
1983 2 : GDALRasterBand *poSrcPrototypeBand = poSrc->GetRasterBand(1);
1984 2 : GDALRasterBand *poDstPrototypeBand = GetRasterBand(1);
1985 2 : GDALRasterBand *poSrcMask = poSrcPrototypeBand->GetMaskBand();
1986 :
1987 2 : const int nPageXSize = current.pagesize.x;
1988 2 : const int nPageYSize = current.pagesize.y;
1989 2 : const double nTotalBlocks =
1990 2 : static_cast<double>(DIV_ROUND_UP(nYSize, nPageYSize)) *
1991 2 : static_cast<double>(DIV_ROUND_UP(nXSize, nPageXSize));
1992 2 : const GDALDataType eDT = poDstPrototypeBand->GetRasterDataType();
1993 :
1994 : // All the bands are done per block
1995 : // this flag tells us to apply the Zen filter to the first band only
1996 2 : const bool bFirstBandOnly = (current.order == IL_Interleaved);
1997 :
1998 2 : if (!pfnProgress(0.0, nullptr, pProgressData))
1999 : {
2000 0 : CPLError(CE_Failure, CPLE_UserInterrupt,
2001 : "User terminated CreateCopy()");
2002 0 : return CE_Failure;
2003 : }
2004 :
2005 2 : const int nPixelCount = nPageXSize * nPageYSize;
2006 2 : const int dts = GDALGetDataTypeSizeBytes(eDT);
2007 2 : void *buffer = VSI_MALLOC3_VERBOSE(nPixelCount, nBandCount, dts);
2008 2 : GByte *buffer_mask = nullptr;
2009 2 : if (buffer)
2010 2 : buffer_mask = static_cast<GByte *>(VSI_MALLOC_VERBOSE(nPixelCount));
2011 :
2012 2 : if (!buffer || !buffer_mask)
2013 : {
2014 : // Just in case buffers did get allocated
2015 0 : CPLFree(buffer);
2016 0 : CPLFree(buffer_mask);
2017 0 : CPLError(CE_Failure, CPLE_OutOfMemory, "Can't allocate copy buffer");
2018 0 : return CE_Failure;
2019 : }
2020 :
2021 2 : int nBlocksDone = 0;
2022 2 : CPLErr eErr = CE_None;
2023 : // Advise the source that a complete read will be done
2024 2 : poSrc->AdviseRead(0, 0, nXSize, nYSize, nXSize, nYSize, eDT, nBandCount,
2025 2 : nullptr, nullptr);
2026 :
2027 : // For every block, break on error
2028 4 : for (int row = 0; row < nYSize && eErr == CE_None; row += nPageYSize)
2029 : {
2030 2 : int nRows = std::min(nPageYSize, nYSize - row);
2031 4 : for (int col = 0; col < nXSize && eErr == CE_None; col += nPageXSize)
2032 : {
2033 2 : int nCols = std::min(nPageXSize, nXSize - col);
2034 :
2035 : // Report
2036 2 : if (eErr == CE_None && !pfnProgress(nBlocksDone++ / nTotalBlocks,
2037 : nullptr, pProgressData))
2038 : {
2039 0 : eErr = CE_Failure;
2040 0 : CPLError(CE_Failure, CPLE_UserInterrupt,
2041 : "User terminated CreateCopy()");
2042 0 : break;
2043 : }
2044 :
2045 : // Get the data mask as byte
2046 2 : eErr = poSrcMask->RasterIO(GF_Read, col, row, nCols, nRows,
2047 : buffer_mask, nCols, nRows, GDT_UInt8, 0,
2048 : 0, nullptr);
2049 :
2050 2 : if (eErr != CE_None)
2051 0 : break;
2052 :
2053 : // If there is no data at all, skip this block
2054 2 : if (MatchCount(buffer_mask, nPixelCount, static_cast<GByte>(0)) ==
2055 : nPixelCount)
2056 0 : continue;
2057 :
2058 : // get the data in the buffer, interleaved
2059 4 : eErr = poSrc->RasterIO(
2060 : GF_Read, col, row, nCols, nRows, buffer, nCols, nRows, eDT,
2061 2 : nBandCount, nullptr, static_cast<GSpacing>(nBands) * dts,
2062 2 : static_cast<GSpacing>(nBands) * dts * nCols, dts, nullptr);
2063 :
2064 2 : if (eErr != CE_None)
2065 0 : break;
2066 :
2067 : // This is JPEG, only 8 and 12(16) bits unsigned integer types are
2068 : // valid
2069 2 : switch (eDT)
2070 : {
2071 2 : case GDT_UInt8:
2072 2 : ZenFilter(reinterpret_cast<GByte *>(buffer), buffer_mask,
2073 : nPixelCount, nBandCount, bFirstBandOnly);
2074 2 : break;
2075 0 : case GDT_UInt16:
2076 0 : ZenFilter(reinterpret_cast<GUInt16 *>(buffer), buffer_mask,
2077 : nPixelCount, nBandCount, bFirstBandOnly);
2078 0 : break;
2079 0 : default:
2080 0 : CPLError(CE_Failure, CPLE_AppDefined,
2081 : "Unsupported data type for Zen filter");
2082 0 : eErr = CE_Failure;
2083 0 : break;
2084 : }
2085 :
2086 : // Write
2087 2 : if (eErr == CE_None)
2088 2 : eErr = RasterIO(
2089 : GF_Write, col, row, nCols, nRows, buffer, nCols, nRows, eDT,
2090 2 : nBandCount, nullptr, static_cast<GSpacing>(nBands) * dts,
2091 2 : static_cast<GSpacing>(nBands) * dts * nCols, dts, nullptr);
2092 :
2093 : } // Columns
2094 2 : if (eErr != CE_None)
2095 0 : break;
2096 :
2097 : } // Rows
2098 :
2099 : // Cleanup
2100 2 : CPLFree(buffer);
2101 2 : CPLFree(buffer_mask);
2102 :
2103 : // Final report
2104 2 : if (eErr == CE_None && !pfnProgress(1.0, nullptr, pProgressData))
2105 : {
2106 0 : eErr = CE_Failure;
2107 0 : CPLError(CE_Failure, CPLE_UserInterrupt,
2108 : "User terminated CreateCopy()");
2109 : }
2110 :
2111 2 : return eErr;
2112 : }
2113 :
2114 : // Apply open options to the current dataset
2115 : // Called before the configuration is read
2116 205 : void MRFDataset::ProcessOpenOptions(CSLConstList papszOptions)
2117 : {
2118 205 : no_errors = CSLFetchBoolean(papszOptions, "NOERRORS", FALSE);
2119 205 : const char *val = CSLFetchNameValue(papszOptions, "ZSLICE");
2120 205 : if (val)
2121 0 : zslice = atoi(val);
2122 205 : }
2123 :
2124 : // Apply create options to the current dataset, only valid during creation
2125 162 : void MRFDataset::ProcessCreateOptions(CSLConstList papszOptions)
2126 : {
2127 162 : assert(!bCrystalized);
2128 324 : const CPLStringList opt(papszOptions);
2129 162 : ILImage &img(full);
2130 :
2131 162 : const char *val = opt.FetchNameValue("COMPRESS");
2132 162 : if (val && IL_ERR_COMP == (img.comp = CompToken(val)))
2133 0 : throw CPLString("GDAL MRF: Error setting compression");
2134 :
2135 162 : val = opt.FetchNameValue("INTERLEAVE");
2136 162 : if (val && IL_ERR_ORD == (img.order = OrderToken(val)))
2137 0 : throw CPLString("GDAL MRF: Error setting interleave");
2138 :
2139 162 : val = opt.FetchNameValue("QUALITY");
2140 162 : if (val)
2141 6 : img.quality = atoi(val);
2142 :
2143 162 : val = opt.FetchNameValue("ZSIZE");
2144 162 : if (val)
2145 0 : img.size.z = atoi(val);
2146 :
2147 162 : val = opt.FetchNameValue("BLOCKXSIZE");
2148 162 : if (val)
2149 1 : img.pagesize.x = atoi(val);
2150 :
2151 162 : val = opt.FetchNameValue("BLOCKYSIZE");
2152 162 : if (val)
2153 1 : img.pagesize.y = atoi(val);
2154 :
2155 162 : val = opt.FetchNameValue("BLOCKSIZE");
2156 162 : if (val)
2157 30 : img.pagesize.x = img.pagesize.y = atoi(val);
2158 :
2159 162 : img.nbo = opt.FetchBoolean("NETBYTEORDER", FALSE) != FALSE;
2160 :
2161 162 : val = opt.FetchNameValue("CACHEDSOURCE");
2162 162 : if (val)
2163 : {
2164 2 : source = val;
2165 2 : nocopy = opt.FetchBoolean("NOCOPY", FALSE);
2166 : }
2167 :
2168 162 : val = opt.FetchNameValue("UNIFORM_SCALE");
2169 162 : if (val)
2170 0 : scale = atoi(val);
2171 :
2172 162 : val = opt.FetchNameValue("PHOTOMETRIC");
2173 162 : if (val)
2174 2 : photometric = val;
2175 :
2176 162 : val = opt.FetchNameValue("DATANAME");
2177 162 : if (val)
2178 0 : img.datfname = val;
2179 :
2180 162 : val = opt.FetchNameValue("INDEXNAME");
2181 162 : if (val)
2182 0 : img.idxfname = val;
2183 :
2184 162 : val = opt.FetchNameValue("SPACING");
2185 162 : if (val)
2186 0 : spacing = atoi(val);
2187 :
2188 : optlist.Assign(
2189 : CSLTokenizeString2(opt.FetchNameValue("OPTIONS"), " \t\n\r",
2190 162 : CSLT_STRIPLEADSPACES | CSLT_STRIPENDSPACES));
2191 :
2192 : // General Fixups
2193 162 : if (img.order == IL_Interleaved)
2194 41 : img.pagesize.c = img.size.c;
2195 :
2196 : // Compression dependent fixups
2197 162 : }
2198 :
2199 : /**
2200 : *\brief Create an MRF dataset, some settings can be changed later
2201 : * papszOptions might be anything that an MRF might take
2202 : * Still missing are the georeference ...
2203 : *
2204 : */
2205 :
2206 166 : GDALDataset *MRFDataset::Create(const char *pszName, int nXSize, int nYSize,
2207 : int nBandsIn, GDALDataType eType,
2208 : CSLConstList papszOptions)
2209 : {
2210 166 : if (nBandsIn == 0)
2211 : {
2212 1 : CPLError(CE_Failure, CPLE_NotSupported, "No bands defined");
2213 1 : return nullptr;
2214 : }
2215 :
2216 165 : MRFDataset *poDS = new MRFDataset();
2217 165 : CPLErr err = CE_None;
2218 165 : poDS->fname = pszName;
2219 165 : poDS->nBands = nBandsIn;
2220 :
2221 : // Don't know what to do with these in this call
2222 : // int level = -1;
2223 : // int version = 0;
2224 :
2225 165 : size_t pos = poDS->fname.find(":MRF:");
2226 165 : if (string::npos != pos)
2227 : { // Tokenize and pick known options
2228 0 : vector<string> tokens;
2229 0 : stringSplit(tokens, poDS->fname, pos + 5, ':');
2230 : // level = getnum(tokens, 'L', -1);
2231 : // version = getnum(tokens, 'V', 0);
2232 0 : poDS->zslice = getnum(tokens, 'Z', 0);
2233 0 : poDS->fname.resize(pos); // Cut the ornamentations
2234 : }
2235 :
2236 : // Try creating the mrf file early, to avoid failing on Crystalize later
2237 165 : if (!STARTS_WITH(poDS->fname.c_str(), "<MRF_META>"))
2238 : {
2239 : // Try opening it first, even though we still clobber it later
2240 165 : VSILFILE *mainfile = VSIFOpenL(poDS->fname.c_str(), "r+b");
2241 165 : if (!mainfile)
2242 : { // Then try creating it
2243 115 : mainfile = VSIFOpenL(poDS->fname.c_str(), "w+b");
2244 115 : if (!mainfile)
2245 : {
2246 3 : CPLError(CE_Failure, CPLE_OpenFailed,
2247 : "MRF: Can't open %s for writing", poDS->fname.c_str());
2248 3 : delete poDS;
2249 3 : return nullptr;
2250 : }
2251 : }
2252 162 : VSIFCloseL(mainfile);
2253 : }
2254 :
2255 : // Use the full, set some initial parameters
2256 162 : ILImage &img = poDS->full;
2257 162 : img.size = ILSize(nXSize, nYSize, 1, nBandsIn);
2258 : #ifdef HAVE_PNG
2259 162 : img.comp = IL_PNG;
2260 : #else
2261 : img.comp = IL_NONE;
2262 : #endif
2263 162 : img.order = (nBandsIn < 5) ? IL_Interleaved : IL_Separate;
2264 162 : img.pagesize = ILSize(512, 512, 1, 1);
2265 162 : img.quality = 85;
2266 162 : img.dt = eType;
2267 162 : img.dataoffset = 0;
2268 162 : img.idxoffset = 0;
2269 162 : img.hasNoData = false;
2270 162 : img.nbo = false;
2271 :
2272 : // Set the guard that tells us it needs saving before IO can take place
2273 162 : poDS->bCrystalized = FALSE;
2274 :
2275 : // Process the options, anything that an MRF might take
2276 :
2277 : try
2278 : {
2279 : // Adjust the dataset and the full image
2280 162 : poDS->ProcessCreateOptions(papszOptions);
2281 :
2282 : // Set default file names
2283 162 : if (img.datfname.empty())
2284 162 : img.datfname = getFname(poDS->GetFname(), ILComp_Ext[img.comp]);
2285 162 : if (img.idxfname.empty())
2286 162 : img.idxfname = getFname(poDS->GetFname(), ".idx");
2287 :
2288 162 : poDS->eAccess = GA_Update;
2289 : }
2290 :
2291 0 : catch (const CPLString &e)
2292 : {
2293 0 : CPLError(CE_Failure, CPLE_OpenFailed, "%s", e.c_str());
2294 0 : delete poDS;
2295 0 : return nullptr;
2296 : }
2297 :
2298 162 : poDS->current = poDS->full;
2299 162 : poDS->SetDescription(poDS->GetFname());
2300 :
2301 : // Build a MRF XML and initialize from it, this creates the bands
2302 162 : CPLXMLNode *config = poDS->BuildConfig();
2303 162 : err = poDS->Initialize(config);
2304 162 : CPLDestroyXMLNode(config);
2305 :
2306 162 : if (CPLE_None != err)
2307 : {
2308 30 : delete poDS;
2309 30 : return nullptr;
2310 : }
2311 :
2312 : // If not set by the band, get a pageSizeBytes buffer
2313 132 : if (poDS->GetPBufferSize() == 0 &&
2314 0 : !poDS->SetPBuffer(poDS->current.pageSizeBytes))
2315 : {
2316 0 : delete poDS;
2317 0 : return nullptr;
2318 : }
2319 :
2320 : // Tell PAM what our real file name is, to help it find the aux.xml
2321 132 : poDS->SetPhysicalFilename(poDS->GetFname());
2322 132 : return poDS;
2323 : }
2324 :
2325 172 : int MRFDataset::Crystalize()
2326 : {
2327 172 : if (bCrystalized || eAccess != GA_Update)
2328 : {
2329 0 : bCrystalized = TRUE;
2330 0 : return TRUE;
2331 : }
2332 :
2333 : // No need to write to disk if there is no filename. This is a
2334 : // memory only dataset.
2335 344 : if (strlen(GetDescription()) == 0 ||
2336 172 : EQUALN(GetDescription(), "<MRF_META>", 10))
2337 : {
2338 0 : bCrystalized = TRUE;
2339 0 : return TRUE;
2340 : }
2341 :
2342 172 : CPLXMLNode *config = BuildConfig();
2343 172 : if (!WriteConfig(config))
2344 20 : return FALSE;
2345 152 : CPLDestroyXMLNode(config);
2346 152 : if (!nocopy && (!IdxFP() || !DataFP()))
2347 0 : return FALSE;
2348 152 : bCrystalized = TRUE;
2349 152 : return TRUE;
2350 : }
2351 :
2352 : // Copy the first index at the end of the file and bump the version count
2353 1 : CPLErr MRFDataset::AddVersion()
2354 : {
2355 1 : VSILFILE *l_ifp = IdxFP();
2356 1 : void *tbuff = CPLMalloc(static_cast<size_t>(idxSize));
2357 1 : VSIFSeekL(l_ifp, 0, SEEK_SET);
2358 1 : VSIFReadL(tbuff, 1, static_cast<size_t>(idxSize), l_ifp);
2359 1 : verCount++; // The one we write
2360 1 : VSIFSeekL(l_ifp, static_cast<vsi_l_offset>(idxSize) * verCount,
2361 : SEEK_SET); // At the end, this can mess things up royally
2362 1 : VSIFWriteL(tbuff, 1, static_cast<size_t>(idxSize), l_ifp);
2363 1 : CPLFree(tbuff);
2364 1 : return CE_None;
2365 : }
2366 :
2367 : //
2368 : // Write a tile at the end of the data file
2369 : // If buff and size are zero, it is equivalent to erasing the tile
2370 : // If only size is zero, it is a special empty tile,
2371 : // when used for caching, offset should be 1
2372 : //
2373 : // To make it multi-processor safe, open the file in append mode
2374 : // and verify after write
2375 : //
2376 5096 : CPLErr MRFDataset::WriteTile(void *buff, GUIntBig infooffset, GUIntBig size)
2377 : {
2378 5096 : CPLErr ret = CE_None;
2379 5096 : ILIdx tinfo = {0, 0};
2380 :
2381 5096 : VSILFILE *l_dfp = DataFP();
2382 5096 : VSILFILE *l_ifp = IdxFP();
2383 :
2384 : // Verify buffer
2385 10192 : std::vector<GByte> tbuff;
2386 :
2387 5096 : if (l_ifp == nullptr || l_dfp == nullptr)
2388 0 : return CE_Failure;
2389 :
2390 : // Flag that versioned access requires a write even if empty
2391 5096 : int new_tile = false;
2392 : // If it has versions, might need to start a new one
2393 5096 : if (hasVersions)
2394 : {
2395 1 : int new_version = false; // Assume no need to build new version
2396 :
2397 : // Read the current tile info
2398 1 : VSIFSeekL(l_ifp, infooffset, SEEK_SET);
2399 1 : VSIFReadL(&tinfo, 1, sizeof(ILIdx), l_ifp);
2400 :
2401 1 : if (verCount == 0)
2402 1 : new_version = true; // No previous yet, might create a new version
2403 : else
2404 : { // We need at least two versions before we can test for changes
2405 0 : ILIdx prevtinfo = {0, 0};
2406 :
2407 : // Read the previous one
2408 0 : VSIFSeekL(l_ifp, infooffset + verCount * idxSize, SEEK_SET);
2409 0 : VSIFReadL(&prevtinfo, 1, sizeof(ILIdx), l_ifp);
2410 :
2411 : // current and previous tiles are different, might create version
2412 0 : if (tinfo.size != prevtinfo.size ||
2413 0 : tinfo.offset != prevtinfo.offset)
2414 0 : new_version = true;
2415 : }
2416 :
2417 : // tinfo contains the current info or 0,0
2418 1 : if (tinfo.size == GIntBig(net64(size)))
2419 : { // Might be identical
2420 0 : if (size != 0)
2421 : {
2422 : // Use the temporary buffer
2423 0 : tbuff.resize(static_cast<size_t>(size));
2424 0 : VSIFSeekL(l_dfp, infooffset, SEEK_SET);
2425 0 : VSIFReadL(tbuff.data(), 1, tbuff.size(), l_dfp);
2426 : // Need to write it if not the same
2427 0 : new_tile = !std::equal(tbuff.begin(), tbuff.end(),
2428 : static_cast<GByte *>(buff));
2429 0 : tbuff.clear();
2430 : }
2431 : else
2432 : {
2433 : // Writing a null tile on top of a null tile, does it count?
2434 0 : if (tinfo.offset != GIntBig(net64(GUIntBig(buff))))
2435 0 : new_tile = true;
2436 : }
2437 : }
2438 : else
2439 : {
2440 1 : new_tile = true; // Need to write it because it is different
2441 1 : if (verCount == 0 && tinfo.size == 0)
2442 0 : new_version = false; // Don't create a version if current is
2443 : // empty and there is no previous
2444 : }
2445 :
2446 1 : if (!new_tile)
2447 0 : return CE_None; // No reason to write
2448 :
2449 : // Do we need to start a new version before writing the tile?
2450 1 : if (new_version)
2451 1 : AddVersion();
2452 : }
2453 :
2454 5096 : bool same = true;
2455 5096 : if (size)
2456 0 : do
2457 : {
2458 : // start of critical MP section
2459 5085 : VSIFSeekL(l_dfp, 0, SEEK_END);
2460 5085 : GUIntBig offset = VSIFTellL(l_dfp) + spacing;
2461 :
2462 : // Spacing should be 0 in MP safe mode, this doesn't have much of
2463 : // effect Use the existing data, spacing content is not guaranteed
2464 5085 : for (GUIntBig pending = spacing; pending != 0;
2465 0 : pending -= std::min(pending, size))
2466 0 : VSIFWriteL(buff, 1,
2467 0 : static_cast<size_t>(std::min(pending, size)),
2468 : l_dfp); // Usually only once
2469 :
2470 5085 : if (static_cast<size_t>(size) !=
2471 5085 : VSIFWriteL(buff, 1, static_cast<size_t>(size), l_dfp))
2472 0 : ret = CE_Failure;
2473 : // End of critical section
2474 :
2475 5085 : tinfo.offset = net64(offset);
2476 : //
2477 : // For MP ops, check that we can read the same content, otherwise
2478 : // try again This makes the caching MRF MP safe on file systems that
2479 : // implement append mode fully, without using explicit locks
2480 : //
2481 5085 : if (CE_None == ret && mp_safe)
2482 : { // readback and check
2483 3 : if (tbuff.size() < size)
2484 3 : tbuff.resize(static_cast<size_t>(size));
2485 3 : VSIFSeekL(l_dfp, offset, SEEK_SET);
2486 3 : VSIFReadL(tbuff.data(), 1, tbuff.size(), l_dfp);
2487 3 : same = std::equal(tbuff.begin(), tbuff.end(),
2488 : static_cast<GByte *>(buff));
2489 : }
2490 5085 : } while (CE_None == ret && mp_safe && !same);
2491 :
2492 5096 : if (CE_None != ret)
2493 : {
2494 0 : CPLError(CE_Failure, CPLE_AppDefined, "MRF: Tile write failed");
2495 0 : return ret;
2496 : }
2497 :
2498 : // Convert index to net format, offset is set already
2499 5096 : tinfo.size = net64(size);
2500 : // Do nothing if the tile is empty and the file record is also empty
2501 5096 : if (!new_tile && 0 == size && nullptr == buff)
2502 : {
2503 10 : VSIFSeekL(l_ifp, infooffset, SEEK_SET);
2504 10 : VSIFReadL(&tinfo, 1, sizeof(ILIdx), l_ifp);
2505 10 : if (0 == tinfo.offset && 0 == tinfo.size)
2506 10 : return ret;
2507 : }
2508 :
2509 : // Special case, any non-zero offset will do
2510 5086 : if (nullptr != buff && 0 == size)
2511 0 : tinfo.offset = ~GUIntBig(0);
2512 :
2513 5086 : VSIFSeekL(l_ifp, infooffset, SEEK_SET);
2514 5086 : if (sizeof(tinfo) != VSIFWriteL(&tinfo, 1, sizeof(tinfo), l_ifp))
2515 : {
2516 0 : CPLError(CE_Failure, CPLE_AppDefined, "MRF: Index write failed");
2517 0 : ret = CE_Failure;
2518 : }
2519 :
2520 5086 : return ret;
2521 : }
2522 :
2523 128 : CPLErr MRFDataset::SetGeoTransform(const GDALGeoTransform >)
2524 : {
2525 128 : if (GetAccess() != GA_Update || bCrystalized)
2526 : {
2527 0 : CPLError(CE_Failure, CPLE_NotSupported,
2528 : "SetGeoTransform only works during Create call");
2529 0 : return CE_Failure;
2530 : }
2531 128 : m_gt = gt;
2532 128 : bGeoTransformValid = TRUE;
2533 128 : return CE_None;
2534 : }
2535 :
2536 18 : bool MRFDataset::IsSingleTile()
2537 : {
2538 18 : if (current.pagecount.l != 1 || !source.empty() || nullptr == DataFP())
2539 4 : return FALSE;
2540 14 : return 0 == cpl::down_cast<MRFRasterBand *>(GetRasterBand(1))
2541 14 : ->GetOverviewCount();
2542 : }
2543 :
2544 : /*
2545 : * Returns 0,1,0,0,0,1 even if it was not set
2546 : */
2547 455 : CPLErr MRFDataset::GetGeoTransform(GDALGeoTransform >) const
2548 : {
2549 455 : gt = m_gt;
2550 455 : MRFDataset *nonConstThis = const_cast<MRFDataset *>(this);
2551 455 : if (nonConstThis->GetMetadata("RPC") || nonConstThis->GetGCPCount())
2552 0 : bGeoTransformValid = FALSE;
2553 455 : if (!bGeoTransformValid)
2554 0 : return CE_Failure;
2555 455 : return CE_None;
2556 : }
2557 :
2558 : /**
2559 : *\brief Read a tile index
2560 : *
2561 : * It handles the non-existent index case, for no compression
2562 : * The bias is non-zero only when the cloned index is read
2563 : */
2564 :
2565 2021 : CPLErr MRFDataset::ReadTileIdx(ILIdx &tinfo, const ILSize &pos,
2566 : const ILImage &img, const GIntBig bias)
2567 : {
2568 2021 : VSILFILE *l_ifp = IdxFP();
2569 :
2570 : // Initialize the tinfo structure, in case the files are missing
2571 2021 : if (missing)
2572 0 : return CE_None;
2573 :
2574 2021 : GIntBig offset = bias + IdxOffset(pos, img);
2575 2021 : if (l_ifp == nullptr && img.comp == IL_NONE)
2576 : {
2577 0 : tinfo.size = current.pageSizeBytes;
2578 0 : tinfo.offset = offset * tinfo.size;
2579 0 : return CE_None;
2580 : }
2581 :
2582 2021 : if (l_ifp == nullptr && IsSingleTile())
2583 : {
2584 7 : tinfo.offset = 0;
2585 7 : VSILFILE *l_dfp = DataFP(); // IsSingleTile() checks that fp is valid
2586 7 : VSIFSeekL(l_dfp, 0, SEEK_END);
2587 7 : tinfo.size = VSIFTellL(l_dfp);
2588 :
2589 : // It should be less than the pagebuffer
2590 7 : tinfo.size = std::min(tinfo.size, static_cast<GIntBig>(pbsize));
2591 7 : return CE_None;
2592 : }
2593 :
2594 2014 : if (l_ifp == nullptr)
2595 : {
2596 0 : CPLError(CE_Failure, CPLE_FileIO, "Can't open index file");
2597 0 : return CE_Failure;
2598 : }
2599 :
2600 2014 : VSIFSeekL(l_ifp, static_cast<vsi_l_offset>(offset), SEEK_SET);
2601 2014 : if (1 != VSIFReadL(&tinfo, sizeof(ILIdx), 1, l_ifp))
2602 0 : return CE_Failure;
2603 : // Convert them to native form
2604 2014 : tinfo.offset = net64(tinfo.offset);
2605 2014 : tinfo.size = net64(tinfo.size);
2606 :
2607 2014 : if (0 == bias || 0 != tinfo.size || 0 != tinfo.offset)
2608 2013 : return CE_None;
2609 :
2610 : // zero size and zero offset in sourced index means that this portion is
2611 : // un-initialized
2612 :
2613 : // Should be cloned and the offset within the cloned index
2614 1 : offset -= bias;
2615 1 : assert(offset < bias);
2616 1 : assert(clonedSource);
2617 :
2618 : // Read this block from the remote index, prepare it and store it in the
2619 : // right place The block size in bytes, should be a multiple of 16, to have
2620 : // full index entries
2621 1 : const int CPYSZ = 32768;
2622 : // Adjust offset to the start of the block
2623 1 : offset = (offset / CPYSZ) * CPYSZ;
2624 1 : GIntBig size = std::min(size_t(CPYSZ), size_t(bias - offset));
2625 1 : size /= sizeof(ILIdx); // In records
2626 2 : vector<ILIdx> buf(static_cast<size_t>(size));
2627 1 : ILIdx *buffer = &buf[0]; // Buffer to copy the source to the clone index
2628 :
2629 : // Fetch the data from the cloned index
2630 1 : MRFDataset *pSrc = static_cast<MRFDataset *>(GetSrcDS());
2631 1 : if (nullptr == pSrc)
2632 : {
2633 0 : CPLError(CE_Failure, CPLE_FileIO, "Can't open cloned source index");
2634 0 : return CE_Failure; // Source reported the error
2635 : }
2636 :
2637 1 : VSILFILE *srcidx = pSrc->IdxFP();
2638 1 : if (nullptr == srcidx)
2639 : {
2640 0 : CPLError(CE_Failure, CPLE_FileIO, "Can't open cloned source index");
2641 0 : return CE_Failure; // Source reported the error
2642 : }
2643 :
2644 1 : VSIFSeekL(srcidx, static_cast<vsi_l_offset>(offset), SEEK_SET);
2645 1 : size = VSIFReadL(buffer, sizeof(ILIdx), static_cast<size_t>(size), srcidx);
2646 1 : if (size != GIntBig(buf.size()))
2647 : {
2648 0 : CPLError(CE_Failure, CPLE_FileIO, "Can't read cloned source index");
2649 0 : return CE_Failure; // Source reported the error
2650 : }
2651 :
2652 : // Mark the empty records as checked, by making the offset non-zero
2653 2 : for (vector<ILIdx>::iterator it = buf.begin(); it != buf.end(); ++it)
2654 : {
2655 1 : if (it->offset == 0 && it->size == 0)
2656 0 : it->offset = net64(1);
2657 : }
2658 :
2659 : // Write it in the right place in the local index file
2660 1 : VSIFSeekL(l_ifp, static_cast<vsi_l_offset>(bias + offset), SEEK_SET);
2661 1 : size = VSIFWriteL(&buf[0], sizeof(ILIdx), static_cast<size_t>(size), l_ifp);
2662 1 : if (size != GIntBig(buf.size()))
2663 : {
2664 0 : CPLError(CE_Failure, CPLE_FileIO, "Can't write to cloning MRF index");
2665 0 : return CE_Failure; // Source reported the error
2666 : }
2667 :
2668 : // Cloned index updated, restart this function, it will work now
2669 1 : return ReadTileIdx(tinfo, pos, img, bias);
2670 : }
2671 :
2672 : NAMESPACE_MRF_END
|
