Line data Source code
1 : /*
2 : * Copyright (c) 2002-2012, California Institute of Technology.
3 : * All rights reserved. Based on Government Sponsored Research under contracts
4 : * NAS7-1407 and/or NAS7-03001.
5 : *
6 : * Redistribution and use in source and binary forms, with or without
7 : * modification, are permitted provided that the following conditions are met:
8 : * 1. Redistributions of source code must retain the above copyright notice,
9 : * this list of conditions and the following disclaimer.
10 : * 2. Redistributions in binary form must reproduce the above copyright
11 : * notice, this list of conditions and the following disclaimer in the
12 : * documentation and/or other materials provided with the distribution.
13 : * 3. Neither the name of the California Institute of Technology (Caltech),
14 : * its operating division the Jet Propulsion Laboratory (JPL), the National
15 : * Aeronautics and Space Administration (NASA), nor the names of its
16 : * contributors may be used to endorse or promote products derived from this
17 : * software without specific prior written permission.
18 : *
19 : * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20 : * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 : * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 : * ARE DISCLAIMED. IN NO EVENT SHALL THE CALIFORNIA INSTITUTE OF TECHNOLOGY BE
23 : * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 : * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 : * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 : * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 : * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 : * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 : * POSSIBILITY OF SUCH DAMAGE.
30 : *
31 : * Portions copyright 2014-2021 Esri
32 : *
33 : * Licensed under the Apache License, Version 2.0 (the "License");
34 : * you may not use this file except in compliance with the License.
35 : * You may obtain a copy of the License at
36 : *
37 : * http://www.apache.org/licenses/LICENSE-2.0
38 : *
39 : * Unless required by applicable law or agreed to in writing, software
40 : * distributed under the License is distributed on an "AS IS" BASIS,
41 : * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
42 : * See the License for the specific language governing permissions and
43 : * limitations under the License.
44 : */
45 :
46 : /******************************************************************************
47 : *
48 : * Project: Meta Raster File Format Driver Implementation, Dataset
49 : * Purpose: Implementation of GDAL dataset
50 : *
51 : * Author: Lucian Plesea, Lucian.Plesea jpl.nasa.gov, lplesea esri.com
52 : *
53 : ******************************************************************************
54 : *
55 : * The MRF dataset and the band are closely tied together, they should be
56 : * considered a single class, or a class (dataset) with extensions (bands).
57 : *
58 : *
59 : ****************************************************************************/
60 :
61 : #include "marfa.h"
62 : #include "mrfdrivercore.h"
63 : #include "cpl_multiproc.h" /* for CPLSleep() */
64 : #include "gdal_priv.h"
65 : #include <assert.h>
66 :
67 : #include <algorithm>
68 : #include <limits>
69 : #include <vector>
70 : using std::string;
71 : using std::vector;
72 :
73 : NAMESPACE_MRF_START
74 :
75 : // Initialize as invalid
76 369 : MRFDataset::MRFDataset()
77 : : zslice(0), idxSize(0), clonedSource(FALSE), nocopy(FALSE),
78 : bypass_cache(
79 369 : !CPLTestBool(CPLGetConfigOption("MRF_ENABLE_CACHING", "FALSE"))),
80 : mp_safe(FALSE), hasVersions(FALSE), verCount(0),
81 : bCrystalized(TRUE), // Assume not in create mode
82 : spacing(0), no_errors(0), missing(0), poSrcDS(nullptr), level(-1),
83 : cds(nullptr), scale(0.0), pbuffer(nullptr), pbsize(0), tile(ILSize()),
84 : bdirty(0), bGeoTransformValid(TRUE), poColorTable(nullptr), Quality(0),
85 738 : pzscctx(nullptr), pzsdctx(nullptr), read_timer(), write_timer(0)
86 : {
87 369 : m_oSRS.SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER);
88 369 : ifp.FP = dfp.FP = nullptr;
89 369 : dfp.acc = GF_Read;
90 369 : ifp.acc = GF_Read;
91 369 : }
92 :
93 181 : bool MRFDataset::SetPBuffer(unsigned int sz)
94 : {
95 181 : if (sz == 0)
96 : {
97 0 : CPLFree(pbuffer);
98 0 : pbuffer = nullptr;
99 : }
100 181 : void *pbufferNew = VSIRealloc(pbuffer, sz);
101 181 : if (pbufferNew == nullptr)
102 : {
103 0 : CPLError(CE_Failure, CPLE_OutOfMemory, "Cannot allocate %u bytes", sz);
104 0 : return false;
105 : }
106 181 : pbuffer = pbufferNew;
107 181 : pbsize = sz;
108 181 : return true;
109 : }
110 :
111 256 : static GDALColorEntry GetXMLColorEntry(CPLXMLNode *p)
112 : {
113 : GDALColorEntry ce;
114 256 : ce.c1 = static_cast<short>(getXMLNum(p, "c1", 0));
115 256 : ce.c2 = static_cast<short>(getXMLNum(p, "c2", 0));
116 256 : ce.c3 = static_cast<short>(getXMLNum(p, "c3", 0));
117 256 : ce.c4 = static_cast<short>(getXMLNum(p, "c4", 255));
118 256 : return ce;
119 : }
120 :
121 : //
122 : // Called by dataset destructor or at GDAL termination, to avoid
123 : // closing datasets whose drivers have already been unloaded
124 : //
125 369 : int MRFDataset::CloseDependentDatasets()
126 : {
127 369 : int bHasDroppedRef = GDALPamDataset::CloseDependentDatasets();
128 :
129 369 : if (poSrcDS)
130 : {
131 3 : bHasDroppedRef = TRUE;
132 3 : GDALClose(GDALDataset::ToHandle(poSrcDS));
133 3 : poSrcDS = nullptr;
134 : }
135 :
136 369 : if (cds)
137 : {
138 2 : bHasDroppedRef = TRUE;
139 2 : GDALClose(GDALDataset::ToHandle(cds));
140 2 : cds = nullptr;
141 : }
142 :
143 369 : return bHasDroppedRef;
144 : }
145 :
146 738 : MRFDataset::~MRFDataset()
147 : { // Make sure everything gets written
148 369 : if (eAccess != GA_ReadOnly && !bCrystalized)
149 50 : if (!MRFDataset::Crystalize())
150 : {
151 : // Can't return error code from a destructor, just emit the error
152 10 : CPLError(CE_Failure, CPLE_FileIO, "Error creating files");
153 : }
154 :
155 369 : MRFDataset::FlushCache(true);
156 369 : MRFDataset::CloseDependentDatasets();
157 :
158 369 : if (ifp.FP)
159 263 : VSIFCloseL(ifp.FP);
160 369 : if (dfp.FP)
161 264 : VSIFCloseL(dfp.FP);
162 :
163 369 : delete poColorTable;
164 :
165 : // CPLFree ignores being called with NULL
166 369 : CPLFree(pbuffer);
167 369 : pbsize = 0;
168 : #if defined(ZSTD_SUPPORT)
169 369 : ZSTD_freeCCtx(static_cast<ZSTD_CCtx *>(pzscctx));
170 369 : ZSTD_freeDCtx(static_cast<ZSTD_DCtx *>(pzsdctx));
171 : #endif
172 : // total time spend doing compression and decompression
173 369 : if (0 != write_timer.count())
174 112 : CPLDebug("MRF_Timing", "Compression took %fms",
175 112 : 1e-6 * write_timer.count());
176 :
177 369 : if (0 != read_timer.count())
178 140 : CPLDebug("MRF_Timing", "Decompression took %fms",
179 140 : 1e-6 * read_timer.count());
180 738 : }
181 :
182 : /*
183 : *\brief Format specific RasterIO, may be bypassed by BlockBasedRasterIO by
184 : *setting GDAL_FORCE_CACHING to Yes, in which case the band ReadBlock and
185 : *WriteBLock are called directly
186 : */
187 140 : CPLErr MRFDataset::IRasterIO(GDALRWFlag eRWFlag, int nXOff, int nYOff,
188 : int nXSize, int nYSize, void *pData, int nBufXSize,
189 : int nBufYSize, GDALDataType eBufType,
190 : int nBandCount, BANDMAP_TYPE panBandMap,
191 : GSpacing nPixelSpace, GSpacing nLineSpace,
192 : GSpacing nBandSpace,
193 : GDALRasterIOExtraArg *psExtraArgs)
194 : {
195 140 : CPLDebug("MRF_IO",
196 : "IRasterIO %s, %d, %d, %d, %d, bufsz %d,%d,%d strides P %d, L %d, "
197 : "B %d \n",
198 : eRWFlag == GF_Write ? "Write" : "Read", nXOff, nYOff, nXSize,
199 : nYSize, nBufXSize, nBufYSize, nBandCount,
200 : static_cast<int>(nPixelSpace), static_cast<int>(nLineSpace),
201 : static_cast<int>(nBandSpace));
202 :
203 140 : if (eRWFlag == GF_Write && !bCrystalized && !Crystalize())
204 : {
205 0 : CPLError(CE_Failure, CPLE_FileIO, "MRF: Error creating files");
206 0 : return CE_Failure;
207 : }
208 :
209 : //
210 : // Call the parent implementation, which splits it into bands and calls
211 : // their IRasterIO
212 : //
213 140 : return GDALPamDataset::IRasterIO(eRWFlag, nXOff, nYOff, nXSize, nYSize,
214 : pData, nBufXSize, nBufYSize, eBufType,
215 : nBandCount, panBandMap, nPixelSpace,
216 140 : nLineSpace, nBandSpace, psExtraArgs);
217 : }
218 :
219 : /**
220 : *\brief Build some overviews
221 : *
222 : * if nOverviews is 0, erase the overviews (reduce to base image only)
223 : */
224 :
225 31 : CPLErr MRFDataset::IBuildOverviews(const char *pszResampling, int nOverviews,
226 : const int *panOverviewList, int nBandsIn,
227 : const int *panBandList,
228 : GDALProgressFunc pfnProgress,
229 : void *pProgressData,
230 : CSLConstList papszOptions)
231 :
232 : {
233 31 : if (pfnProgress == nullptr)
234 0 : pfnProgress = GDALDummyProgress;
235 :
236 31 : CPLErr eErr = CE_None;
237 31 : CPLDebug("MRF_OVERLAY", "IBuildOverviews %d, bands %d\n", nOverviews,
238 : nBandsIn);
239 :
240 31 : if (nBands != nBandsIn)
241 : {
242 0 : CPLError(CE_Failure, CPLE_NotSupported, "nBands = %d not supported",
243 : nBandsIn);
244 0 : return CE_Failure;
245 : }
246 :
247 : // If we don't have write access, then create external overviews
248 31 : if (GetAccess() != GA_Update)
249 : {
250 1 : CPLDebug("MRF", "File open read-only, creating overviews externally.");
251 1 : return GDALDataset::IBuildOverviews(
252 : pszResampling, nOverviews, panOverviewList, nBands, panBandList,
253 1 : pfnProgress, pProgressData, papszOptions);
254 : }
255 :
256 30 : if (nOverviews == 0)
257 : {
258 : // If there are none, nothing to do
259 0 : if (GetRasterBand(1)->GetOverviewCount() == 0)
260 0 : return CE_None;
261 :
262 0 : auto *b = static_cast<MRFRasterBand *>(GetRasterBand(1));
263 : // If the first band internal overviews don't exist, they are external
264 0 : if (b->overviews.empty())
265 0 : return GDALDataset::IBuildOverviews(
266 : pszResampling, nOverviews, panOverviewList, nBands, panBandList,
267 0 : pfnProgress, pProgressData, papszOptions);
268 :
269 : // We should clean overviews, but this is not allowed in an MRF
270 0 : CPLError(CE_Warning, CPLE_NotSupported,
271 : "MRF: Internal overviews cannot be removed, "
272 : "but they can be rebuilt");
273 0 : return CE_None;
274 : }
275 :
276 30 : std::vector<int> panOverviewListNew(nOverviews);
277 61 : for (int i = 0; i < nOverviews; i++)
278 31 : panOverviewListNew[i] = panOverviewList[i];
279 : try
280 : { // Throw an error code, to make sure memory gets freed properly
281 : // Modify the metadata file if it doesn't already have the Rset model
282 : // set
283 30 : if (0.0 == scale)
284 : {
285 30 : CPLXMLNode *config = ReadConfig();
286 : try
287 : {
288 : const char *model =
289 30 : CPLGetXMLValue(config, "Rsets.model", "uniform");
290 30 : if (!EQUAL(model, "uniform"))
291 : {
292 0 : CPLError(CE_Failure, CPLE_AppDefined,
293 : "MRF:IBuildOverviews, Overviews not implemented "
294 : "for model %s",
295 : model);
296 0 : throw CE_Failure;
297 : }
298 :
299 : // The scale value is the same as first overview
300 30 : scale =
301 30 : strtod(CPLGetXMLValue(
302 : config, "Rsets.scale",
303 60 : CPLOPrintf("%d", panOverviewList[0]).c_str()),
304 : nullptr);
305 30 : if (scale == 0.0)
306 : {
307 0 : CPLError(CE_Failure, CPLE_IllegalArg,
308 : "Invalid Rsets.scale value");
309 0 : throw CE_Failure;
310 : }
311 :
312 30 : if (static_cast<int>(scale) != 2 &&
313 0 : (EQUALN("Avg", pszResampling, 3) ||
314 0 : EQUALN("Nnb", pszResampling, 3)))
315 : {
316 0 : CPLError(CE_Failure, CPLE_IllegalArg,
317 : "MRF internal resampling requires a scale factor "
318 : "of two");
319 0 : throw CE_Failure;
320 : }
321 :
322 : // Initialize the empty overlays, all of them for a given scale
323 : // They could already exist, in which case they are not erased
324 30 : idxSize = AddOverviews(int(scale));
325 :
326 : // If we don't have overviews, don't try to generate them
327 30 : if (GetRasterBand(1)->GetOverviewCount() == 0)
328 0 : throw CE_None;
329 :
330 30 : if (!CheckFileSize(current.idxfname, idxSize, GA_Update))
331 : {
332 0 : CPLError(CE_Failure, CPLE_AppDefined,
333 : "MRF: Can't extend index file");
334 0 : throw CE_Failure;
335 : }
336 :
337 : // Set the uniform node, in case it was not set before, and
338 : // save the new configuration
339 30 : CPLSetXMLValue(config, "Rsets.#model", "uniform");
340 30 : CPLSetXMLValue(config, "Rsets.#scale", PrintDouble(scale));
341 :
342 30 : if (!WriteConfig(config))
343 : {
344 0 : CPLError(CE_Failure, CPLE_AppDefined,
345 : "MRF: Can't rewrite the metadata file");
346 0 : throw CE_Failure;
347 : }
348 30 : CPLDestroyXMLNode(config);
349 30 : config = nullptr;
350 : }
351 0 : catch (const CPLErr &)
352 : {
353 0 : CPLDestroyXMLNode(config);
354 0 : throw; // Rethrow
355 : }
356 :
357 : // To avoid issues with blacks overviews, generate all of them
358 : // if the user asked for a couple of overviews in the correct
359 : // sequence and starting with the lowest one
360 90 : if (!EQUAL(pszResampling, "NONE") &&
361 32 : nOverviews != GetRasterBand(1)->GetOverviewCount() &&
362 2 : CPLTestBool(
363 : CPLGetConfigOption("MRF_ALL_OVERVIEW_LEVELS", "YES")))
364 : {
365 2 : bool bIncreasingPowers =
366 2 : (panOverviewList[0] == static_cast<int>(scale));
367 3 : for (int i = 1; i < nOverviews; i++)
368 1 : bIncreasingPowers =
369 2 : bIncreasingPowers &&
370 1 : (panOverviewList[i] ==
371 1 : static_cast<int>(scale * panOverviewList[i - 1]));
372 :
373 2 : int ovrcount = GetRasterBand(1)->GetOverviewCount();
374 2 : if (bIncreasingPowers && nOverviews != ovrcount)
375 : {
376 2 : CPLDebug("MRF",
377 : "Generating %d levels instead of the %d requested",
378 : ovrcount, nOverviews);
379 2 : nOverviews = ovrcount;
380 2 : panOverviewListNew.resize(ovrcount);
381 7 : for (int i = 0; i < ovrcount; i++)
382 5 : panOverviewListNew[i] = int(std::pow(scale, i + 1));
383 : }
384 : }
385 : }
386 :
387 30 : if (static_cast<int>(scale) != 2 && (EQUALN("Avg", pszResampling, 3) ||
388 0 : EQUALN("Nnb", pszResampling, 3)))
389 : {
390 0 : CPLError(CE_Failure, CPLE_IllegalArg,
391 : "MRF internal resampling requires a scale factor of two");
392 0 : throw CE_Failure;
393 : }
394 :
395 : // First pass, count the pixels to be processed, mark invalid levels
396 : // Smallest positive value to avoid Coverity Scan complaining about
397 : // division by zero
398 30 : double dfTotalPixels = std::numeric_limits<double>::min();
399 30 : double dfPixels = double(nRasterXSize) * nRasterYSize;
400 63 : for (int i = 0; i < nOverviews; i++)
401 : {
402 : // Verify that scales are reasonable, val/scale has to be an integer
403 33 : if (!IsPower(panOverviewListNew[i], scale))
404 : {
405 0 : CPLError(CE_Warning, CPLE_AppDefined,
406 : "MRF:IBuildOverviews, skipping overview factor %d,"
407 : " it is not a power of %f",
408 0 : panOverviewListNew[i], scale);
409 0 : panOverviewListNew[i] = -1;
410 0 : continue;
411 : };
412 :
413 33 : int srclevel = int(logbase(panOverviewListNew[i], scale) - 0.5);
414 33 : MRFRasterBand *b = static_cast<MRFRasterBand *>(GetRasterBand(1));
415 :
416 : // Warn for requests for invalid levels
417 33 : if (srclevel >= b->GetOverviewCount())
418 : {
419 0 : CPLError(CE_Warning, CPLE_AppDefined,
420 : "MRF:IBuildOverviews, overview factor %d is not valid "
421 : "for this dataset",
422 0 : panOverviewListNew[i]);
423 0 : panOverviewListNew[i] = -1;
424 0 : continue;
425 : }
426 33 : dfTotalPixels += dfPixels * std::pow(scale, -srclevel);
427 : }
428 :
429 30 : dfPixels = 0;
430 63 : for (int i = 0; i < nOverviews; i++)
431 : {
432 : // Skip the invalid levels
433 33 : if (panOverviewListNew[i] < 0)
434 0 : continue;
435 :
436 33 : int srclevel = int(logbase(panOverviewListNew[i], scale) - 0.5);
437 : auto dfLevelPixels =
438 33 : std::pow(scale, -srclevel) * nRasterXSize * nRasterYSize;
439 : // Use "avg" flag to trigger the internal average sampling
440 33 : if (EQUALN("Avg", pszResampling, 3) ||
441 19 : EQUALN("Nnb", pszResampling, 3))
442 : {
443 26 : int sampling = EQUALN("Avg", pszResampling, 3) ? SAMPLING_Avg
444 : : SAMPLING_Near;
445 26 : auto b = static_cast<MRFRasterBand *>(GetRasterBand(1));
446 : // Internal, using PatchOverview
447 26 : if (srclevel > 0)
448 : b = static_cast<MRFRasterBand *>(
449 3 : b->GetOverview(srclevel - 1));
450 :
451 : eErr =
452 26 : PatchOverview(0, 0, b->nBlocksPerRow, b->nBlocksPerColumn,
453 : srclevel, 0, sampling);
454 26 : if (eErr == CE_Failure)
455 26 : throw eErr;
456 : }
457 : else
458 : {
459 : // Use the GDAL method
460 14 : std::vector<GDALRasterBand *> apoSrcBandList(nBands);
461 : std::vector<std::vector<GDALRasterBand *>> aapoOverviewBandList(
462 14 : nBands);
463 14 : for (int iBand = 0; iBand < nBands; iBand++)
464 : {
465 : // This is the base level
466 7 : apoSrcBandList[iBand] = GetRasterBand(panBandList[iBand]);
467 : // Set up the destination
468 14 : aapoOverviewBandList[iBand] = std::vector<GDALRasterBand *>{
469 14 : apoSrcBandList[iBand]->GetOverview(srclevel)};
470 : // Use the previous level as the source
471 7 : if (srclevel > 0)
472 0 : apoSrcBandList[iBand] =
473 0 : apoSrcBandList[iBand]->GetOverview(srclevel - 1);
474 : }
475 :
476 14 : auto pScaledProgress = GDALCreateScaledProgress(
477 : dfPixels / dfTotalPixels,
478 7 : (dfPixels + dfLevelPixels) / dfTotalPixels, pfnProgress,
479 : pProgressData);
480 7 : eErr = GDALRegenerateOverviewsMultiBand(
481 : apoSrcBandList, aapoOverviewBandList, pszResampling,
482 : GDALScaledProgress, pScaledProgress, papszOptions);
483 7 : GDALDestroyScaledProgress(pScaledProgress);
484 : }
485 33 : dfPixels += dfLevelPixels;
486 33 : pfnProgress(dfPixels / dfTotalPixels, "", pProgressData);
487 33 : if (eErr == CE_Failure)
488 0 : throw eErr;
489 : }
490 :
491 30 : pfnProgress(1.0, "", pProgressData);
492 : }
493 0 : catch (const CPLErr &e)
494 : {
495 0 : eErr = e;
496 : }
497 :
498 30 : return eErr;
499 : }
500 :
501 : /*
502 : *\brief blank separated list to vector of doubles
503 : */
504 33 : static void list2vec(std::vector<double> &v, const char *pszList)
505 : {
506 33 : if ((pszList == nullptr) || (pszList[0] == 0))
507 0 : return;
508 33 : char **papszTokens = CSLTokenizeString2(
509 : pszList, " \t\n\r", CSLT_STRIPLEADSPACES | CSLT_STRIPENDSPACES);
510 33 : v.clear();
511 66 : for (int i = 0; i < CSLCount(papszTokens); i++)
512 33 : v.push_back(CPLStrtod(papszTokens[i], nullptr));
513 33 : CSLDestroy(papszTokens);
514 : }
515 :
516 19 : void MRFDataset::SetNoDataValue(const char *pszVal)
517 : {
518 19 : list2vec(vNoData, pszVal);
519 19 : }
520 :
521 7 : void MRFDataset::SetMinValue(const char *pszVal)
522 : {
523 7 : list2vec(vMin, pszVal);
524 7 : }
525 :
526 7 : void MRFDataset::SetMaxValue(const char *pszVal)
527 : {
528 7 : list2vec(vMax, pszVal);
529 7 : }
530 :
531 : /**
532 : *
533 : *\brief Read the XML config tree, from file
534 : * Caller is responsible for freeing the memory
535 : *
536 : * @return NULL on failure, or the document tree on success.
537 : *
538 : */
539 30 : CPLXMLNode *MRFDataset::ReadConfig() const
540 : {
541 30 : if (fname[0] == '<')
542 0 : return CPLParseXMLString(fname);
543 30 : return CPLParseXMLFile(fname);
544 : }
545 :
546 : /**
547 : *\brief Write the XML config tree
548 : * Caller is responsible for correctness of data
549 : * and for freeing the memory
550 : *
551 : * @param config The document tree to write
552 : * @return TRUE on success, FALSE otherwise
553 : */
554 202 : int MRFDataset::WriteConfig(CPLXMLNode *config)
555 : {
556 202 : if (fname[0] == '<')
557 0 : return FALSE;
558 202 : return CPLSerializeXMLTreeToFile(config, fname);
559 : }
560 :
561 : static void
562 5 : stringSplit(vector<string> &theStringVector, // Altered/returned value
563 : const string &theString, size_t start = 0,
564 : const char theDelimiter = ' ')
565 : {
566 : while (true)
567 : {
568 5 : size_t end = theString.find(theDelimiter, start);
569 5 : if (string::npos == end)
570 : {
571 5 : theStringVector.push_back(theString.substr(start));
572 5 : return;
573 : }
574 0 : theStringVector.push_back(theString.substr(start, end - start));
575 0 : start = end + 1;
576 0 : }
577 : }
578 :
579 : // Returns the number following the prefix if it exists in one of the vector
580 : // strings Otherwise it returns the default
581 15 : static int getnum(const vector<string> &theStringVector, const char prefix,
582 : int def)
583 : {
584 25 : for (unsigned int i = 0; i < theStringVector.size(); i++)
585 15 : if (theStringVector[i][0] == prefix)
586 5 : return atoi(theStringVector[i].c_str() + 1);
587 10 : return def;
588 : }
589 :
590 : /**
591 : *\brief Open a MRF file
592 : *
593 : */
594 202 : GDALDataset *MRFDataset::Open(GDALOpenInfo *poOpenInfo)
595 : {
596 202 : if (!MRFDriverIdentify(poOpenInfo))
597 0 : return nullptr;
598 :
599 202 : CPLXMLNode *config = nullptr;
600 202 : CPLErr ret = CE_None;
601 202 : const char *pszFileName = poOpenInfo->pszFilename;
602 :
603 202 : int level = -1; // All levels
604 202 : int version = 0; // Current
605 202 : int zslice = 0;
606 404 : string fn; // Used to parse and adjust the file name
607 404 : string insidefn; // inside tar file name
608 :
609 : // Different ways to open an MRF
610 202 : if (poOpenInfo->nHeaderBytes >= 10)
611 : {
612 197 : const char *pszHeader =
613 : reinterpret_cast<char *>(poOpenInfo->pabyHeader);
614 197 : fn.assign(pszHeader, poOpenInfo->nHeaderBytes);
615 197 : if (STARTS_WITH(pszHeader, "<MRF_META>")) // Regular file name
616 192 : config = CPLParseXMLFile(pszFileName);
617 5 : else if (poOpenInfo->eAccess == GA_ReadOnly && fn.size() > 600 &&
618 5 : (fn[262] == 0 || fn[262] == 32) &&
619 1 : STARTS_WITH(fn.c_str() + 257, "ustar") &&
620 11 : strlen(CPLGetPathSafe(fn.c_str()).c_str()) == 0 &&
621 1 : STARTS_WITH(fn.c_str() + 512, "<MRF_META>"))
622 : { // An MRF inside a tar
623 1 : insidefn = string("/vsitar/") + pszFileName + "/" + pszHeader;
624 1 : config = CPLParseXMLFile(insidefn.c_str());
625 : }
626 : #if defined(GDAL_USE_LERC_INTERNAL)
627 : else
628 4 : config = LERC_Band::GetMRFConfig(poOpenInfo);
629 : #endif
630 : }
631 : else
632 : {
633 5 : if (EQUALN(pszFileName, "<MRF_META>", 10)) // Content as file name
634 0 : config = CPLParseXMLString(pszFileName);
635 : else
636 : { // Try Ornate file name
637 5 : fn = pszFileName;
638 5 : size_t pos = fn.find(":MRF:");
639 5 : if (string::npos != pos)
640 : { // Tokenize and pick known options
641 5 : vector<string> tokens;
642 5 : stringSplit(tokens, fn, pos + 5, ':');
643 5 : level = getnum(tokens, 'L', -1);
644 5 : version = getnum(tokens, 'V', 0);
645 5 : zslice = getnum(tokens, 'Z', 0);
646 5 : fn.resize(pos); // Cut the ornamentations
647 5 : pszFileName = fn.c_str();
648 5 : config = CPLParseXMLFile(pszFileName);
649 : }
650 : }
651 : }
652 :
653 202 : if (!config)
654 0 : return nullptr;
655 :
656 202 : MRFDataset *ds = new MRFDataset();
657 202 : ds->fname = pszFileName;
658 202 : if (!insidefn.empty())
659 : {
660 1 : ds->publicname = pszFileName;
661 1 : ds->fname = insidefn;
662 : }
663 202 : ds->eAccess = poOpenInfo->eAccess;
664 202 : ds->level = level;
665 202 : ds->zslice = zslice;
666 :
667 : // OpenOptions can override file name arguments
668 202 : ds->ProcessOpenOptions(poOpenInfo->papszOpenOptions);
669 :
670 202 : if (level == -1)
671 200 : ret = ds->Initialize(config);
672 : else
673 : {
674 : // Open the whole dataset, then pick one level
675 2 : ds->cds = new MRFDataset();
676 2 : ds->cds->fname = ds->fname;
677 2 : ds->cds->eAccess = ds->eAccess;
678 2 : ds->zslice = zslice;
679 2 : ret = ds->cds->Initialize(config);
680 2 : if (ret == CE_None)
681 2 : ret = ds->LevelInit(level);
682 : }
683 202 : CPLDestroyXMLNode(config);
684 :
685 202 : if (ret != CE_None)
686 : {
687 23 : delete ds;
688 23 : return nullptr;
689 : }
690 :
691 : // Caching requires a tile compression
692 179 : if (!ds->source.empty() && IL_NONE == ds->current.comp)
693 : {
694 0 : delete ds;
695 0 : CPLError(CE_Failure, CPLE_AppDefined,
696 : "GDAL MRF: Cached MRF must use some compression");
697 0 : return nullptr;
698 : }
699 :
700 : // Open a single version
701 179 : if (version != 0)
702 2 : ret = ds->SetVersion(version);
703 :
704 179 : if (ret != CE_None)
705 : {
706 1 : delete ds;
707 1 : return nullptr;
708 : }
709 :
710 : // Tell PAM what our real file name is, to help it find the aux.xml
711 178 : ds->SetPhysicalFilename(ds->fname);
712 : // Don't mess with metadata after this, otherwise PAM will re-write the
713 : // aux.xml
714 178 : ds->TryLoadXML();
715 :
716 : /* -------------------------------------------------------------------- */
717 : /* Open external overviews. */
718 : /* -------------------------------------------------------------------- */
719 178 : ds->oOvManager.Initialize(ds, ds->fname);
720 :
721 178 : return ds;
722 : }
723 :
724 : // Adjust the band images with the right offset, then adjust the sizes
725 2 : CPLErr MRFDataset::SetVersion(int version)
726 : {
727 2 : if (!hasVersions || version > verCount)
728 : {
729 1 : CPLError(CE_Failure, CPLE_AppDefined,
730 : "GDAL MRF: Version number error!");
731 1 : return CE_Failure;
732 : }
733 : // Size of one version index
734 2 : for (int bcount = 1; bcount <= nBands; bcount++)
735 : {
736 : MRFRasterBand *srcband =
737 1 : cpl::down_cast<MRFRasterBand *>(GetRasterBand(bcount));
738 1 : srcband->img.idxoffset += idxSize * verCount;
739 1 : for (int l = 0; l < srcband->GetOverviewCount(); l++)
740 : {
741 : MRFRasterBand *band =
742 0 : cpl::down_cast<MRFRasterBand *>(srcband->GetOverview(l));
743 0 : if (band != nullptr)
744 0 : band->img.idxoffset += idxSize * verCount;
745 : }
746 : }
747 1 : hasVersions = 0;
748 1 : return CE_None;
749 : }
750 :
751 2 : CPLErr MRFDataset::LevelInit(const int l)
752 : {
753 : // Test that this level does exist
754 2 : if (l < 0 || l >= cds->GetRasterBand(1)->GetOverviewCount())
755 : {
756 1 : CPLError(CE_Failure, CPLE_AppDefined,
757 : "GDAL MRF: Overview not present!");
758 1 : return CE_Failure;
759 : }
760 :
761 : MRFRasterBand *srcband =
762 1 : cpl::down_cast<MRFRasterBand *>(cds->GetRasterBand(1)->GetOverview(l));
763 1 : if (!srcband)
764 0 : return CE_Failure;
765 :
766 : // Copy the sizes from this level
767 1 : full = srcband->img;
768 1 : current = srcband->img;
769 1 : current.size.c = cds->current.size.c;
770 1 : scale = cds->scale;
771 1 : const auto poSRS = cds->GetSpatialRef();
772 1 : if (poSRS)
773 1 : m_oSRS = *poSRS;
774 :
775 1 : SetMetadataItem(GDALMD_INTERLEAVE, OrderName(current.order),
776 1 : GDAL_MDD_IMAGE_STRUCTURE);
777 1 : SetMetadataItem(GDALMD_COMPRESSION, CompName(current.comp),
778 1 : GDAL_MDD_IMAGE_STRUCTURE);
779 :
780 1 : bGeoTransformValid = (CE_None == cds->GetGeoTransform(m_gt));
781 4 : for (int i = 0; i < l + 1; i++)
782 : {
783 3 : m_gt.xscale *= scale;
784 3 : m_gt.yscale *= scale;
785 : }
786 :
787 1 : nRasterXSize = current.size.x;
788 1 : nRasterYSize = current.size.y;
789 1 : nBands = current.size.c;
790 :
791 : // Add the bands, copy constructor so they can be closed independently
792 2 : for (int i = 1; i <= nBands; i++)
793 1 : SetBand(i, new MRFLRasterBand(reinterpret_cast<MRFRasterBand *>(
794 1 : cds->GetRasterBand(i)->GetOverview(l))));
795 1 : return CE_None;
796 : }
797 :
798 : // Is the string positive or not
799 1273 : inline bool on(const char *pszValue)
800 : {
801 1273 : if (!pszValue || pszValue[0] == 0)
802 110 : return false;
803 2314 : return EQUAL(pszValue, "ON") || EQUAL(pszValue, "TRUE") ||
804 2314 : EQUAL(pszValue, "YES");
805 : }
806 :
807 : /**
808 : *\brief Initialize the image structure and the dataset from the XML Raster node
809 : *
810 : * @param image the structure to be initialized
811 : * @param ds the parent dataset, some things get inherited
812 : * @param defimage defimage
813 : *
814 : * The structure should be initialized with the default values as much as
815 : *possible
816 : *
817 : */
818 :
819 364 : static CPLErr Init_Raster(ILImage &image, MRFDataset *ds, CPLXMLNode *defimage)
820 : {
821 : CPLXMLNode *node; // temporary
822 364 : if (!defimage)
823 : {
824 0 : CPLError(CE_Failure, CPLE_AppDefined,
825 : "GDAL MRF: Can't find raster info");
826 0 : return CE_Failure;
827 : }
828 :
829 : // Size is mandatory
830 364 : node = CPLGetXMLNode(defimage, "Size");
831 :
832 364 : if (node)
833 : {
834 364 : image.size = ILSize(static_cast<int>(getXMLNum(node, "x", -1)),
835 364 : static_cast<int>(getXMLNum(node, "y", -1)),
836 364 : static_cast<int>(getXMLNum(node, "z", 1)),
837 364 : static_cast<int>(getXMLNum(node, "c", 1)), 0);
838 : }
839 :
840 : // Basic checks
841 364 : if (!node || image.size.x < 1 || image.size.y < 1 || image.size.z < 0 ||
842 728 : image.size.c < 0 || !GDALCheckBandCount(image.size.c, FALSE))
843 : {
844 0 : CPLError(CE_Failure, CPLE_AppDefined, "Raster size missing or invalid");
845 0 : return CE_Failure;
846 : }
847 :
848 : // Pagesize, defaults to 512,512,1,c
849 364 : image.pagesize = ILSize(std::min(512, image.size.x),
850 364 : std::min(512, image.size.y), 1, image.size.c);
851 :
852 364 : node = CPLGetXMLNode(defimage, "PageSize");
853 364 : if (node)
854 : {
855 364 : image.pagesize =
856 364 : ILSize(static_cast<int>(getXMLNum(node, "x", image.pagesize.x)),
857 364 : static_cast<int>(getXMLNum(node, "y", image.pagesize.y)),
858 : 1, // One slice at a time, forced
859 364 : static_cast<int>(getXMLNum(node, "c", image.pagesize.c)));
860 364 : if (image.pagesize.x < 1 || image.pagesize.y < 1 ||
861 364 : image.pagesize.c <= 0)
862 : {
863 0 : CPLError(CE_Failure, CPLE_IllegalArg, "Invalid PageSize");
864 0 : return CE_Failure;
865 : }
866 : }
867 :
868 : // Page Encoding, defaults to PNG
869 364 : const char *pszCompression = CPLGetXMLValue(defimage, "Compression", "PNG");
870 364 : image.comp = CompToken(pszCompression);
871 364 : if (image.comp == IL_ERR_COMP)
872 : {
873 0 : CPLError(CE_Failure, CPLE_IllegalArg,
874 : "GDAL MRF: Compression %s is unknown", pszCompression);
875 0 : return CE_Failure;
876 : }
877 :
878 : // Is there a palette?
879 : //
880 : // GDAL only supports RGB+A palette, the other modes don't work
881 : //
882 :
883 672 : if ((image.pagesize.c == 1) &&
884 308 : (nullptr != (node = CPLGetXMLNode(defimage, "Palette"))))
885 : {
886 1 : int entries = static_cast<int>(getXMLNum(node, "Size", 255));
887 1 : GDALPaletteInterp eInterp = GPI_RGB;
888 1 : if ((entries > 0) && (entries < 257))
889 : {
890 1 : GDALColorEntry ce_start = {0, 0, 0, 255}, ce_end = {0, 0, 0, 255};
891 :
892 : // Create it and initialize it to black opaque
893 1 : GDALColorTable *poColorTable = new GDALColorTable(eInterp);
894 1 : poColorTable->CreateColorRamp(0, &ce_start, entries - 1, &ce_end);
895 : // Read the values
896 1 : CPLXMLNode *p = CPLGetXMLNode(node, "Entry");
897 1 : if (p)
898 : {
899 : // Initialize the first entry
900 1 : ce_start = GetXMLColorEntry(p);
901 1 : int start_idx = static_cast<int>(getXMLNum(p, "idx", 0));
902 1 : if (start_idx < 0)
903 : {
904 0 : CPLError(CE_Failure, CPLE_IllegalArg,
905 : "GDAL MRF: Palette index %d not allowed",
906 : start_idx);
907 0 : delete poColorTable;
908 0 : return CE_Failure;
909 : }
910 1 : poColorTable->SetColorEntry(start_idx, &ce_start);
911 256 : while (nullptr != (p = SearchXMLSiblings(p, "Entry")))
912 : {
913 : // For every entry, create a ramp
914 255 : ce_end = GetXMLColorEntry(p);
915 : int end_idx =
916 255 : static_cast<int>(getXMLNum(p, "idx", start_idx + 1));
917 255 : if ((end_idx <= start_idx) || (start_idx >= entries))
918 : {
919 0 : CPLError(CE_Failure, CPLE_IllegalArg,
920 : "GDAL MRF: Index Error at index %d", end_idx);
921 0 : delete poColorTable;
922 0 : return CE_Failure;
923 : }
924 255 : poColorTable->CreateColorRamp(start_idx, &ce_start, end_idx,
925 : &ce_end);
926 255 : ce_start = ce_end;
927 255 : start_idx = end_idx;
928 : }
929 : }
930 :
931 1 : ds->SetColorTable(poColorTable);
932 : }
933 : else
934 : {
935 0 : CPLError(CE_Failure, CPLE_IllegalArg,
936 : "GDAL MRF: Palette definition error");
937 0 : return CE_Failure;
938 : }
939 : }
940 :
941 : // Order of increment
942 364 : if (image.pagesize.c != image.size.c && image.pagesize.c != 1)
943 : {
944 : // Fixes heap buffer overflow in
945 : // GDALMRFRasterBand::ReadInterleavedBlock() See
946 : // https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=2884
947 0 : CPLError(CE_Failure, CPLE_NotSupported,
948 : "GDAL MRF: image.pagesize.c = %d and image.size.c = %d",
949 : image.pagesize.c, image.size.c);
950 0 : return CE_Failure;
951 : }
952 :
953 364 : image.order = OrderToken(
954 : CPLGetXMLValue(defimage, "Order",
955 364 : (image.pagesize.c != image.size.c) ? "BAND" : "PIXEL"));
956 364 : if (image.order == IL_ERR_ORD)
957 : {
958 0 : CPLError(CE_Failure, CPLE_AppDefined, "GDAL MRF: Order %s is unknown",
959 : CPLGetXMLValue(defimage, "Order", nullptr));
960 0 : return CE_Failure;
961 : }
962 :
963 364 : const char *photo_val = CPLGetXMLValue(defimage, "Photometric", nullptr);
964 364 : if (photo_val)
965 5 : ds->SetPhotometricInterpretation(photo_val);
966 :
967 364 : image.quality = atoi(CPLGetXMLValue(defimage, "Quality", "85"));
968 364 : if (image.quality < 0 || image.quality > 99)
969 : {
970 0 : CPLError(CE_Warning, CPLE_AppDefined,
971 : "GDAL MRF: Quality setting error, using default of 85");
972 0 : image.quality = 85;
973 : }
974 :
975 : // Data Type, use GDAL Names
976 364 : image.dt = GDALGetDataTypeByName(
977 : CPLGetXMLValue(defimage, "DataType", GDALGetDataTypeName(image.dt)));
978 364 : if (image.dt == GDT_Unknown || GDALGetDataTypeSizeBytes(image.dt) == 0)
979 : {
980 0 : CPLError(CE_Failure, CPLE_AppDefined, "GDAL MRF: Unsupported type");
981 0 : return CE_Failure;
982 : }
983 :
984 : // Check the endianness if needed, assume host order
985 364 : if (is_Endianness_Dependent(image.dt, image.comp))
986 69 : image.nbo = on(CPLGetXMLValue(defimage, "NetByteOrder", "No"));
987 :
988 364 : CPLXMLNode *DataValues = CPLGetXMLNode(defimage, "DataValues");
989 364 : if (nullptr != DataValues)
990 : {
991 26 : const char *pszValue = CPLGetXMLValue(DataValues, "NoData", nullptr);
992 26 : if (pszValue)
993 19 : ds->SetNoDataValue(pszValue);
994 26 : pszValue = CPLGetXMLValue(DataValues, "min", nullptr);
995 26 : if (pszValue)
996 7 : ds->SetMinValue(pszValue);
997 26 : pszValue = CPLGetXMLValue(DataValues, "max", nullptr);
998 26 : if (pszValue)
999 7 : ds->SetMaxValue(pszValue);
1000 : }
1001 :
1002 : // Check that pagesize.c is 64 or less so we can use bitmasks
1003 364 : if (image.pagesize.c > 64)
1004 : {
1005 0 : CPLError(CE_Failure, CPLE_AppDefined,
1006 : "MRF: Max number of pixel interleaved bands is 64");
1007 0 : return CE_Failure;
1008 : }
1009 :
1010 : // Calculate the page size in bytes
1011 364 : const int nDTSize = GDALGetDataTypeSizeBytes(image.dt);
1012 364 : if (nDTSize <= 0 || image.pagesize.z <= 0 ||
1013 364 : image.pagesize.x > INT_MAX / image.pagesize.y ||
1014 364 : image.pagesize.x * image.pagesize.y > INT_MAX / image.pagesize.z ||
1015 364 : image.pagesize.x * image.pagesize.y * image.pagesize.z >
1016 364 : INT_MAX / image.pagesize.c ||
1017 364 : image.pagesize.x * image.pagesize.y * image.pagesize.z *
1018 364 : image.pagesize.c >
1019 364 : INT_MAX / nDTSize)
1020 : {
1021 0 : CPLError(CE_Failure, CPLE_AppDefined, "MRF page size is too large");
1022 0 : return CE_Failure;
1023 : }
1024 364 : image.pageSizeBytes = nDTSize * image.pagesize.x * image.pagesize.y *
1025 364 : image.pagesize.z * image.pagesize.c;
1026 :
1027 : // Calculate the page count, including the total for the level
1028 364 : image.pagecount = pcount(image.size, image.pagesize);
1029 364 : if (image.pagecount.l < 0)
1030 : {
1031 0 : return CE_Failure;
1032 : }
1033 :
1034 : // Data File Name and base offset
1035 : image.datfname =
1036 364 : getFname(defimage, "DataFile", ds->GetFname(), ILComp_Ext[image.comp]);
1037 364 : image.dataoffset = static_cast<int>(
1038 364 : getXMLNum(CPLGetXMLNode(defimage, "DataFile"), "offset", 0.0));
1039 :
1040 : // Index File Name and base offset
1041 364 : image.idxfname = getFname(defimage, "IndexFile", ds->GetFname(), ".idx");
1042 364 : image.idxoffset = static_cast<int>(
1043 364 : getXMLNum(CPLGetXMLNode(defimage, "IndexFile"), "offset", 0.0));
1044 :
1045 364 : return CE_None;
1046 : }
1047 :
1048 162 : char **MRFDataset::GetFileList()
1049 : {
1050 162 : char **papszFileList = nullptr;
1051 :
1052 162 : string usename = fname;
1053 162 : if (!publicname.empty())
1054 0 : usename = publicname;
1055 : // Add the header file name if it is real
1056 : VSIStatBufL sStat;
1057 162 : if (VSIStatExL(usename.c_str(), &sStat, VSI_STAT_EXISTS_FLAG) == 0)
1058 162 : papszFileList = CSLAddString(papszFileList, usename.c_str());
1059 :
1060 : // These two should be real
1061 : // We don't really want to add these files, since they will be erased when
1062 : // an mrf is overwritten This collides with the concept that the data file
1063 : // never shrinks. Same goes with the index, in case we just want to add
1064 : // things to it.
1065 : // papszFileList = CSLAddString( papszFileList, full.datfname);
1066 : // papszFileList = CSLAddString( papszFileList, full.idxfname);
1067 : // if (!source.empty())
1068 : // papszFileList = CSLAddString( papszFileList, source);
1069 :
1070 324 : return papszFileList;
1071 : }
1072 :
1073 : // Try to create all the folders in the path in sequence, ignore errors
1074 4 : static void mkdir_r(string const &fname)
1075 : {
1076 4 : size_t loc = fname.find_first_of("\\/");
1077 4 : if (loc == string::npos)
1078 0 : return;
1079 : while (true)
1080 : {
1081 5 : ++loc;
1082 5 : loc = fname.find_first_of("\\/", loc);
1083 5 : if (loc == string::npos)
1084 4 : break;
1085 1 : VSIMkdir(fname.substr(0, loc).c_str(), 0);
1086 : }
1087 : }
1088 :
1089 : // Returns the dataset index file or null
1090 7266 : VSILFILE *MRFDataset::IdxFP()
1091 : {
1092 7266 : if (ifp.FP != nullptr)
1093 6999 : return ifp.FP;
1094 :
1095 : // If missing is set, we already checked, there is no index
1096 267 : if (missing)
1097 0 : return nullptr;
1098 :
1099 : // If name starts with '(' it is not a real file name
1100 267 : if (current.idxfname[0] == '(')
1101 0 : return nullptr;
1102 :
1103 267 : const char *mode = "rb";
1104 267 : ifp.acc = GF_Read;
1105 :
1106 267 : if (eAccess == GA_Update || !source.empty())
1107 : {
1108 159 : mode = "r+b";
1109 159 : ifp.acc = GF_Write;
1110 : }
1111 :
1112 267 : ifp.FP = VSIFOpenL(current.idxfname, mode);
1113 :
1114 : // If file didn't open for reading and no_errors is set, just return null
1115 : // and make a note
1116 267 : if (ifp.FP == nullptr && eAccess == GA_ReadOnly && no_errors)
1117 : {
1118 0 : missing = 1;
1119 0 : return nullptr;
1120 : }
1121 :
1122 : // need to create the index file
1123 367 : if (ifp.FP == nullptr && !bCrystalized &&
1124 100 : (eAccess == GA_Update || !source.empty()))
1125 : {
1126 100 : mode = "w+b";
1127 100 : ifp.FP = VSIFOpenL(current.idxfname, mode);
1128 : }
1129 :
1130 267 : if (nullptr == ifp.FP && !source.empty())
1131 : {
1132 : // caching and cloning, try making the folder and attempt again
1133 4 : mkdir_r(current.idxfname);
1134 4 : ifp.FP = VSIFOpenL(current.idxfname, mode);
1135 : }
1136 :
1137 267 : GIntBig expected_size = idxSize;
1138 267 : if (clonedSource)
1139 2 : expected_size *= 2;
1140 :
1141 267 : if (nullptr != ifp.FP)
1142 : {
1143 409 : if (!bCrystalized &&
1144 150 : !CheckFileSize(current.idxfname, expected_size, GA_Update))
1145 : {
1146 0 : CPLError(CE_Failure, CPLE_FileIO,
1147 : "MRF: Can't extend the cache index file %s",
1148 : current.idxfname.c_str());
1149 0 : return nullptr;
1150 : }
1151 :
1152 259 : if (source.empty())
1153 255 : return ifp.FP;
1154 :
1155 : // Make sure the index is large enough before proceeding
1156 : // Timeout in .1 seconds, can't really guarantee the accuracy
1157 : // So this is about half second, should be sufficient
1158 4 : int timeout = 5;
1159 0 : do
1160 : {
1161 4 : if (CheckFileSize(current.idxfname, expected_size, GA_ReadOnly))
1162 4 : return ifp.FP;
1163 0 : CPLSleep(0.100); /* 100 ms */
1164 0 : } while (--timeout);
1165 :
1166 : // If we get here it is a time-out
1167 0 : CPLError(CE_Failure, CPLE_AppDefined,
1168 : "GDAL MRF: Timeout on fetching cloned index file %s\n",
1169 : current.idxfname.c_str());
1170 0 : return nullptr;
1171 : }
1172 :
1173 : // If single tile, and no index file, let the caller figure it out
1174 8 : if (IsSingleTile())
1175 4 : return nullptr;
1176 :
1177 : // Error if this is not a caching MRF
1178 4 : if (source.empty())
1179 : {
1180 0 : CPLError(CE_Failure, CPLE_AppDefined,
1181 : "GDAL MRF: Can't open index file %s\n",
1182 : current.idxfname.c_str());
1183 0 : return nullptr;
1184 : }
1185 :
1186 : // Caching/Cloning MRF and index could be read only
1187 : // Is this actually works, we should try again, maybe somebody else just
1188 : // created the file?
1189 4 : mode = "rb";
1190 4 : ifp.acc = GF_Read;
1191 4 : ifp.FP = VSIFOpenL(current.idxfname, mode);
1192 4 : if (nullptr != ifp.FP)
1193 0 : return ifp.FP;
1194 :
1195 : // Caching and index file absent, create it
1196 : // Due to a race, multiple processes might do this at the same time, but
1197 : // that is fine
1198 4 : ifp.FP = VSIFOpenL(current.idxfname, "wb");
1199 4 : if (nullptr == ifp.FP)
1200 : {
1201 0 : CPLError(CE_Failure, CPLE_AppDefined,
1202 : "Can't create the MRF cache index file %s",
1203 : current.idxfname.c_str());
1204 0 : return nullptr;
1205 : }
1206 4 : VSIFCloseL(ifp.FP);
1207 4 : ifp.FP = nullptr;
1208 :
1209 : // Make it large enough for caching and for cloning
1210 4 : if (!CheckFileSize(current.idxfname, expected_size, GA_Update))
1211 : {
1212 0 : CPLError(CE_Failure, CPLE_AppDefined,
1213 : "Can't extend the cache index file %s",
1214 : current.idxfname.c_str());
1215 0 : return nullptr;
1216 : }
1217 :
1218 : // Try opening it again in rw mode so we can read and write
1219 4 : mode = "r+b";
1220 4 : ifp.acc = GF_Write;
1221 4 : ifp.FP = VSIFOpenL(current.idxfname.c_str(), mode);
1222 :
1223 4 : if (nullptr == ifp.FP)
1224 : {
1225 0 : CPLError(CE_Failure, CPLE_AppDefined,
1226 : "GDAL MRF: Can't reopen cache index file %s\n",
1227 : full.idxfname.c_str());
1228 0 : return nullptr;
1229 : }
1230 4 : return ifp.FP;
1231 : }
1232 :
1233 : //
1234 : // Returns the dataset data file or null
1235 : // Data file is opened either in Read or Append mode, never in straight write
1236 : //
1237 7135 : VSILFILE *MRFDataset::DataFP()
1238 : {
1239 7135 : if (dfp.FP != nullptr)
1240 6871 : return dfp.FP;
1241 264 : const char *mode = "rb";
1242 264 : dfp.acc = GF_Read;
1243 :
1244 : // Open it for writing if updating or if caching
1245 264 : if (eAccess == GA_Update || !source.empty())
1246 : {
1247 158 : mode = "a+b";
1248 158 : dfp.acc = GF_Write;
1249 : }
1250 :
1251 264 : dfp.FP = VSIFOpenL(current.datfname, mode);
1252 264 : if (dfp.FP)
1253 264 : return dfp.FP;
1254 :
1255 : // It could be a caching MRF
1256 0 : if (source.empty())
1257 0 : goto io_error;
1258 :
1259 : // May be there but read only, remember that it was open that way
1260 0 : mode = "rb";
1261 0 : dfp.acc = GF_Read;
1262 0 : dfp.FP = VSIFOpenL(current.datfname, mode);
1263 0 : if (nullptr != dfp.FP)
1264 : {
1265 0 : CPLDebug("MRF_IO", "Opened %s RO mode %s\n", current.datfname.c_str(),
1266 : mode);
1267 0 : return dfp.FP;
1268 : }
1269 :
1270 0 : if (source.empty())
1271 0 : goto io_error;
1272 :
1273 : // caching, maybe the folder didn't exist
1274 0 : mkdir_r(current.datfname);
1275 0 : mode = "a+b";
1276 0 : dfp.acc = GF_Write;
1277 0 : dfp.FP = VSIFOpenL(current.datfname, mode);
1278 0 : if (dfp.FP)
1279 0 : return dfp.FP;
1280 :
1281 0 : io_error:
1282 0 : dfp.FP = nullptr;
1283 0 : CPLError(CE_Failure, CPLE_FileIO, "GDAL MRF: %s : %s", strerror(errno),
1284 : current.datfname.c_str());
1285 0 : return nullptr;
1286 : }
1287 :
1288 : // Builds an XML tree from the current MRF. If written to a file it becomes an
1289 : // MRF
1290 334 : CPLXMLNode *MRFDataset::BuildConfig()
1291 : {
1292 334 : CPLXMLNode *config = CPLCreateXMLNode(nullptr, CXT_Element, "MRF_META");
1293 :
1294 334 : if (!source.empty())
1295 : {
1296 : CPLXMLNode *psCachedSource =
1297 4 : CPLCreateXMLNode(config, CXT_Element, "CachedSource");
1298 : // Should wrap the string in CDATA, in case it is XML
1299 : CPLXMLNode *psSource =
1300 4 : CPLCreateXMLElementAndValue(psCachedSource, "Source", source);
1301 4 : if (clonedSource)
1302 0 : CPLSetXMLValue(psSource, "#clone", "true");
1303 : }
1304 :
1305 : // Use the full size
1306 334 : CPLXMLNode *raster = CPLCreateXMLNode(config, CXT_Element, "Raster");
1307 :
1308 : // Preserve the file names if not the default ones
1309 334 : if (full.datfname != getFname(GetFname(), ILComp_Ext[full.comp]))
1310 0 : CPLCreateXMLElementAndValue(raster, "DataFile", full.datfname.c_str());
1311 334 : if (full.idxfname != getFname(GetFname(), ".idx"))
1312 0 : CPLCreateXMLElementAndValue(raster, "IndexFile", full.idxfname.c_str());
1313 334 : if (spacing != 0)
1314 0 : XMLSetAttributeVal(raster, "Spacing", static_cast<double>(spacing),
1315 : "%.0f");
1316 :
1317 334 : XMLSetAttributeVal(raster, "Size", full.size, "%.0f");
1318 334 : XMLSetAttributeVal(raster, "PageSize", full.pagesize, "%.0f");
1319 :
1320 : #ifdef HAVE_PNG
1321 334 : if (full.comp != IL_PNG)
1322 : #endif
1323 : {
1324 194 : CPLCreateXMLElementAndValue(raster, "Compression", CompName(full.comp));
1325 : }
1326 :
1327 334 : if (full.dt != GDT_UInt8)
1328 202 : CPLCreateXMLElementAndValue(raster, "DataType",
1329 : GDALGetDataTypeName(full.dt));
1330 :
1331 : // special photometric interpretation
1332 334 : if (!photometric.empty())
1333 4 : CPLCreateXMLElementAndValue(raster, "Photometric", photometric);
1334 :
1335 334 : if (!vNoData.empty() || !vMin.empty() || !vMax.empty())
1336 : {
1337 : CPLXMLNode *values =
1338 43 : CPLCreateXMLNode(raster, CXT_Element, "DataValues");
1339 43 : XMLSetAttributeVal(values, "NoData", vNoData);
1340 43 : XMLSetAttributeVal(values, "min", vMin);
1341 43 : XMLSetAttributeVal(values, "max", vMax);
1342 : }
1343 :
1344 : // palette, if we have one
1345 334 : if (poColorTable != nullptr)
1346 : {
1347 1 : const char *pfrmt = "%.0f";
1348 1 : CPLXMLNode *pal = CPLCreateXMLNode(raster, CXT_Element, "Palette");
1349 1 : int sz = poColorTable->GetColorEntryCount();
1350 1 : if (sz != 256)
1351 0 : XMLSetAttributeVal(pal, "Size", poColorTable->GetColorEntryCount());
1352 : // RGB or RGBA for now
1353 257 : for (int i = 0; i < sz; i++)
1354 : {
1355 256 : CPLXMLNode *entry = CPLCreateXMLNode(pal, CXT_Element, "Entry");
1356 256 : const GDALColorEntry *ent = poColorTable->GetColorEntry(i);
1357 : // No need to set the index, it is always from 0 no size-1
1358 256 : XMLSetAttributeVal(entry, "c1", ent->c1, pfrmt);
1359 256 : XMLSetAttributeVal(entry, "c2", ent->c2, pfrmt);
1360 256 : XMLSetAttributeVal(entry, "c3", ent->c3, pfrmt);
1361 256 : if (ent->c4 != 255)
1362 0 : XMLSetAttributeVal(entry, "c4", ent->c4, pfrmt);
1363 : }
1364 : }
1365 :
1366 334 : if (is_Endianness_Dependent(full.dt, full.comp)) // Need to set the order
1367 62 : CPLCreateXMLElementAndValue(raster, "NetByteOrder",
1368 62 : (full.nbo || NET_ORDER) ? "TRUE" : "FALSE");
1369 :
1370 334 : if (full.quality > 0 && full.quality != 85)
1371 12 : CPLCreateXMLElementAndValue(raster, "Quality",
1372 24 : CPLOPrintf("%d", full.quality));
1373 :
1374 : // Done with the raster node
1375 :
1376 334 : if (scale != 0.0)
1377 : {
1378 0 : CPLCreateXMLNode(config, CXT_Element, "Rsets");
1379 0 : CPLSetXMLValue(config, "Rsets.#model", "uniform");
1380 0 : CPLSetXMLValue(config, "Rsets.#scale", PrintDouble(scale));
1381 : }
1382 334 : CPLXMLNode *gtags = CPLCreateXMLNode(config, CXT_Element, "GeoTags");
1383 :
1384 : // Do we have an affine transform different from identity?
1385 334 : GDALGeoTransform gt;
1386 668 : if ((MRFDataset::GetGeoTransform(gt) == CE_None) &&
1387 334 : (gt.xorig != 0 || gt.xscale != 1 || gt.xrot != 0 || gt.yorig != 0 ||
1388 197 : gt.yrot != 0 || gt.yscale != 1))
1389 : {
1390 137 : double minx = gt.xorig;
1391 137 : double maxx = gt.xscale * full.size.x + minx;
1392 137 : double maxy = gt.yorig;
1393 137 : double miny = gt.yscale * full.size.y + maxy;
1394 137 : CPLXMLNode *bbox = CPLCreateXMLNode(gtags, CXT_Element, "BoundingBox");
1395 137 : XMLSetAttributeVal(bbox, "minx", minx);
1396 137 : XMLSetAttributeVal(bbox, "miny", miny);
1397 137 : XMLSetAttributeVal(bbox, "maxx", maxx);
1398 137 : XMLSetAttributeVal(bbox, "maxy", maxy);
1399 : }
1400 :
1401 334 : const char *pszProj = GetProjectionRef();
1402 334 : if (pszProj && (!EQUAL(pszProj, "")))
1403 138 : CPLCreateXMLElementAndValue(gtags, "Projection", pszProj);
1404 :
1405 334 : if (optlist.Count() != 0)
1406 : {
1407 52 : CPLString options;
1408 54 : for (int i = 0; i < optlist.size(); i++)
1409 : {
1410 28 : options += optlist[i];
1411 28 : options += ' ';
1412 : }
1413 26 : options.pop_back();
1414 26 : CPLCreateXMLElementAndValue(config, "Options", options);
1415 : }
1416 :
1417 334 : return config;
1418 : }
1419 :
1420 : /**
1421 : * \brief Populates the dataset variables from the XML definition
1422 : *
1423 : *
1424 : */
1425 364 : CPLErr MRFDataset::Initialize(CPLXMLNode *config)
1426 : {
1427 : // We only need a basic initialization here, usually gets overwritten by the
1428 : // image params
1429 364 : full.dt = GDT_UInt8;
1430 364 : full.hasNoData = false;
1431 364 : full.NoDataValue = 0;
1432 364 : Quality = 85;
1433 :
1434 364 : CPLErr ret = Init_Raster(full, this, CPLGetXMLNode(config, "Raster"));
1435 364 : if (CE_None != ret)
1436 0 : return ret;
1437 :
1438 364 : hasVersions = on(CPLGetXMLValue(config, "Raster.versioned", "no"));
1439 364 : mp_safe = on(CPLGetXMLValue(config, "Raster.mp_safe", "no"));
1440 364 : spacing = atoi(CPLGetXMLValue(config, "Raster.Spacing", "0"));
1441 :
1442 : // The zslice defined in the file wins over the oo or the file argument
1443 364 : if (CPLGetXMLNode(config, "Raster.zslice"))
1444 0 : zslice = atoi(CPLGetXMLValue(config, "Raster.zslice", "0"));
1445 :
1446 364 : Quality = full.quality;
1447 :
1448 : // Bounding box
1449 364 : CPLXMLNode *bbox = CPLGetXMLNode(config, "GeoTags.BoundingBox");
1450 364 : if (nullptr != bbox)
1451 : {
1452 : double x0, x1, y0, y1;
1453 :
1454 161 : x0 = atof(CPLGetXMLValue(bbox, "minx", "0"));
1455 161 : x1 = atof(CPLGetXMLValue(bbox, "maxx", "1"));
1456 161 : y1 = atof(CPLGetXMLValue(bbox, "maxy", "1"));
1457 161 : y0 = atof(CPLGetXMLValue(bbox, "miny", "0"));
1458 :
1459 161 : m_gt.xorig = x0;
1460 161 : m_gt.xscale = (x1 - x0) / full.size.x;
1461 161 : m_gt.xrot = 0;
1462 161 : m_gt.yorig = y1;
1463 161 : m_gt.yrot = 0;
1464 161 : m_gt.yscale = (y0 - y1) / full.size.y;
1465 161 : bGeoTransformValid = TRUE;
1466 : }
1467 :
1468 : const char *pszRawProjFromXML =
1469 364 : CPLGetXMLValue(config, "GeoTags.Projection", "");
1470 364 : if (strlen(pszRawProjFromXML) != 0)
1471 162 : m_oSRS.SetFromUserInput(
1472 : pszRawProjFromXML,
1473 : OGRSpatialReference::SET_FROM_USER_INPUT_LIMITATIONS_get());
1474 :
1475 : // Copy the full size to current, data and index are not yet open
1476 364 : current = full;
1477 364 : if (current.size.z != 1)
1478 : {
1479 0 : SetMetadataItem("ZSIZE", CPLOPrintf("%d", current.size.z),
1480 0 : GDAL_MDD_IMAGE_STRUCTURE);
1481 0 : SetMetadataItem("ZSLICE", CPLOPrintf("%d", zslice),
1482 0 : GDAL_MDD_IMAGE_STRUCTURE);
1483 : // Capture the zslice in pagesize.l
1484 0 : current.pagesize.l = zslice;
1485 : // Adjust offset for base image
1486 0 : if (full.size.z <= 0)
1487 : {
1488 0 : CPLError(CE_Failure, CPLE_AppDefined,
1489 : "GDAL MRF: Invalid Raster.z value");
1490 0 : return CE_Failure;
1491 : }
1492 0 : if (zslice >= full.size.z)
1493 : {
1494 0 : CPLError(CE_Failure, CPLE_AppDefined, "GDAL MRF: Invalid slice");
1495 0 : return CE_Failure;
1496 : }
1497 :
1498 0 : current.idxoffset +=
1499 0 : (current.pagecount.l / full.size.z) * zslice * sizeof(ILIdx);
1500 : }
1501 :
1502 : // Dataset metadata setup
1503 364 : SetMetadataItem(GDALMD_INTERLEAVE, OrderName(current.order),
1504 364 : GDAL_MDD_IMAGE_STRUCTURE);
1505 364 : SetMetadataItem(GDALMD_COMPRESSION, CompName(current.comp),
1506 364 : GDAL_MDD_IMAGE_STRUCTURE);
1507 :
1508 364 : if (is_Endianness_Dependent(current.dt, current.comp))
1509 69 : SetMetadataItem("NETBYTEORDER", current.nbo ? "TRUE" : "FALSE",
1510 69 : GDAL_MDD_IMAGE_STRUCTURE);
1511 :
1512 : // Open the files for the current image, either RW or RO
1513 364 : nRasterXSize = current.size.x;
1514 364 : nRasterYSize = current.size.y;
1515 364 : nBands = current.size.c;
1516 :
1517 364 : if (!nBands || !nRasterXSize || !nRasterYSize)
1518 : {
1519 0 : CPLError(CE_Failure, CPLE_AppDefined, "GDAL MRF: Image size missing");
1520 0 : return CE_Failure;
1521 : }
1522 :
1523 : // Pick up the source data image, if there is one
1524 364 : source = CPLGetXMLValue(config, "CachedSource.Source", "");
1525 : // Is it a clone?
1526 364 : clonedSource =
1527 364 : on(CPLGetXMLValue(config, "CachedSource.Source.clone", "no"));
1528 : // Pick up the options, if any
1529 : optlist.Assign(CSLTokenizeString2(
1530 : CPLGetXMLValue(config, "Options", nullptr), " \t\n\r",
1531 364 : CSLT_STRIPLEADSPACES | CSLT_STRIPENDSPACES));
1532 :
1533 : // Load all the options in the IMAGE_STRUCTURE metadata
1534 395 : for (int i = 0; i < optlist.Count(); i++)
1535 : {
1536 62 : CPLString s(optlist[i]);
1537 31 : size_t nSepPos = s.find_first_of(":=");
1538 31 : if (std::string::npos != nSepPos)
1539 : {
1540 31 : s.resize(nSepPos);
1541 31 : SetMetadataItem(s, optlist.FetchNameValue(s),
1542 31 : GDAL_MDD_IMAGE_STRUCTURE);
1543 : }
1544 : }
1545 :
1546 : // We have the options, so we can call rasterband
1547 794 : for (int i = 1; i <= nBands; i++)
1548 : {
1549 : // The overviews are low resolution copies of the current one.
1550 482 : MRFRasterBand *band = newMRFRasterBand(this, current, i);
1551 482 : if (!band)
1552 52 : return CE_Failure;
1553 :
1554 430 : GDALColorInterp ci = GCI_Undefined;
1555 :
1556 : // Default color interpretation
1557 430 : switch (nBands)
1558 : {
1559 262 : case 1:
1560 : case 2:
1561 262 : ci = (i == 1) ? GCI_GrayIndex : GCI_AlphaBand;
1562 262 : break;
1563 153 : case 3:
1564 : case 4:
1565 153 : if (i < 3)
1566 100 : ci = (i == 1) ? GCI_RedBand : GCI_GreenBand;
1567 : else
1568 53 : ci = (i == 3) ? GCI_BlueBand : GCI_AlphaBand;
1569 : }
1570 :
1571 430 : if (GetColorTable())
1572 1 : ci = GCI_PaletteIndex;
1573 :
1574 : // Legacy, deprecated
1575 430 : if (optlist.FetchBoolean("MULTISPECTRAL", FALSE))
1576 0 : ci = GCI_Undefined;
1577 :
1578 : // New style
1579 430 : if (!photometric.empty())
1580 : {
1581 15 : if ("MULTISPECTRAL" == photometric)
1582 0 : ci = GCI_Undefined;
1583 : }
1584 :
1585 430 : band->SetColorInterpretation(ci);
1586 430 : SetBand(i, band);
1587 : }
1588 :
1589 312 : CPLXMLNode *rsets = CPLGetXMLNode(config, "Rsets");
1590 312 : if (nullptr != rsets && nullptr != rsets->psChild)
1591 : {
1592 : // We have rsets
1593 :
1594 : // Regular spaced overlays, until everything fits in a single tile
1595 32 : if (EQUAL("uniform", CPLGetXMLValue(rsets, "model", "uniform")))
1596 : {
1597 32 : scale = getXMLNum(rsets, "scale", 2.0);
1598 32 : if (scale <= 1)
1599 : {
1600 0 : CPLError(CE_Failure, CPLE_AppDefined,
1601 : "MRF: zoom factor less than unit not allowed");
1602 0 : return CE_Failure;
1603 : }
1604 : // Looks like there are overlays
1605 32 : AddOverviews(int(scale));
1606 : }
1607 : else
1608 : {
1609 0 : CPLError(CE_Failure, CPLE_AppDefined, "Unknown Rset definition");
1610 0 : return CE_Failure;
1611 : }
1612 : }
1613 :
1614 312 : idxSize = IdxSize(full, int(scale));
1615 312 : if (idxSize == 0)
1616 0 : return CE_Failure;
1617 :
1618 : // If not set by the bands, get a pageSizeBytes buffer
1619 312 : if (GetPBufferSize() == 0 && !SetPBuffer(current.pageSizeBytes))
1620 0 : return CE_Failure;
1621 :
1622 312 : if (hasVersions)
1623 : { // It has versions, but how many?
1624 5 : verCount = 0; // Assume it only has one
1625 : VSIStatBufL statb;
1626 : // If the file exists, compute the last version number
1627 5 : if (0 == VSIStatL(full.idxfname, &statb))
1628 5 : verCount = int(statb.st_size / idxSize - 1);
1629 : }
1630 :
1631 312 : return CE_None;
1632 : }
1633 :
1634 0 : static inline bool has_path(const CPLString &name)
1635 : {
1636 0 : return name.find_first_of("/\\") != string::npos;
1637 : }
1638 :
1639 : // Does name look like an absolute gdal file name?
1640 4 : static inline bool is_absolute(const CPLString &name)
1641 : {
1642 4 : return (name.find_first_of("/\\") == 0) // Starts with root
1643 4 : || (name.size() > 1 && name[1] == ':' &&
1644 0 : isalpha(static_cast<unsigned char>(
1645 0 : name[0]))) // Starts with drive letter
1646 8 : || (name[0] == '<'); // Maybe it is XML
1647 : }
1648 :
1649 : // Add the dirname of path to the beginning of name, if it is relative
1650 : // returns true if name was modified
1651 4 : static inline bool make_absolute(CPLString &name, const CPLString &path)
1652 : {
1653 4 : if (!is_absolute(name) && (path.find_first_of("/\\") != string::npos))
1654 : {
1655 4 : name = path.substr(0, path.find_last_of("/\\") + 1) + name;
1656 4 : return true;
1657 : }
1658 0 : return false;
1659 : }
1660 :
1661 : /**
1662 : *\brief Get the source dataset, open it if necessary
1663 : */
1664 5 : GDALDataset *MRFDataset::GetSrcDS()
1665 : {
1666 5 : if (poSrcDS)
1667 1 : return poSrcDS;
1668 4 : if (source.empty())
1669 0 : return nullptr;
1670 :
1671 : // Stub out the error handler
1672 4 : CPLPushErrorHandler(CPLQuietErrorHandler);
1673 : // Try open the source dataset as is
1674 4 : poSrcDS =
1675 4 : GDALDataset::FromHandle(GDALOpenShared(source.c_str(), GA_ReadOnly));
1676 4 : CPLPopErrorHandler();
1677 :
1678 : // It the open fails, try again with the current dataset path prepended
1679 4 : if (!poSrcDS && make_absolute(source, fname))
1680 4 : poSrcDS = GDALDataset::FromHandle(
1681 : GDALOpenShared(source.c_str(), GA_ReadOnly));
1682 :
1683 4 : if (0 == source.find("<MRF_META>") && has_path(fname))
1684 : {
1685 : // MRF XML source, might need to patch the file names with the current
1686 : // one
1687 0 : MRFDataset *poMRFDS = dynamic_cast<MRFDataset *>(poSrcDS);
1688 0 : if (!poMRFDS)
1689 : {
1690 0 : delete poSrcDS;
1691 0 : poSrcDS = nullptr;
1692 0 : return nullptr;
1693 : }
1694 0 : make_absolute(poMRFDS->current.datfname, fname);
1695 0 : make_absolute(poMRFDS->current.idxfname, fname);
1696 : }
1697 4 : mp_safe = true; // Turn on MP safety
1698 4 : return poSrcDS;
1699 : }
1700 :
1701 : /**
1702 : *\brief Add or verify that all overlays exits
1703 : *
1704 : * @return size of the index file
1705 : */
1706 :
1707 62 : GIntBig MRFDataset::AddOverviews(int scaleIn)
1708 : {
1709 : // Fit the overlays
1710 62 : ILImage img = current;
1711 134 : while (1 != img.pagecount.x * img.pagecount.y)
1712 : {
1713 : // Adjust raster data for next level
1714 : // Adjust the offsets for indices left at this level
1715 72 : img.idxoffset += sizeof(ILIdx) * img.pagecount.l / img.size.z *
1716 72 : (img.size.z - zslice);
1717 :
1718 : // Next overview size
1719 72 : img.size.x = pcount(img.size.x, scaleIn);
1720 72 : img.size.y = pcount(img.size.y, scaleIn);
1721 72 : img.size.l++; // Increment the level
1722 72 : img.pagecount = pcount(img.size, img.pagesize);
1723 :
1724 : // And adjust the offset again, within next level
1725 72 : img.idxoffset += sizeof(ILIdx) * img.pagecount.l / img.size.z * zslice;
1726 72 : int l = static_cast<int>(img.size.l);
1727 : // Create and register the overviews for each band
1728 144 : for (int i = 1; i <= nBands; i++)
1729 : {
1730 : MRFRasterBand *b =
1731 72 : reinterpret_cast<MRFRasterBand *>(GetRasterBand(i));
1732 72 : if (!(b->GetOverview(l - 1)))
1733 72 : b->AddOverview(newMRFRasterBand(this, img, i, l));
1734 : }
1735 : }
1736 :
1737 : // Last adjustment, should be a single set of c and leftover z tiles
1738 62 : return img.idxoffset +
1739 124 : sizeof(ILIdx) * img.pagecount.l / img.size.z * (img.size.z - zslice);
1740 : }
1741 :
1742 : //
1743 : // set an entry if it doesn't already exist
1744 : //
1745 135 : static char **CSLAddIfMissing(char **papszList, const char *pszName,
1746 : const char *pszValue)
1747 : {
1748 135 : if (CSLFetchNameValue(papszList, pszName))
1749 14 : return papszList;
1750 121 : return CSLSetNameValue(papszList, pszName, pszValue);
1751 : }
1752 :
1753 : // CreateCopy implemented based on Create
1754 134 : GDALDataset *MRFDataset::CreateCopy(const char *pszFilename,
1755 : GDALDataset *poSrcDS, int /*bStrict*/,
1756 : CSLConstList papszOptions,
1757 : GDALProgressFunc pfnProgress,
1758 : void *pProgressData)
1759 : {
1760 268 : ILImage img;
1761 :
1762 134 : int x = poSrcDS->GetRasterXSize();
1763 134 : int y = poSrcDS->GetRasterYSize();
1764 134 : int nBands = poSrcDS->GetRasterCount();
1765 134 : if (nBands == 0)
1766 : {
1767 1 : CPLError(CE_Failure, CPLE_NotSupported, "nBands == 0 not supported");
1768 1 : return nullptr;
1769 : }
1770 133 : GDALRasterBand *poSrcBand1 = poSrcDS->GetRasterBand(1);
1771 :
1772 133 : GDALDataType dt = poSrcBand1->GetRasterDataType();
1773 : // Have our own options, to modify as we want
1774 133 : char **options = CSLDuplicate(papszOptions);
1775 :
1776 : const char *pszValue =
1777 133 : poSrcDS->GetMetadataItem(GDALMD_INTERLEAVE, GDAL_MDD_IMAGE_STRUCTURE);
1778 133 : options = CSLAddIfMissing(options, GDALMD_INTERLEAVE,
1779 : pszValue ? pszValue : "PIXEL");
1780 : int xb, yb;
1781 133 : poSrcBand1->GetBlockSize(&xb, &yb);
1782 :
1783 : // Keep input block size if it exists and not explicitly set
1784 134 : if (CSLFetchNameValue(options, "BLOCKSIZE") == nullptr && xb != x &&
1785 1 : yb != y)
1786 : {
1787 1 : options = CSLAddIfMissing(options, "BLOCKXSIZE",
1788 2 : PrintDouble(xb, "%d").c_str());
1789 1 : options = CSLAddIfMissing(options, "BLOCKYSIZE",
1790 2 : PrintDouble(yb, "%d").c_str());
1791 : }
1792 :
1793 133 : MRFDataset *poDS = nullptr;
1794 : try
1795 : {
1796 : poDS = reinterpret_cast<MRFDataset *>(
1797 133 : Create(pszFilename, x, y, nBands, dt, options));
1798 :
1799 133 : if (poDS == nullptr || poDS->bCrystalized)
1800 11 : throw CPLOPrintf("MRF: Can't create %s", pszFilename);
1801 :
1802 122 : img = poDS->current; // Deal with the current one here
1803 :
1804 : // Copy data values from source
1805 284 : for (int i = 0; i < poDS->nBands; i++)
1806 : {
1807 : int bHas;
1808 : double dfData;
1809 162 : GDALRasterBand *srcBand = poSrcDS->GetRasterBand(i + 1);
1810 162 : GDALRasterBand *mBand = poDS->GetRasterBand(i + 1);
1811 162 : dfData = srcBand->GetNoDataValue(&bHas);
1812 162 : if (bHas)
1813 : {
1814 16 : poDS->vNoData.push_back(dfData);
1815 16 : mBand->SetNoDataValue(dfData);
1816 : }
1817 162 : dfData = srcBand->GetMinimum(&bHas);
1818 162 : if (bHas)
1819 17 : poDS->vMin.push_back(dfData);
1820 162 : dfData = srcBand->GetMaximum(&bHas);
1821 162 : if (bHas)
1822 17 : poDS->vMax.push_back(dfData);
1823 :
1824 : // Copy the band metadata, PAM will handle it
1825 162 : CSLConstList meta = srcBand->GetMetadata(GDAL_MDD_IMAGE_STRUCTURE);
1826 162 : if (CSLCount(meta))
1827 7 : mBand->SetMetadata(meta, GDAL_MDD_IMAGE_STRUCTURE);
1828 :
1829 162 : meta = srcBand->GetMetadata();
1830 162 : if (CSLCount(meta))
1831 18 : mBand->SetMetadata(meta);
1832 : }
1833 :
1834 : // Geotags
1835 122 : GDALGeoTransform gt;
1836 122 : if (CE_None == poSrcDS->GetGeoTransform(gt))
1837 119 : poDS->SetGeoTransform(gt);
1838 :
1839 122 : const auto poSRS = poSrcDS->GetSpatialRef();
1840 122 : if (poSRS)
1841 118 : poDS->m_oSRS = *poSRS;
1842 :
1843 : // Color palette if we only have one band
1844 225 : if (1 == nBands &&
1845 103 : GCI_PaletteIndex == poSrcBand1->GetColorInterpretation())
1846 1 : poDS->SetColorTable(poSrcBand1->GetColorTable()->Clone());
1847 :
1848 : // Finally write the XML in the right file name
1849 122 : if (!poDS->Crystalize())
1850 10 : throw CPLString("MRF: Error creating files");
1851 : }
1852 21 : catch (const CPLString &e)
1853 : {
1854 21 : if (nullptr != poDS)
1855 10 : delete poDS;
1856 21 : CPLError(CE_Failure, CPLE_ObjectNull, "%s", e.c_str());
1857 21 : poDS = nullptr;
1858 : }
1859 :
1860 133 : CSLDestroy(options);
1861 133 : if (nullptr == poDS)
1862 21 : return nullptr;
1863 :
1864 112 : char **papszFileList = poDS->GetFileList();
1865 112 : poDS->oOvManager.Initialize(poDS, poDS->GetPhysicalFilename(),
1866 : papszFileList);
1867 112 : CSLDestroy(papszFileList);
1868 :
1869 112 : CPLErr err = CE_None;
1870 : // Have PAM copy all, but skip the mask
1871 112 : int nCloneFlags = GCIF_PAM_DEFAULT & ~GCIF_MASK;
1872 :
1873 : // If copy is disabled, we're done, we just created an empty MRF
1874 112 : if (!on(CSLFetchNameValue(papszOptions, "NOCOPY")))
1875 : {
1876 : // Use the GDAL copy call
1877 : // Need to flag the dataset as compressed (COMPRESSED=TRUE) to force
1878 : // block writes This might not be what we want, if the input and out
1879 : // order is truly separate
1880 110 : nCloneFlags |= GCIF_MASK; // We do copy the data, so copy the mask too
1881 : // if necessary
1882 110 : char **papszCWROptions = nullptr;
1883 : papszCWROptions =
1884 110 : CSLAddNameValue(papszCWROptions, "COMPRESSED", "TRUE");
1885 :
1886 : #ifdef HAVE_JPEG
1887 : // Use the Zen version of the CopyWholeRaster if input has a dataset
1888 : // mask and JPEGs are generated
1889 112 : if (GMF_PER_DATASET == poSrcDS->GetRasterBand(1)->GetMaskFlags() &&
1890 2 : (poDS->current.comp == IL_JPEG
1891 : #ifdef HAVE_PNG
1892 0 : || poDS->current.comp == IL_JPNG
1893 : #endif
1894 : ))
1895 : {
1896 2 : err = poDS->ZenCopy(poSrcDS, pfnProgress, pProgressData);
1897 2 : nCloneFlags ^= GCIF_MASK; // Turn the external mask off
1898 : }
1899 : else
1900 : #endif
1901 : {
1902 108 : err = GDALDatasetCopyWholeRaster(
1903 : (GDALDatasetH)poSrcDS, (GDALDatasetH)poDS, papszCWROptions,
1904 : pfnProgress, pProgressData);
1905 : }
1906 :
1907 110 : CSLDestroy(papszCWROptions);
1908 : }
1909 :
1910 112 : if (CE_None == err)
1911 112 : err = poDS->CloneInfo(poSrcDS, nCloneFlags);
1912 :
1913 112 : if (CE_Failure == err)
1914 : {
1915 0 : delete poDS;
1916 0 : return nullptr;
1917 : }
1918 :
1919 112 : return poDS;
1920 : }
1921 :
1922 : // Prepares the data so it is suitable for Zen JPEG encoding, based on input
1923 : // mask If bFBO is set, only the values of the first band are set non-zero when
1924 : // needed
1925 : template <typename T>
1926 2 : static void ZenFilter(T *buffer, GByte *mask, int nPixels, int nBands,
1927 : bool bFBO)
1928 : {
1929 524290 : for (int i = 0; i < nPixels; i++)
1930 : {
1931 524288 : if (mask[i] == 0)
1932 : { // enforce zero values
1933 516096 : for (int b = 0; b < nBands; b++)
1934 387072 : buffer[nBands * i + b] = 0;
1935 : }
1936 : else
1937 : { // enforce non-zero
1938 395264 : if (bFBO)
1939 : { // First band only
1940 197632 : bool f = true;
1941 790528 : for (int b = 0; b < nBands; b++)
1942 : {
1943 592896 : if (0 == buffer[nBands * i + b])
1944 : {
1945 0 : f = false;
1946 0 : break;
1947 : }
1948 : }
1949 197632 : if (f)
1950 197632 : buffer[nBands * i] = 1;
1951 : }
1952 : else
1953 : { // Every band
1954 790528 : for (int b = 0; b < nBands; b++)
1955 592896 : if (0 == buffer[nBands * i + b])
1956 0 : buffer[nBands * i + b] = 1;
1957 : }
1958 : }
1959 : }
1960 2 : }
1961 :
1962 : // Custom CopyWholeRaster for Zen JPEG, called when the input has a PER_DATASET
1963 : // mask Works like GDALDatasetCopyWholeRaster, but it does filter the input data
1964 : // based on the mask
1965 : //
1966 2 : CPLErr MRFDataset::ZenCopy(GDALDataset *poSrc, GDALProgressFunc pfnProgress,
1967 : void *pProgressData)
1968 : {
1969 2 : VALIDATE_POINTER1(poSrc, "MRF:ZenCopy", CE_Failure);
1970 :
1971 2 : if (!pfnProgress)
1972 0 : pfnProgress = GDALDummyProgress;
1973 :
1974 : /* -------------------------------------------------------------------- */
1975 : /* Confirm the datasets match in size and band counts. */
1976 : /* -------------------------------------------------------------------- */
1977 2 : const int nXSize = GetRasterXSize();
1978 2 : const int nYSize = GetRasterYSize();
1979 2 : const int nBandCount = GetRasterCount();
1980 :
1981 2 : if (poSrc->GetRasterXSize() != nXSize ||
1982 4 : poSrc->GetRasterYSize() != nYSize ||
1983 2 : poSrc->GetRasterCount() != nBandCount)
1984 : {
1985 0 : CPLError(CE_Failure, CPLE_AppDefined,
1986 : "Input and output dataset sizes or band counts do not\n"
1987 : "match in GDALDatasetCopyWholeRaster()");
1988 0 : return CE_Failure;
1989 : }
1990 :
1991 : /* -------------------------------------------------------------------- */
1992 : /* Get our prototype band, and assume the others are similarly */
1993 : /* configured. Also get the per_dataset mask */
1994 : /* -------------------------------------------------------------------- */
1995 2 : GDALRasterBand *poSrcPrototypeBand = poSrc->GetRasterBand(1);
1996 2 : GDALRasterBand *poDstPrototypeBand = GetRasterBand(1);
1997 2 : GDALRasterBand *poSrcMask = poSrcPrototypeBand->GetMaskBand();
1998 :
1999 2 : const int nPageXSize = current.pagesize.x;
2000 2 : const int nPageYSize = current.pagesize.y;
2001 2 : const double nTotalBlocks =
2002 2 : static_cast<double>(DIV_ROUND_UP(nYSize, nPageYSize)) *
2003 2 : static_cast<double>(DIV_ROUND_UP(nXSize, nPageXSize));
2004 2 : const GDALDataType eDT = poDstPrototypeBand->GetRasterDataType();
2005 :
2006 : // All the bands are done per block
2007 : // this flag tells us to apply the Zen filter to the first band only
2008 2 : const bool bFirstBandOnly = (current.order == IL_Interleaved);
2009 :
2010 2 : if (!pfnProgress(0.0, nullptr, pProgressData))
2011 : {
2012 0 : CPLError(CE_Failure, CPLE_UserInterrupt,
2013 : "User terminated CreateCopy()");
2014 0 : return CE_Failure;
2015 : }
2016 :
2017 2 : const int nPixelCount = nPageXSize * nPageYSize;
2018 2 : const int dts = GDALGetDataTypeSizeBytes(eDT);
2019 2 : void *buffer = VSI_MALLOC3_VERBOSE(nPixelCount, nBandCount, dts);
2020 2 : GByte *buffer_mask = nullptr;
2021 2 : if (buffer)
2022 2 : buffer_mask = static_cast<GByte *>(VSI_MALLOC_VERBOSE(nPixelCount));
2023 :
2024 2 : if (!buffer || !buffer_mask)
2025 : {
2026 : // Just in case buffers did get allocated
2027 0 : CPLFree(buffer);
2028 0 : CPLFree(buffer_mask);
2029 0 : CPLError(CE_Failure, CPLE_OutOfMemory, "Can't allocate copy buffer");
2030 0 : return CE_Failure;
2031 : }
2032 :
2033 2 : int nBlocksDone = 0;
2034 2 : CPLErr eErr = CE_None;
2035 : // Advise the source that a complete read will be done
2036 2 : poSrc->AdviseRead(0, 0, nXSize, nYSize, nXSize, nYSize, eDT, nBandCount,
2037 2 : nullptr, nullptr);
2038 :
2039 : // For every block, break on error
2040 4 : for (int row = 0; row < nYSize && eErr == CE_None; row += nPageYSize)
2041 : {
2042 2 : int nRows = std::min(nPageYSize, nYSize - row);
2043 4 : for (int col = 0; col < nXSize && eErr == CE_None; col += nPageXSize)
2044 : {
2045 2 : int nCols = std::min(nPageXSize, nXSize - col);
2046 :
2047 : // Report
2048 2 : if (eErr == CE_None && !pfnProgress(nBlocksDone++ / nTotalBlocks,
2049 : nullptr, pProgressData))
2050 : {
2051 0 : eErr = CE_Failure;
2052 0 : CPLError(CE_Failure, CPLE_UserInterrupt,
2053 : "User terminated CreateCopy()");
2054 0 : break;
2055 : }
2056 :
2057 : // Get the data mask as byte
2058 2 : eErr = poSrcMask->RasterIO(GF_Read, col, row, nCols, nRows,
2059 : buffer_mask, nCols, nRows, GDT_UInt8, 0,
2060 : 0, nullptr);
2061 :
2062 2 : if (eErr != CE_None)
2063 0 : break;
2064 :
2065 : // If there is no data at all, skip this block
2066 2 : if (MatchCount(buffer_mask, nPixelCount, static_cast<GByte>(0)) ==
2067 : nPixelCount)
2068 0 : continue;
2069 :
2070 : // get the data in the buffer, interleaved
2071 4 : eErr = poSrc->RasterIO(
2072 : GF_Read, col, row, nCols, nRows, buffer, nCols, nRows, eDT,
2073 2 : nBandCount, nullptr, static_cast<GSpacing>(nBands) * dts,
2074 2 : static_cast<GSpacing>(nBands) * dts * nCols, dts, nullptr);
2075 :
2076 2 : if (eErr != CE_None)
2077 0 : break;
2078 :
2079 : // This is JPEG, only 8 and 12(16) bits unsigned integer types are
2080 : // valid
2081 2 : switch (eDT)
2082 : {
2083 2 : case GDT_UInt8:
2084 2 : ZenFilter(reinterpret_cast<GByte *>(buffer), buffer_mask,
2085 : nPixelCount, nBandCount, bFirstBandOnly);
2086 2 : break;
2087 0 : case GDT_UInt16:
2088 0 : ZenFilter(reinterpret_cast<GUInt16 *>(buffer), buffer_mask,
2089 : nPixelCount, nBandCount, bFirstBandOnly);
2090 0 : break;
2091 0 : default:
2092 0 : CPLError(CE_Failure, CPLE_AppDefined,
2093 : "Unsupported data type for Zen filter");
2094 0 : eErr = CE_Failure;
2095 0 : break;
2096 : }
2097 :
2098 : // Write
2099 2 : if (eErr == CE_None)
2100 2 : eErr = RasterIO(
2101 : GF_Write, col, row, nCols, nRows, buffer, nCols, nRows, eDT,
2102 2 : nBandCount, nullptr, static_cast<GSpacing>(nBands) * dts,
2103 2 : static_cast<GSpacing>(nBands) * dts * nCols, dts, nullptr);
2104 :
2105 : } // Columns
2106 2 : if (eErr != CE_None)
2107 0 : break;
2108 :
2109 : } // Rows
2110 :
2111 : // Cleanup
2112 2 : CPLFree(buffer);
2113 2 : CPLFree(buffer_mask);
2114 :
2115 : // Final report
2116 2 : if (eErr == CE_None && !pfnProgress(1.0, nullptr, pProgressData))
2117 : {
2118 0 : eErr = CE_Failure;
2119 0 : CPLError(CE_Failure, CPLE_UserInterrupt,
2120 : "User terminated CreateCopy()");
2121 : }
2122 :
2123 2 : return eErr;
2124 : }
2125 :
2126 : // Apply open options to the current dataset
2127 : // Called before the configuration is read
2128 202 : void MRFDataset::ProcessOpenOptions(CSLConstList papszOptions)
2129 : {
2130 202 : no_errors = CSLFetchBoolean(papszOptions, "NOERRORS", FALSE);
2131 202 : const char *val = CSLFetchNameValue(papszOptions, "ZSLICE");
2132 202 : if (val)
2133 0 : zslice = atoi(val);
2134 202 : }
2135 :
2136 : // Apply create options to the current dataset, only valid during creation
2137 162 : void MRFDataset::ProcessCreateOptions(CSLConstList papszOptions)
2138 : {
2139 162 : assert(!bCrystalized);
2140 324 : const CPLStringList opt(papszOptions);
2141 162 : ILImage &img(full);
2142 :
2143 162 : const char *val = opt.FetchNameValue("COMPRESS");
2144 162 : if (val && IL_ERR_COMP == (img.comp = CompToken(val)))
2145 0 : throw CPLString("GDAL MRF: Error setting compression");
2146 :
2147 162 : val = opt.FetchNameValue(GDALMD_INTERLEAVE);
2148 162 : if (val && IL_ERR_ORD == (img.order = OrderToken(val)))
2149 0 : throw CPLString("GDAL MRF: Error setting interleave");
2150 :
2151 162 : val = opt.FetchNameValue("QUALITY");
2152 162 : if (val)
2153 6 : img.quality = atoi(val);
2154 :
2155 162 : val = opt.FetchNameValue("ZSIZE");
2156 162 : if (val)
2157 0 : img.size.z = atoi(val);
2158 :
2159 162 : val = opt.FetchNameValue("BLOCKXSIZE");
2160 162 : if (val)
2161 1 : img.pagesize.x = atoi(val);
2162 :
2163 162 : val = opt.FetchNameValue("BLOCKYSIZE");
2164 162 : if (val)
2165 1 : img.pagesize.y = atoi(val);
2166 :
2167 162 : val = opt.FetchNameValue("BLOCKSIZE");
2168 162 : if (val)
2169 30 : img.pagesize.x = img.pagesize.y = atoi(val);
2170 :
2171 162 : img.nbo = opt.FetchBoolean("NETBYTEORDER", FALSE) != FALSE;
2172 :
2173 162 : val = opt.FetchNameValue("CACHEDSOURCE");
2174 162 : if (val)
2175 : {
2176 2 : if (IL_NONE == img.comp)
2177 0 : throw CPLString("GDAL MRF: Compression is required when caching");
2178 2 : source = val;
2179 2 : nocopy = opt.FetchBoolean("NOCOPY", FALSE);
2180 : }
2181 :
2182 162 : val = opt.FetchNameValue("UNIFORM_SCALE");
2183 162 : if (val)
2184 0 : scale = atoi(val);
2185 :
2186 162 : val = opt.FetchNameValue("PHOTOMETRIC");
2187 162 : if (val)
2188 2 : photometric = val;
2189 :
2190 162 : val = opt.FetchNameValue("DATANAME");
2191 162 : if (val)
2192 0 : img.datfname = val;
2193 :
2194 162 : val = opt.FetchNameValue("INDEXNAME");
2195 162 : if (val)
2196 0 : img.idxfname = val;
2197 :
2198 162 : val = opt.FetchNameValue("SPACING");
2199 162 : if (val)
2200 0 : spacing = atoi(val);
2201 :
2202 : optlist.Assign(
2203 : CSLTokenizeString2(opt.FetchNameValue("OPTIONS"), " \t\n\r",
2204 162 : CSLT_STRIPLEADSPACES | CSLT_STRIPENDSPACES));
2205 :
2206 : // General Fixups
2207 162 : if (img.order == IL_Interleaved)
2208 41 : img.pagesize.c = img.size.c;
2209 162 : }
2210 :
2211 : /**
2212 : *\brief Create an MRF dataset, some settings can be changed later
2213 : * papszOptions might be anything that an MRF might take
2214 : * Still missing are the georeference ...
2215 : *
2216 : */
2217 :
2218 166 : GDALDataset *MRFDataset::Create(const char *pszName, int nXSize, int nYSize,
2219 : int nBandsIn, GDALDataType eType,
2220 : CSLConstList papszOptions)
2221 : {
2222 166 : if (nBandsIn == 0)
2223 : {
2224 1 : CPLError(CE_Failure, CPLE_NotSupported, "No bands defined");
2225 1 : return nullptr;
2226 : }
2227 :
2228 165 : MRFDataset *poDS = new MRFDataset();
2229 165 : CPLErr err = CE_None;
2230 165 : poDS->fname = pszName;
2231 165 : poDS->nBands = nBandsIn;
2232 :
2233 : // Don't know what to do with these in this call
2234 : // int level = -1;
2235 : // int version = 0;
2236 :
2237 165 : size_t pos = poDS->fname.find(":MRF:");
2238 165 : if (string::npos != pos)
2239 : { // Tokenize and pick known options
2240 0 : vector<string> tokens;
2241 0 : stringSplit(tokens, poDS->fname, pos + 5, ':');
2242 : // level = getnum(tokens, 'L', -1);
2243 : // version = getnum(tokens, 'V', 0);
2244 0 : poDS->zslice = getnum(tokens, 'Z', 0);
2245 0 : poDS->fname.resize(pos); // Cut the ornamentations
2246 : }
2247 :
2248 : // Try creating the mrf file early, to avoid failing on Crystalize later
2249 165 : if (!STARTS_WITH(poDS->fname.c_str(), "<MRF_META>"))
2250 : {
2251 : // Try opening it first, even though we still clobber it later
2252 165 : VSILFILE *mainfile = VSIFOpenL(poDS->fname.c_str(), "r+b");
2253 165 : if (!mainfile)
2254 : { // Then try creating it
2255 115 : mainfile = VSIFOpenL(poDS->fname.c_str(), "w+b");
2256 115 : if (!mainfile)
2257 : {
2258 3 : CPLError(CE_Failure, CPLE_OpenFailed,
2259 : "MRF: Can't open %s for writing", poDS->fname.c_str());
2260 3 : delete poDS;
2261 3 : return nullptr;
2262 : }
2263 : }
2264 162 : VSIFCloseL(mainfile);
2265 : }
2266 :
2267 : // Use the full, set some initial parameters
2268 162 : ILImage &img = poDS->full;
2269 162 : img.size = ILSize(nXSize, nYSize, 1, nBandsIn);
2270 : #ifdef HAVE_PNG
2271 162 : img.comp = IL_PNG;
2272 : #else
2273 : img.comp = IL_NONE;
2274 : #endif
2275 162 : img.order = (nBandsIn < 5) ? IL_Interleaved : IL_Separate;
2276 162 : img.pagesize = ILSize(512, 512, 1, 1);
2277 162 : img.quality = 85;
2278 162 : img.dt = eType;
2279 162 : img.dataoffset = 0;
2280 162 : img.idxoffset = 0;
2281 162 : img.hasNoData = false;
2282 162 : img.nbo = false;
2283 :
2284 : // Set the guard that tells us it needs saving before IO can take place
2285 162 : poDS->bCrystalized = FALSE;
2286 :
2287 : // Process the options, anything that an MRF might take
2288 :
2289 : try
2290 : {
2291 : // Adjust the dataset and the full image
2292 162 : poDS->ProcessCreateOptions(papszOptions);
2293 :
2294 : // Set default file names
2295 162 : if (img.datfname.empty())
2296 162 : img.datfname = getFname(poDS->GetFname(), ILComp_Ext[img.comp]);
2297 162 : if (img.idxfname.empty())
2298 162 : img.idxfname = getFname(poDS->GetFname(), ".idx");
2299 :
2300 162 : poDS->eAccess = GA_Update;
2301 : }
2302 :
2303 0 : catch (const CPLString &e)
2304 : {
2305 0 : CPLError(CE_Failure, CPLE_OpenFailed, "%s", e.c_str());
2306 0 : delete poDS;
2307 0 : return nullptr;
2308 : }
2309 :
2310 162 : poDS->current = poDS->full;
2311 162 : poDS->SetDescription(poDS->GetFname());
2312 :
2313 : // Build a MRF XML and initialize from it, this creates the bands
2314 162 : CPLXMLNode *config = poDS->BuildConfig();
2315 162 : err = poDS->Initialize(config);
2316 162 : CPLDestroyXMLNode(config);
2317 :
2318 162 : if (CPLE_None != err)
2319 : {
2320 30 : delete poDS;
2321 30 : return nullptr;
2322 : }
2323 :
2324 : // If not set by the band, get a pageSizeBytes buffer
2325 132 : if (poDS->GetPBufferSize() == 0 &&
2326 0 : !poDS->SetPBuffer(poDS->current.pageSizeBytes))
2327 : {
2328 0 : delete poDS;
2329 0 : return nullptr;
2330 : }
2331 :
2332 : // Tell PAM what our real file name is, to help it find the aux.xml
2333 132 : poDS->SetPhysicalFilename(poDS->GetFname());
2334 132 : return poDS;
2335 : }
2336 :
2337 172 : int MRFDataset::Crystalize()
2338 : {
2339 172 : if (bCrystalized || eAccess != GA_Update)
2340 : {
2341 0 : bCrystalized = TRUE;
2342 0 : return TRUE;
2343 : }
2344 :
2345 : // No need to write to disk if there is no filename. This is a
2346 : // memory only dataset.
2347 344 : if (strlen(GetDescription()) == 0 ||
2348 172 : EQUALN(GetDescription(), "<MRF_META>", 10))
2349 : {
2350 0 : bCrystalized = TRUE;
2351 0 : return TRUE;
2352 : }
2353 :
2354 172 : CPLXMLNode *config = BuildConfig();
2355 172 : if (!WriteConfig(config))
2356 20 : return FALSE;
2357 152 : CPLDestroyXMLNode(config);
2358 152 : if (!nocopy && (!IdxFP() || !DataFP()))
2359 0 : return FALSE;
2360 152 : bCrystalized = TRUE;
2361 152 : return TRUE;
2362 : }
2363 :
2364 : // Copy the first index at the end of the file and bump the version count
2365 1 : CPLErr MRFDataset::AddVersion()
2366 : {
2367 1 : VSILFILE *l_ifp = IdxFP();
2368 1 : void *tbuff = CPLMalloc(static_cast<size_t>(idxSize));
2369 1 : VSIFSeekL(l_ifp, 0, SEEK_SET);
2370 1 : VSIFReadL(tbuff, 1, static_cast<size_t>(idxSize), l_ifp);
2371 1 : verCount++; // The one we write
2372 1 : VSIFSeekL(l_ifp, static_cast<vsi_l_offset>(idxSize) * verCount,
2373 : SEEK_SET); // At the end, this can mess things up royally
2374 1 : VSIFWriteL(tbuff, 1, static_cast<size_t>(idxSize), l_ifp);
2375 1 : CPLFree(tbuff);
2376 1 : return CE_None;
2377 : }
2378 :
2379 : //
2380 : // Write a tile at the end of the data file
2381 : // If buff and size are zero, it is equivalent to erasing the tile
2382 : // If only size is zero, it is a special empty tile,
2383 : // when used for caching, offset should be 1
2384 : //
2385 : // To make it multi-processor safe, open the file in append mode
2386 : // and verify after write
2387 : //
2388 5096 : CPLErr MRFDataset::WriteTile(void *buff, GUIntBig infooffset, GUIntBig size)
2389 : {
2390 5096 : CPLErr ret = CE_None;
2391 5096 : ILIdx tinfo = {0, 0};
2392 :
2393 5096 : VSILFILE *l_dfp = DataFP();
2394 5096 : VSILFILE *l_ifp = IdxFP();
2395 :
2396 : // Verify buffer
2397 10192 : std::vector<GByte> tbuff;
2398 :
2399 5096 : if (l_ifp == nullptr || l_dfp == nullptr)
2400 0 : return CE_Failure;
2401 :
2402 : // Flag that versioned access requires a write even if empty
2403 5096 : int new_tile = false;
2404 : // If it has versions, might need to start a new one
2405 5096 : if (hasVersions)
2406 : {
2407 1 : int new_version = false; // Assume no need to build new version
2408 :
2409 : // Read the current tile info
2410 1 : VSIFSeekL(l_ifp, infooffset, SEEK_SET);
2411 1 : VSIFReadL(&tinfo, 1, sizeof(ILIdx), l_ifp);
2412 :
2413 1 : if (verCount == 0)
2414 1 : new_version = true; // No previous yet, might create a new version
2415 : else
2416 : { // We need at least two versions before we can test for changes
2417 0 : ILIdx prevtinfo = {0, 0};
2418 :
2419 : // Read the previous one
2420 0 : VSIFSeekL(l_ifp, infooffset + verCount * idxSize, SEEK_SET);
2421 0 : VSIFReadL(&prevtinfo, 1, sizeof(ILIdx), l_ifp);
2422 :
2423 : // current and previous tiles are different, might create version
2424 0 : if (tinfo.size != prevtinfo.size ||
2425 0 : tinfo.offset != prevtinfo.offset)
2426 0 : new_version = true;
2427 : }
2428 :
2429 : // tinfo contains the current info or 0,0
2430 1 : if (tinfo.size == GIntBig(net64(size)))
2431 : { // Might be identical
2432 0 : if (size != 0)
2433 : {
2434 : // Use the temporary buffer
2435 0 : tbuff.resize(static_cast<size_t>(size));
2436 0 : VSIFSeekL(l_dfp, infooffset, SEEK_SET);
2437 0 : VSIFReadL(tbuff.data(), 1, tbuff.size(), l_dfp);
2438 : // Need to write it if not the same
2439 0 : new_tile = !std::equal(tbuff.begin(), tbuff.end(),
2440 : static_cast<GByte *>(buff));
2441 0 : tbuff.clear();
2442 : }
2443 : else
2444 : {
2445 : // Writing a null tile on top of a null tile, does it count?
2446 0 : if (tinfo.offset != GIntBig(net64(GUIntBig(buff))))
2447 0 : new_tile = true;
2448 : }
2449 : }
2450 : else
2451 : {
2452 1 : new_tile = true; // Need to write it because it is different
2453 1 : if (verCount == 0 && tinfo.size == 0)
2454 0 : new_version = false; // Don't create a version if current is
2455 : // empty and there is no previous
2456 : }
2457 :
2458 1 : if (!new_tile)
2459 0 : return CE_None; // No reason to write
2460 :
2461 : // Do we need to start a new version before writing the tile?
2462 1 : if (new_version)
2463 1 : AddVersion();
2464 : }
2465 :
2466 5096 : bool same = true;
2467 5096 : if (size)
2468 0 : do
2469 : {
2470 : // start of critical MP section
2471 5085 : VSIFSeekL(l_dfp, 0, SEEK_END);
2472 5085 : GUIntBig offset = VSIFTellL(l_dfp) + spacing;
2473 :
2474 : // Spacing should be 0 in MP safe mode, this doesn't have much of
2475 : // effect Use the existing data, spacing content is not guaranteed
2476 5085 : for (GUIntBig pending = spacing; pending != 0;
2477 0 : pending -= std::min(pending, size))
2478 0 : VSIFWriteL(buff, 1,
2479 0 : static_cast<size_t>(std::min(pending, size)),
2480 : l_dfp); // Usually only once
2481 :
2482 5085 : if (static_cast<size_t>(size) !=
2483 5085 : VSIFWriteL(buff, 1, static_cast<size_t>(size), l_dfp))
2484 0 : ret = CE_Failure;
2485 : // End of critical section
2486 :
2487 5085 : tinfo.offset = net64(offset);
2488 : //
2489 : // For MP ops, check that we can read the same content, otherwise
2490 : // try again This makes the caching MRF MP safe on file systems that
2491 : // implement append mode fully, without using explicit locks
2492 : //
2493 5085 : if (CE_None == ret && mp_safe)
2494 : { // readback and check
2495 3 : if (tbuff.size() < size)
2496 3 : tbuff.resize(static_cast<size_t>(size));
2497 3 : VSIFSeekL(l_dfp, offset, SEEK_SET);
2498 3 : VSIFReadL(tbuff.data(), 1, tbuff.size(), l_dfp);
2499 3 : same = std::equal(tbuff.begin(), tbuff.end(),
2500 : static_cast<GByte *>(buff));
2501 : }
2502 5085 : } while (CE_None == ret && mp_safe && !same);
2503 :
2504 5096 : if (CE_None != ret)
2505 : {
2506 0 : CPLError(CE_Failure, CPLE_AppDefined, "MRF: Tile write failed");
2507 0 : return ret;
2508 : }
2509 :
2510 : // Convert index to net format, offset is set already
2511 5096 : tinfo.size = net64(size);
2512 : // Do nothing if the tile is empty and the file record is also empty
2513 5096 : if (!new_tile && 0 == size && nullptr == buff)
2514 : {
2515 10 : VSIFSeekL(l_ifp, infooffset, SEEK_SET);
2516 10 : VSIFReadL(&tinfo, 1, sizeof(ILIdx), l_ifp);
2517 10 : if (0 == tinfo.offset && 0 == tinfo.size)
2518 10 : return ret;
2519 : }
2520 :
2521 : // Special case, any non-zero offset will do
2522 5086 : if (nullptr != buff && 0 == size)
2523 0 : tinfo.offset = ~GUIntBig(0);
2524 :
2525 5086 : VSIFSeekL(l_ifp, infooffset, SEEK_SET);
2526 5086 : if (sizeof(tinfo) != VSIFWriteL(&tinfo, 1, sizeof(tinfo), l_ifp))
2527 : {
2528 0 : CPLError(CE_Failure, CPLE_AppDefined, "MRF: Index write failed");
2529 0 : ret = CE_Failure;
2530 : }
2531 :
2532 5086 : return ret;
2533 : }
2534 :
2535 128 : CPLErr MRFDataset::SetGeoTransform(const GDALGeoTransform >)
2536 : {
2537 128 : if (GetAccess() != GA_Update || bCrystalized)
2538 : {
2539 0 : CPLError(CE_Failure, CPLE_NotSupported,
2540 : "SetGeoTransform only works during Create call");
2541 0 : return CE_Failure;
2542 : }
2543 128 : m_gt = gt;
2544 128 : bGeoTransformValid = TRUE;
2545 128 : return CE_None;
2546 : }
2547 :
2548 12 : bool MRFDataset::IsSingleTile()
2549 : {
2550 12 : if (current.pagecount.l != 1 || !source.empty() || nullptr == DataFP())
2551 4 : return FALSE;
2552 8 : return 0 == cpl::down_cast<MRFRasterBand *>(GetRasterBand(1))
2553 8 : ->GetOverviewCount();
2554 : }
2555 :
2556 : /*
2557 : * Returns 0,1,0,0,0,1 even if it was not set
2558 : */
2559 455 : CPLErr MRFDataset::GetGeoTransform(GDALGeoTransform >) const
2560 : {
2561 455 : gt = m_gt;
2562 455 : MRFDataset *nonConstThis = const_cast<MRFDataset *>(this);
2563 455 : if (nonConstThis->GetMetadata(GDAL_MDD_RPC) || nonConstThis->GetGCPCount())
2564 0 : bGeoTransformValid = FALSE;
2565 455 : if (!bGeoTransformValid)
2566 0 : return CE_Failure;
2567 455 : return CE_None;
2568 : }
2569 :
2570 : /**
2571 : *\brief Read a tile index
2572 : *
2573 : * It handles the non-existent index case, for no compression
2574 : * The bias is non-zero only when the cloned index is read
2575 : */
2576 :
2577 2018 : CPLErr MRFDataset::ReadTileIdx(ILIdx &tinfo, const ILSize &pos,
2578 : const ILImage &img, const GIntBig bias)
2579 : {
2580 2018 : VSILFILE *l_ifp = IdxFP();
2581 :
2582 : // Initialize the tinfo structure, in case the files are missing
2583 2018 : if (missing)
2584 0 : return CE_None;
2585 :
2586 2018 : GIntBig offset = bias + IdxOffset(pos, img);
2587 2018 : if (l_ifp == nullptr && img.comp == IL_NONE)
2588 : {
2589 0 : tinfo.size = current.pageSizeBytes;
2590 0 : tinfo.offset = offset * tinfo.size;
2591 0 : return CE_None;
2592 : }
2593 :
2594 2018 : if (l_ifp == nullptr && IsSingleTile())
2595 : {
2596 4 : tinfo.offset = 0;
2597 4 : VSILFILE *l_dfp = DataFP(); // IsSingleTile() checks that fp is valid
2598 4 : VSIFSeekL(l_dfp, 0, SEEK_END);
2599 4 : tinfo.size = VSIFTellL(l_dfp);
2600 :
2601 : // It should be less than the pagebuffer
2602 4 : tinfo.size = std::min(tinfo.size, static_cast<GIntBig>(pbsize));
2603 4 : return CE_None;
2604 : }
2605 :
2606 2014 : if (l_ifp == nullptr)
2607 : {
2608 0 : CPLError(CE_Failure, CPLE_FileIO, "Can't open index file");
2609 0 : return CE_Failure;
2610 : }
2611 :
2612 2014 : VSIFSeekL(l_ifp, static_cast<vsi_l_offset>(offset), SEEK_SET);
2613 2014 : if (1 != VSIFReadL(&tinfo, sizeof(ILIdx), 1, l_ifp))
2614 0 : return CE_Failure;
2615 : // Convert them to native form
2616 2014 : tinfo.offset = net64(tinfo.offset);
2617 2014 : tinfo.size = net64(tinfo.size);
2618 :
2619 2014 : if (0 == bias || 0 != tinfo.size || 0 != tinfo.offset)
2620 2013 : return CE_None;
2621 :
2622 : // zero size and zero offset in sourced index means that this portion is
2623 : // un-initialized
2624 :
2625 : // Should be cloned and the offset within the cloned index
2626 1 : offset -= bias;
2627 1 : assert(offset < bias);
2628 1 : assert(clonedSource);
2629 :
2630 : // Read this block from the remote index, prepare it and store it in the
2631 : // right place The block size in bytes, should be a multiple of 16, to have
2632 : // full index entries
2633 1 : const int CPYSZ = 32768;
2634 : // Adjust offset to the start of the block
2635 1 : offset = (offset / CPYSZ) * CPYSZ;
2636 1 : GIntBig size = std::min(size_t(CPYSZ), size_t(bias - offset));
2637 1 : size /= sizeof(ILIdx); // In records
2638 2 : vector<ILIdx> buf(static_cast<size_t>(size));
2639 1 : ILIdx *buffer = &buf[0]; // Buffer to copy the source to the clone index
2640 :
2641 : // Fetch the data from the cloned index
2642 1 : MRFDataset *pSrc = static_cast<MRFDataset *>(GetSrcDS());
2643 1 : if (nullptr == pSrc)
2644 : {
2645 0 : CPLError(CE_Failure, CPLE_FileIO, "Can't open cloned source index");
2646 0 : return CE_Failure; // Source reported the error
2647 : }
2648 :
2649 1 : VSILFILE *srcidx = pSrc->IdxFP();
2650 1 : if (nullptr == srcidx)
2651 : {
2652 0 : CPLError(CE_Failure, CPLE_FileIO, "Can't open cloned source index");
2653 0 : return CE_Failure; // Source reported the error
2654 : }
2655 :
2656 1 : VSIFSeekL(srcidx, static_cast<vsi_l_offset>(offset), SEEK_SET);
2657 1 : size = VSIFReadL(buffer, sizeof(ILIdx), static_cast<size_t>(size), srcidx);
2658 1 : if (size != GIntBig(buf.size()))
2659 : {
2660 0 : CPLError(CE_Failure, CPLE_FileIO, "Can't read cloned source index");
2661 0 : return CE_Failure; // Source reported the error
2662 : }
2663 :
2664 : // Mark the empty records as checked, by making the offset non-zero
2665 2 : for (vector<ILIdx>::iterator it = buf.begin(); it != buf.end(); ++it)
2666 : {
2667 1 : if (it->offset == 0 && it->size == 0)
2668 0 : it->offset = net64(1);
2669 : }
2670 :
2671 : // Write it in the right place in the local index file
2672 1 : VSIFSeekL(l_ifp, static_cast<vsi_l_offset>(bias + offset), SEEK_SET);
2673 1 : size = VSIFWriteL(&buf[0], sizeof(ILIdx), static_cast<size_t>(size), l_ifp);
2674 1 : if (size != GIntBig(buf.size()))
2675 : {
2676 0 : CPLError(CE_Failure, CPLE_FileIO, "Can't write to cloning MRF index");
2677 0 : return CE_Failure; // Source reported the error
2678 : }
2679 :
2680 : // Cloned index updated, restart this function, it will work now
2681 1 : return ReadTileIdx(tinfo, pos, img, bias);
2682 : }
2683 :
2684 : NAMESPACE_MRF_END
|
