Line data Source code
1 : /*
2 : * Copyright (c) 2002-2012, California Institute of Technology.
3 : * All rights reserved. Based on Government Sponsored Research under contracts
4 : * NAS7-1407 and/or NAS7-03001.
5 : *
6 : * Redistribution and use in source and binary forms, with or without
7 : * modification, are permitted provided that the following conditions are met:
8 : * 1. Redistributions of source code must retain the above copyright notice,
9 : * this list of conditions and the following disclaimer.
10 : * 2. Redistributions in binary form must reproduce the above copyright
11 : * notice, this list of conditions and the following disclaimer in the
12 : * documentation and/or other materials provided with the distribution.
13 : * 3. Neither the name of the California Institute of Technology (Caltech),
14 : * its operating division the Jet Propulsion Laboratory (JPL), the National
15 : * Aeronautics and Space Administration (NASA), nor the names of its
16 : * contributors may be used to endorse or promote products derived from this
17 : * software without specific prior written permission.
18 : *
19 : * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20 : * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 : * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 : * ARE DISCLAIMED. IN NO EVENT SHALL THE CALIFORNIA INSTITUTE OF TECHNOLOGY BE
23 : * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 : * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 : * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 : * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 : * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 : * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 : * POSSIBILITY OF SUCH DAMAGE.
30 : *
31 : * Portions copyright 2014-2021 Esri
32 : *
33 : * Licensed under the Apache License, Version 2.0 (the "License");
34 : * you may not use this file except in compliance with the License.
35 : * You may obtain a copy of the License at
36 : *
37 : * http://www.apache.org/licenses/LICENSE-2.0
38 : *
39 : * Unless required by applicable law or agreed to in writing, software
40 : * distributed under the License is distributed on an "AS IS" BASIS,
41 : * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
42 : * See the License for the specific language governing permissions and
43 : * limitations under the License.
44 : */
45 :
46 : /******************************************************************************
47 : *
48 : * Project: Meta Raster File Format Driver Implementation, Dataset
49 : * Purpose: Implementation of GDAL dataset
50 : *
51 : * Author: Lucian Plesea, Lucian.Plesea jpl.nasa.gov, lplesea esri.com
52 : *
53 : ******************************************************************************
54 : *
55 : * The MRF dataset and the band are closely tied together, they should be
56 : * considered a single class, or a class (dataset) with extensions (bands).
57 : *
58 : *
59 : ****************************************************************************/
60 :
61 : #include "marfa.h"
62 : #include "mrfdrivercore.h"
63 : #include "cpl_multiproc.h" /* for CPLSleep() */
64 : #include "gdal_priv.h"
65 : #include <assert.h>
66 :
67 : #include <algorithm>
68 : #include <limits>
69 : #include <vector>
70 : #if defined(ZSTD_SUPPORT)
71 : #include <zstd.h>
72 : #endif
73 : using std::string;
74 : using std::vector;
75 :
76 : NAMESPACE_MRF_START
77 :
78 : // Initialize as invalid
79 368 : MRFDataset::MRFDataset()
80 : : zslice(0), idxSize(0), clonedSource(FALSE), nocopy(FALSE),
81 : bypass_cache(
82 368 : CPLTestBool(CPLGetConfigOption("MRF_BYPASSCACHING", "FALSE"))),
83 : mp_safe(FALSE), hasVersions(FALSE), verCount(0),
84 : bCrystalized(TRUE), // Assume not in create mode
85 : spacing(0), no_errors(0), missing(0), poSrcDS(nullptr), level(-1),
86 : cds(nullptr), scale(0.0), pbuffer(nullptr), pbsize(0), tile(ILSize()),
87 : bdirty(0), bGeoTransformValid(TRUE), poColorTable(nullptr), Quality(0),
88 736 : pzscctx(nullptr), pzsdctx(nullptr), read_timer(), write_timer(0)
89 : {
90 368 : m_oSRS.SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER);
91 : // X0 Xx Xy Y0 Yx Yy
92 368 : double gt[6] = {0.0, 1.0, 0.0, 0.0, 0.0, 1.0};
93 :
94 368 : memcpy(GeoTransform, gt, sizeof(gt));
95 368 : ifp.FP = dfp.FP = nullptr;
96 368 : dfp.acc = GF_Read;
97 368 : ifp.acc = GF_Read;
98 368 : }
99 :
100 180 : bool MRFDataset::SetPBuffer(unsigned int sz)
101 : {
102 180 : if (sz == 0)
103 : {
104 0 : CPLFree(pbuffer);
105 0 : pbuffer = nullptr;
106 : }
107 180 : void *pbufferNew = VSIRealloc(pbuffer, sz);
108 180 : if (pbufferNew == nullptr)
109 : {
110 0 : CPLError(CE_Failure, CPLE_OutOfMemory, "Cannot allocate %u bytes", sz);
111 0 : return false;
112 : }
113 180 : pbuffer = pbufferNew;
114 180 : pbsize = sz;
115 180 : return true;
116 : }
117 :
118 256 : static GDALColorEntry GetXMLColorEntry(CPLXMLNode *p)
119 : {
120 : GDALColorEntry ce;
121 256 : ce.c1 = static_cast<short>(getXMLNum(p, "c1", 0));
122 256 : ce.c2 = static_cast<short>(getXMLNum(p, "c2", 0));
123 256 : ce.c3 = static_cast<short>(getXMLNum(p, "c3", 0));
124 256 : ce.c4 = static_cast<short>(getXMLNum(p, "c4", 255));
125 256 : return ce;
126 : }
127 :
128 : //
129 : // Called by dataset destructor or at GDAL termination, to avoid
130 : // closing datasets whose drivers have already been unloaded
131 : //
132 368 : int MRFDataset::CloseDependentDatasets()
133 : {
134 368 : int bHasDroppedRef = GDALPamDataset::CloseDependentDatasets();
135 :
136 368 : if (poSrcDS)
137 : {
138 3 : bHasDroppedRef = TRUE;
139 3 : GDALClose(GDALDataset::ToHandle(poSrcDS));
140 3 : poSrcDS = nullptr;
141 : }
142 :
143 368 : if (cds)
144 : {
145 2 : bHasDroppedRef = TRUE;
146 2 : GDALClose(GDALDataset::ToHandle(cds));
147 2 : cds = nullptr;
148 : }
149 :
150 368 : return bHasDroppedRef;
151 : }
152 :
153 736 : MRFDataset::~MRFDataset()
154 : { // Make sure everything gets written
155 368 : if (eAccess != GA_ReadOnly && !bCrystalized)
156 50 : if (!MRFDataset::Crystalize())
157 : {
158 : // Can't return error code from a destructor, just emit the error
159 10 : CPLError(CE_Failure, CPLE_FileIO, "Error creating files");
160 : }
161 :
162 368 : MRFDataset::FlushCache(true);
163 368 : MRFDataset::CloseDependentDatasets();
164 :
165 368 : if (ifp.FP)
166 261 : VSIFCloseL(ifp.FP);
167 368 : if (dfp.FP)
168 264 : VSIFCloseL(dfp.FP);
169 :
170 368 : delete poColorTable;
171 :
172 : // CPLFree ignores being called with NULL
173 368 : CPLFree(pbuffer);
174 368 : pbsize = 0;
175 : #if defined(ZSTD_SUPPORT)
176 368 : ZSTD_freeCCtx(static_cast<ZSTD_CCtx *>(pzscctx));
177 368 : ZSTD_freeDCtx(static_cast<ZSTD_DCtx *>(pzsdctx));
178 : #endif
179 : // total time spend doing compression and decompression
180 368 : if (0 != write_timer.count())
181 111 : CPLDebug("MRF_Timing", "Compression took %fms",
182 111 : 1e-6 * write_timer.count());
183 :
184 368 : if (0 != read_timer.count())
185 141 : CPLDebug("MRF_Timing", "Decompression took %fms",
186 141 : 1e-6 * read_timer.count());
187 736 : }
188 :
189 : /*
190 : *\brief Format specific RasterIO, may be bypassed by BlockBasedRasterIO by
191 : *setting GDAL_FORCE_CACHING to Yes, in which case the band ReadBlock and
192 : *WriteBLock are called directly
193 : */
194 139 : CPLErr MRFDataset::IRasterIO(GDALRWFlag eRWFlag, int nXOff, int nYOff,
195 : int nXSize, int nYSize, void *pData, int nBufXSize,
196 : int nBufYSize, GDALDataType eBufType,
197 : int nBandCount, BANDMAP_TYPE panBandMap,
198 : GSpacing nPixelSpace, GSpacing nLineSpace,
199 : GSpacing nBandSpace,
200 : GDALRasterIOExtraArg *psExtraArgs)
201 : {
202 139 : CPLDebug("MRF_IO",
203 : "IRasterIO %s, %d, %d, %d, %d, bufsz %d,%d,%d strides P %d, L %d, "
204 : "B %d \n",
205 : eRWFlag == GF_Write ? "Write" : "Read", nXOff, nYOff, nXSize,
206 : nYSize, nBufXSize, nBufYSize, nBandCount,
207 : static_cast<int>(nPixelSpace), static_cast<int>(nLineSpace),
208 : static_cast<int>(nBandSpace));
209 :
210 139 : if (eRWFlag == GF_Write && !bCrystalized && !Crystalize())
211 : {
212 0 : CPLError(CE_Failure, CPLE_FileIO, "MRF: Error creating files");
213 0 : return CE_Failure;
214 : }
215 :
216 : //
217 : // Call the parent implementation, which splits it into bands and calls
218 : // their IRasterIO
219 : //
220 139 : return GDALPamDataset::IRasterIO(eRWFlag, nXOff, nYOff, nXSize, nYSize,
221 : pData, nBufXSize, nBufYSize, eBufType,
222 : nBandCount, panBandMap, nPixelSpace,
223 139 : nLineSpace, nBandSpace, psExtraArgs);
224 : }
225 :
226 : /**
227 : *\brief Build some overviews
228 : *
229 : * if nOverviews is 0, erase the overviews (reduce to base image only)
230 : */
231 :
232 31 : CPLErr MRFDataset::IBuildOverviews(const char *pszResampling, int nOverviews,
233 : const int *panOverviewList, int nBandsIn,
234 : const int *panBandList,
235 : GDALProgressFunc pfnProgress,
236 : void *pProgressData,
237 : CSLConstList papszOptions)
238 :
239 : {
240 31 : if (pfnProgress == nullptr)
241 0 : pfnProgress = GDALDummyProgress;
242 :
243 31 : CPLErr eErr = CE_None;
244 31 : CPLDebug("MRF_OVERLAY", "IBuildOverviews %d, bands %d\n", nOverviews,
245 : nBandsIn);
246 :
247 31 : if (nBands != nBandsIn)
248 : {
249 0 : CPLError(CE_Failure, CPLE_NotSupported, "nBands = %d not supported",
250 : nBandsIn);
251 0 : return CE_Failure;
252 : }
253 :
254 : // If we don't have write access, then create external overviews
255 31 : if (GetAccess() != GA_Update)
256 : {
257 1 : CPLDebug("MRF", "File open read-only, creating overviews externally.");
258 1 : return GDALDataset::IBuildOverviews(
259 : pszResampling, nOverviews, panOverviewList, nBands, panBandList,
260 1 : pfnProgress, pProgressData, papszOptions);
261 : }
262 :
263 30 : if (nOverviews == 0)
264 : {
265 : // If there are none, nothing to do
266 0 : if (GetRasterBand(1)->GetOverviewCount() == 0)
267 0 : return CE_None;
268 :
269 0 : auto *b = static_cast<MRFRasterBand *>(GetRasterBand(1));
270 : // If the first band internal overviews don't exist, they are external
271 0 : if (b->overviews.empty())
272 0 : return GDALDataset::IBuildOverviews(
273 : pszResampling, nOverviews, panOverviewList, nBands, panBandList,
274 0 : pfnProgress, pProgressData, papszOptions);
275 :
276 : // We should clean overviews, but this is not allowed in an MRF
277 0 : CPLError(CE_Warning, CPLE_NotSupported,
278 : "MRF: Internal overviews cannot be removed, "
279 : "but they can be rebuilt");
280 0 : return CE_None;
281 : }
282 :
283 30 : std::vector<int> panOverviewListNew(nOverviews);
284 61 : for (int i = 0; i < nOverviews; i++)
285 31 : panOverviewListNew[i] = panOverviewList[i];
286 : try
287 : { // Throw an error code, to make sure memory gets freed properly
288 : // Modify the metadata file if it doesn't already have the Rset model
289 : // set
290 30 : if (0.0 == scale)
291 : {
292 30 : CPLXMLNode *config = ReadConfig();
293 : try
294 : {
295 : const char *model =
296 30 : CPLGetXMLValue(config, "Rsets.model", "uniform");
297 30 : if (!EQUAL(model, "uniform"))
298 : {
299 0 : CPLError(CE_Failure, CPLE_AppDefined,
300 : "MRF:IBuildOverviews, Overviews not implemented "
301 : "for model %s",
302 : model);
303 0 : throw CE_Failure;
304 : }
305 :
306 : // The scale value is the same as first overview
307 30 : scale =
308 30 : strtod(CPLGetXMLValue(
309 : config, "Rsets.scale",
310 60 : CPLOPrintf("%d", panOverviewList[0]).c_str()),
311 : nullptr);
312 30 : if (scale == 0.0)
313 : {
314 0 : CPLError(CE_Failure, CPLE_IllegalArg,
315 : "Invalid Rsets.scale value");
316 0 : throw CE_Failure;
317 : }
318 :
319 30 : if (static_cast<int>(scale) != 2 &&
320 0 : (EQUALN("Avg", pszResampling, 3) ||
321 0 : EQUALN("Nnb", pszResampling, 3)))
322 : {
323 0 : CPLError(CE_Failure, CPLE_IllegalArg,
324 : "MRF internal resampling requires a scale factor "
325 : "of two");
326 0 : throw CE_Failure;
327 : }
328 :
329 : // Initialize the empty overlays, all of them for a given scale
330 : // They could already exist, in which case they are not erased
331 30 : idxSize = AddOverviews(int(scale));
332 :
333 : // If we don't have overviews, don't try to generate them
334 30 : if (GetRasterBand(1)->GetOverviewCount() == 0)
335 0 : throw CE_None;
336 :
337 30 : if (!CheckFileSize(current.idxfname, idxSize, GA_Update))
338 : {
339 0 : CPLError(CE_Failure, CPLE_AppDefined,
340 : "MRF: Can't extend index file");
341 0 : throw CE_Failure;
342 : }
343 :
344 : // Set the uniform node, in case it was not set before, and
345 : // save the new configuration
346 30 : CPLSetXMLValue(config, "Rsets.#model", "uniform");
347 30 : CPLSetXMLValue(config, "Rsets.#scale", PrintDouble(scale));
348 :
349 30 : if (!WriteConfig(config))
350 : {
351 0 : CPLError(CE_Failure, CPLE_AppDefined,
352 : "MRF: Can't rewrite the metadata file");
353 0 : throw CE_Failure;
354 : }
355 30 : CPLDestroyXMLNode(config);
356 30 : config = nullptr;
357 : }
358 0 : catch (const CPLErr &)
359 : {
360 0 : CPLDestroyXMLNode(config);
361 0 : throw; // Rethrow
362 : }
363 :
364 : // To avoid issues with blacks overviews, generate all of them
365 : // if the user asked for a couple of overviews in the correct
366 : // sequence and starting with the lowest one
367 90 : if (!EQUAL(pszResampling, "NONE") &&
368 32 : nOverviews != GetRasterBand(1)->GetOverviewCount() &&
369 2 : CPLTestBool(
370 : CPLGetConfigOption("MRF_ALL_OVERVIEW_LEVELS", "YES")))
371 : {
372 2 : bool bIncreasingPowers =
373 2 : (panOverviewList[0] == static_cast<int>(scale));
374 3 : for (int i = 1; i < nOverviews; i++)
375 1 : bIncreasingPowers =
376 2 : bIncreasingPowers &&
377 1 : (panOverviewList[i] ==
378 1 : static_cast<int>(scale * panOverviewList[i - 1]));
379 :
380 2 : int ovrcount = GetRasterBand(1)->GetOverviewCount();
381 2 : if (bIncreasingPowers && nOverviews != ovrcount)
382 : {
383 2 : CPLDebug("MRF",
384 : "Generating %d levels instead of the %d requested",
385 : ovrcount, nOverviews);
386 2 : nOverviews = ovrcount;
387 2 : panOverviewListNew.resize(ovrcount);
388 7 : for (int i = 0; i < ovrcount; i++)
389 5 : panOverviewListNew[i] = int(std::pow(scale, i + 1));
390 : }
391 : }
392 : }
393 :
394 30 : if (static_cast<int>(scale) != 2 && (EQUALN("Avg", pszResampling, 3) ||
395 0 : EQUALN("Nnb", pszResampling, 3)))
396 : {
397 0 : CPLError(CE_Failure, CPLE_IllegalArg,
398 : "MRF internal resampling requires a scale factor of two");
399 0 : throw CE_Failure;
400 : }
401 :
402 : // First pass, count the pixels to be processed, mark invalid levels
403 : // Smallest positive value to avoid Coverity Scan complaining about
404 : // division by zero
405 30 : double dfTotalPixels = std::numeric_limits<double>::min();
406 30 : double dfPixels = double(nRasterXSize) * nRasterYSize;
407 63 : for (int i = 0; i < nOverviews; i++)
408 : {
409 : // Verify that scales are reasonable, val/scale has to be an integer
410 33 : if (!IsPower(panOverviewListNew[i], scale))
411 : {
412 0 : CPLError(CE_Warning, CPLE_AppDefined,
413 : "MRF:IBuildOverviews, skipping overview factor %d,"
414 : " it is not a power of %f",
415 0 : panOverviewListNew[i], scale);
416 0 : panOverviewListNew[i] = -1;
417 0 : continue;
418 : };
419 :
420 33 : int srclevel = int(logbase(panOverviewListNew[i], scale) - 0.5);
421 33 : MRFRasterBand *b = static_cast<MRFRasterBand *>(GetRasterBand(1));
422 :
423 : // Warn for requests for invalid levels
424 33 : if (srclevel >= b->GetOverviewCount())
425 : {
426 0 : CPLError(CE_Warning, CPLE_AppDefined,
427 : "MRF:IBuildOverviews, overview factor %d is not valid "
428 : "for this dataset",
429 0 : panOverviewListNew[i]);
430 0 : panOverviewListNew[i] = -1;
431 0 : continue;
432 : }
433 33 : dfTotalPixels += dfPixels * std::pow(scale, -srclevel);
434 : }
435 :
436 30 : dfPixels = 0;
437 63 : for (int i = 0; i < nOverviews; i++)
438 : {
439 : // Skip the invalid levels
440 33 : if (panOverviewListNew[i] < 0)
441 0 : continue;
442 :
443 33 : int srclevel = int(logbase(panOverviewListNew[i], scale) - 0.5);
444 : auto dfLevelPixels =
445 33 : std::pow(scale, -srclevel) * nRasterXSize * nRasterYSize;
446 : // Use "avg" flag to trigger the internal average sampling
447 33 : if (EQUALN("Avg", pszResampling, 3) ||
448 19 : EQUALN("Nnb", pszResampling, 3))
449 : {
450 26 : int sampling = EQUALN("Avg", pszResampling, 3) ? SAMPLING_Avg
451 : : SAMPLING_Near;
452 26 : auto b = static_cast<MRFRasterBand *>(GetRasterBand(1));
453 : // Internal, using PatchOverview
454 26 : if (srclevel > 0)
455 : b = static_cast<MRFRasterBand *>(
456 3 : b->GetOverview(srclevel - 1));
457 :
458 : eErr =
459 26 : PatchOverview(0, 0, b->nBlocksPerRow, b->nBlocksPerColumn,
460 : srclevel, 0, sampling);
461 26 : if (eErr == CE_Failure)
462 26 : throw eErr;
463 : }
464 : else
465 : {
466 : // Use the GDAL method
467 14 : std::vector<GDALRasterBand *> apoSrcBandList(nBands);
468 : std::vector<std::vector<GDALRasterBand *>> aapoOverviewBandList(
469 14 : nBands);
470 14 : for (int iBand = 0; iBand < nBands; iBand++)
471 : {
472 : // This is the base level
473 7 : apoSrcBandList[iBand] = GetRasterBand(panBandList[iBand]);
474 : // Set up the destination
475 14 : aapoOverviewBandList[iBand] = std::vector<GDALRasterBand *>{
476 14 : apoSrcBandList[iBand]->GetOverview(srclevel)};
477 : // Use the previous level as the source
478 7 : if (srclevel > 0)
479 0 : apoSrcBandList[iBand] =
480 0 : apoSrcBandList[iBand]->GetOverview(srclevel - 1);
481 : }
482 :
483 14 : auto pScaledProgress = GDALCreateScaledProgress(
484 : dfPixels / dfTotalPixels,
485 7 : (dfPixels + dfLevelPixels) / dfTotalPixels, pfnProgress,
486 : pProgressData);
487 7 : eErr = GDALRegenerateOverviewsMultiBand(
488 : apoSrcBandList, aapoOverviewBandList, pszResampling,
489 : GDALScaledProgress, pScaledProgress, papszOptions);
490 7 : GDALDestroyScaledProgress(pScaledProgress);
491 : }
492 33 : dfPixels += dfLevelPixels;
493 33 : pfnProgress(dfPixels / dfTotalPixels, "", pProgressData);
494 33 : if (eErr == CE_Failure)
495 0 : throw eErr;
496 : }
497 :
498 30 : pfnProgress(1.0, "", pProgressData);
499 : }
500 0 : catch (const CPLErr &e)
501 : {
502 0 : eErr = e;
503 : }
504 :
505 30 : return eErr;
506 : }
507 :
508 : /*
509 : *\brief blank separated list to vector of doubles
510 : */
511 34 : static void list2vec(std::vector<double> &v, const char *pszList)
512 : {
513 34 : if ((pszList == nullptr) || (pszList[0] == 0))
514 0 : return;
515 34 : char **papszTokens = CSLTokenizeString2(
516 : pszList, " \t\n\r", CSLT_STRIPLEADSPACES | CSLT_STRIPENDSPACES);
517 34 : v.clear();
518 69 : for (int i = 0; i < CSLCount(papszTokens); i++)
519 35 : v.push_back(CPLStrtod(papszTokens[i], nullptr));
520 34 : CSLDestroy(papszTokens);
521 : }
522 :
523 20 : void MRFDataset::SetNoDataValue(const char *pszVal)
524 : {
525 20 : list2vec(vNoData, pszVal);
526 20 : }
527 :
528 7 : void MRFDataset::SetMinValue(const char *pszVal)
529 : {
530 7 : list2vec(vMin, pszVal);
531 7 : }
532 :
533 7 : void MRFDataset::SetMaxValue(const char *pszVal)
534 : {
535 7 : list2vec(vMax, pszVal);
536 7 : }
537 :
538 : /**
539 : *
540 : *\brief Read the XML config tree, from file
541 : * Caller is responsible for freeing the memory
542 : *
543 : * @return NULL on failure, or the document tree on success.
544 : *
545 : */
546 30 : CPLXMLNode *MRFDataset::ReadConfig() const
547 : {
548 30 : if (fname[0] == '<')
549 0 : return CPLParseXMLString(fname);
550 30 : return CPLParseXMLFile(fname);
551 : }
552 :
553 : /**
554 : *\brief Write the XML config tree
555 : * Caller is responsible for correctness of data
556 : * and for freeing the memory
557 : *
558 : * @param config The document tree to write
559 : * @return TRUE on success, FALSE otherwise
560 : */
561 201 : int MRFDataset::WriteConfig(CPLXMLNode *config)
562 : {
563 201 : if (fname[0] == '<')
564 0 : return FALSE;
565 201 : return CPLSerializeXMLTreeToFile(config, fname);
566 : }
567 :
568 : static void
569 5 : stringSplit(vector<string> &theStringVector, // Altered/returned value
570 : const string &theString, size_t start = 0,
571 : const char theDelimiter = ' ')
572 : {
573 : while (true)
574 : {
575 5 : size_t end = theString.find(theDelimiter, start);
576 5 : if (string::npos == end)
577 : {
578 5 : theStringVector.push_back(theString.substr(start));
579 5 : return;
580 : }
581 0 : theStringVector.push_back(theString.substr(start, end - start));
582 0 : start = end + 1;
583 0 : }
584 : }
585 :
586 : // Returns the number following the prefix if it exists in one of the vector
587 : // strings Otherwise it returns the default
588 15 : static int getnum(const vector<string> &theStringVector, const char prefix,
589 : int def)
590 : {
591 25 : for (unsigned int i = 0; i < theStringVector.size(); i++)
592 15 : if (theStringVector[i][0] == prefix)
593 5 : return atoi(theStringVector[i].c_str() + 1);
594 10 : return def;
595 : }
596 :
597 : /**
598 : *\brief Open a MRF file
599 : *
600 : */
601 202 : GDALDataset *MRFDataset::Open(GDALOpenInfo *poOpenInfo)
602 : {
603 202 : if (!MRFDriverIdentify(poOpenInfo))
604 0 : return nullptr;
605 :
606 202 : CPLXMLNode *config = nullptr;
607 202 : CPLErr ret = CE_None;
608 202 : const char *pszFileName = poOpenInfo->pszFilename;
609 :
610 202 : int level = -1; // All levels
611 202 : int version = 0; // Current
612 202 : int zslice = 0;
613 404 : string fn; // Used to parse and adjust the file name
614 404 : string insidefn; // inside tar file name
615 :
616 : // Different ways to open an MRF
617 202 : if (poOpenInfo->nHeaderBytes >= 10)
618 : {
619 197 : const char *pszHeader =
620 : reinterpret_cast<char *>(poOpenInfo->pabyHeader);
621 197 : fn.assign(pszHeader, poOpenInfo->nHeaderBytes);
622 197 : if (STARTS_WITH(pszHeader, "<MRF_META>")) // Regular file name
623 190 : config = CPLParseXMLFile(pszFileName);
624 7 : else if (poOpenInfo->eAccess == GA_ReadOnly && fn.size() > 600 &&
625 7 : (fn[262] == 0 || fn[262] == 32) &&
626 1 : STARTS_WITH(fn.c_str() + 257, "ustar") &&
627 15 : strlen(CPLGetPathSafe(fn.c_str()).c_str()) == 0 &&
628 1 : STARTS_WITH(fn.c_str() + 512, "<MRF_META>"))
629 : { // An MRF inside a tar
630 1 : insidefn = string("/vsitar/") + pszFileName + "/" + pszHeader;
631 1 : config = CPLParseXMLFile(insidefn.c_str());
632 : }
633 : #if defined(LERC)
634 : else
635 6 : config = LERC_Band::GetMRFConfig(poOpenInfo);
636 : #endif
637 : }
638 : else
639 : {
640 5 : if (EQUALN(pszFileName, "<MRF_META>", 10)) // Content as file name
641 0 : config = CPLParseXMLString(pszFileName);
642 : else
643 : { // Try Ornate file name
644 5 : fn = pszFileName;
645 5 : size_t pos = fn.find(":MRF:");
646 5 : if (string::npos != pos)
647 : { // Tokenize and pick known options
648 5 : vector<string> tokens;
649 5 : stringSplit(tokens, fn, pos + 5, ':');
650 5 : level = getnum(tokens, 'L', -1);
651 5 : version = getnum(tokens, 'V', 0);
652 5 : zslice = getnum(tokens, 'Z', 0);
653 5 : fn.resize(pos); // Cut the ornamentations
654 5 : pszFileName = fn.c_str();
655 5 : config = CPLParseXMLFile(pszFileName);
656 : }
657 : }
658 : }
659 :
660 202 : if (!config)
661 0 : return nullptr;
662 :
663 202 : MRFDataset *ds = new MRFDataset();
664 202 : ds->fname = pszFileName;
665 202 : if (!insidefn.empty())
666 : {
667 1 : ds->publicname = pszFileName;
668 1 : ds->fname = insidefn;
669 : }
670 202 : ds->eAccess = poOpenInfo->eAccess;
671 202 : ds->level = level;
672 202 : ds->zslice = zslice;
673 :
674 : // OpenOptions can override file name arguments
675 202 : ds->ProcessOpenOptions(poOpenInfo->papszOpenOptions);
676 :
677 202 : if (level == -1)
678 200 : ret = ds->Initialize(config);
679 : else
680 : {
681 : // Open the whole dataset, then pick one level
682 2 : ds->cds = new MRFDataset();
683 2 : ds->cds->fname = ds->fname;
684 2 : ds->cds->eAccess = ds->eAccess;
685 2 : ds->zslice = zslice;
686 2 : ret = ds->cds->Initialize(config);
687 2 : if (ret == CE_None)
688 2 : ret = ds->LevelInit(level);
689 : }
690 202 : CPLDestroyXMLNode(config);
691 :
692 202 : if (ret != CE_None)
693 : {
694 23 : delete ds;
695 23 : return nullptr;
696 : }
697 :
698 : // Open a single version
699 179 : if (version != 0)
700 2 : ret = ds->SetVersion(version);
701 :
702 179 : if (ret != CE_None)
703 : {
704 1 : delete ds;
705 1 : return nullptr;
706 : }
707 :
708 : // Tell PAM what our real file name is, to help it find the aux.xml
709 178 : ds->SetPhysicalFilename(ds->fname);
710 : // Don't mess with metadata after this, otherwise PAM will re-write the
711 : // aux.xml
712 178 : ds->TryLoadXML();
713 :
714 : /* -------------------------------------------------------------------- */
715 : /* Open external overviews. */
716 : /* -------------------------------------------------------------------- */
717 178 : ds->oOvManager.Initialize(ds, ds->fname);
718 :
719 178 : return ds;
720 : }
721 :
722 : // Adjust the band images with the right offset, then adjust the sizes
723 2 : CPLErr MRFDataset::SetVersion(int version)
724 : {
725 2 : if (!hasVersions || version > verCount)
726 : {
727 1 : CPLError(CE_Failure, CPLE_AppDefined,
728 : "GDAL MRF: Version number error!");
729 1 : return CE_Failure;
730 : }
731 : // Size of one version index
732 2 : for (int bcount = 1; bcount <= nBands; bcount++)
733 : {
734 : MRFRasterBand *srcband =
735 1 : reinterpret_cast<MRFRasterBand *>(GetRasterBand(bcount));
736 1 : srcband->img.idxoffset += idxSize * verCount;
737 1 : for (int l = 0; l < srcband->GetOverviewCount(); l++)
738 : {
739 : MRFRasterBand *band =
740 0 : reinterpret_cast<MRFRasterBand *>(srcband->GetOverview(l));
741 0 : if (band != nullptr)
742 0 : band->img.idxoffset += idxSize * verCount;
743 : }
744 : }
745 1 : hasVersions = 0;
746 1 : return CE_None;
747 : }
748 :
749 2 : CPLErr MRFDataset::LevelInit(const int l)
750 : {
751 : // Test that this level does exist
752 2 : if (l < 0 || l >= cds->GetRasterBand(1)->GetOverviewCount())
753 : {
754 1 : CPLError(CE_Failure, CPLE_AppDefined,
755 : "GDAL MRF: Overview not present!");
756 1 : return CE_Failure;
757 : }
758 :
759 : MRFRasterBand *srcband = reinterpret_cast<MRFRasterBand *>(
760 1 : cds->GetRasterBand(1)->GetOverview(l));
761 :
762 : // Copy the sizes from this level
763 1 : full = srcband->img;
764 1 : current = srcband->img;
765 1 : current.size.c = cds->current.size.c;
766 1 : scale = cds->scale;
767 1 : const auto poSRS = cds->GetSpatialRef();
768 1 : if (poSRS)
769 1 : m_oSRS = *poSRS;
770 :
771 1 : SetMetadataItem("INTERLEAVE", OrderName(current.order), "IMAGE_STRUCTURE");
772 1 : SetMetadataItem("COMPRESSION", CompName(current.comp), "IMAGE_STRUCTURE");
773 :
774 1 : bGeoTransformValid = (CE_None == cds->GetGeoTransform(GeoTransform));
775 4 : for (int i = 0; i < l + 1; i++)
776 : {
777 3 : GeoTransform[1] *= scale;
778 3 : GeoTransform[5] *= scale;
779 : }
780 :
781 1 : nRasterXSize = current.size.x;
782 1 : nRasterYSize = current.size.y;
783 1 : nBands = current.size.c;
784 :
785 : // Add the bands, copy constructor so they can be closed independently
786 2 : for (int i = 1; i <= nBands; i++)
787 1 : SetBand(i, new MRFLRasterBand(reinterpret_cast<MRFRasterBand *>(
788 1 : cds->GetRasterBand(i)->GetOverview(l))));
789 1 : return CE_None;
790 : }
791 :
792 : // Is the string positive or not
793 1269 : inline bool on(const char *pszValue)
794 : {
795 1269 : if (!pszValue || pszValue[0] == 0)
796 109 : return false;
797 2308 : return EQUAL(pszValue, "ON") || EQUAL(pszValue, "TRUE") ||
798 2308 : EQUAL(pszValue, "YES");
799 : }
800 :
801 : /**
802 : *\brief Initialize the image structure and the dataset from the XML Raster node
803 : *
804 : * @param image the structure to be initialized
805 : * @param ds the parent dataset, some things get inherited
806 : * @param defimage defimage
807 : *
808 : * The structure should be initialized with the default values as much as
809 : *possible
810 : *
811 : */
812 :
813 363 : static CPLErr Init_Raster(ILImage &image, MRFDataset *ds, CPLXMLNode *defimage)
814 : {
815 : CPLXMLNode *node; // temporary
816 363 : if (!defimage)
817 : {
818 0 : CPLError(CE_Failure, CPLE_AppDefined,
819 : "GDAL MRF: Can't find raster info");
820 0 : return CE_Failure;
821 : }
822 :
823 : // Size is mandatory
824 363 : node = CPLGetXMLNode(defimage, "Size");
825 :
826 363 : if (node)
827 : {
828 363 : image.size = ILSize(static_cast<int>(getXMLNum(node, "x", -1)),
829 363 : static_cast<int>(getXMLNum(node, "y", -1)),
830 363 : static_cast<int>(getXMLNum(node, "z", 1)),
831 363 : static_cast<int>(getXMLNum(node, "c", 1)), 0);
832 : }
833 :
834 : // Basic checks
835 363 : if (!node || image.size.x < 1 || image.size.y < 1 || image.size.z < 0 ||
836 726 : image.size.c < 0 || !GDALCheckBandCount(image.size.c, FALSE))
837 : {
838 0 : CPLError(CE_Failure, CPLE_AppDefined, "Raster size missing or invalid");
839 0 : return CE_Failure;
840 : }
841 :
842 : // Pagesize, defaults to 512,512,1,c
843 363 : image.pagesize = ILSize(std::min(512, image.size.x),
844 363 : std::min(512, image.size.y), 1, image.size.c);
845 :
846 363 : node = CPLGetXMLNode(defimage, "PageSize");
847 363 : if (node)
848 : {
849 363 : image.pagesize =
850 363 : ILSize(static_cast<int>(getXMLNum(node, "x", image.pagesize.x)),
851 363 : static_cast<int>(getXMLNum(node, "y", image.pagesize.y)),
852 : 1, // One slice at a time, forced
853 363 : static_cast<int>(getXMLNum(node, "c", image.pagesize.c)));
854 363 : if (image.pagesize.x < 1 || image.pagesize.y < 1 ||
855 363 : image.pagesize.c <= 0)
856 : {
857 0 : CPLError(CE_Failure, CPLE_IllegalArg, "Invalid PageSize");
858 0 : return CE_Failure;
859 : }
860 : }
861 :
862 : // Page Encoding, defaults to PNG
863 363 : const char *pszCompression = CPLGetXMLValue(defimage, "Compression", "PNG");
864 363 : image.comp = CompToken(pszCompression);
865 363 : if (image.comp == IL_ERR_COMP)
866 : {
867 0 : CPLError(CE_Failure, CPLE_IllegalArg,
868 : "GDAL MRF: Compression %s is unknown", pszCompression);
869 0 : return CE_Failure;
870 : }
871 :
872 : // Is there a palette?
873 : //
874 : // GDAL only supports RGB+A palette, the other modes don't work
875 : //
876 :
877 673 : if ((image.pagesize.c == 1) &&
878 310 : (nullptr != (node = CPLGetXMLNode(defimage, "Palette"))))
879 : {
880 1 : int entries = static_cast<int>(getXMLNum(node, "Size", 255));
881 1 : GDALPaletteInterp eInterp = GPI_RGB;
882 1 : if ((entries > 0) && (entries < 257))
883 : {
884 1 : GDALColorEntry ce_start = {0, 0, 0, 255}, ce_end = {0, 0, 0, 255};
885 :
886 : // Create it and initialize it to black opaque
887 1 : GDALColorTable *poColorTable = new GDALColorTable(eInterp);
888 1 : poColorTable->CreateColorRamp(0, &ce_start, entries - 1, &ce_end);
889 : // Read the values
890 1 : CPLXMLNode *p = CPLGetXMLNode(node, "Entry");
891 1 : if (p)
892 : {
893 : // Initialize the first entry
894 1 : ce_start = GetXMLColorEntry(p);
895 1 : int start_idx = static_cast<int>(getXMLNum(p, "idx", 0));
896 1 : if (start_idx < 0)
897 : {
898 0 : CPLError(CE_Failure, CPLE_IllegalArg,
899 : "GDAL MRF: Palette index %d not allowed",
900 : start_idx);
901 0 : delete poColorTable;
902 0 : return CE_Failure;
903 : }
904 1 : poColorTable->SetColorEntry(start_idx, &ce_start);
905 256 : while (nullptr != (p = SearchXMLSiblings(p, "Entry")))
906 : {
907 : // For every entry, create a ramp
908 255 : ce_end = GetXMLColorEntry(p);
909 : int end_idx =
910 255 : static_cast<int>(getXMLNum(p, "idx", start_idx + 1));
911 255 : if ((end_idx <= start_idx) || (start_idx >= entries))
912 : {
913 0 : CPLError(CE_Failure, CPLE_IllegalArg,
914 : "GDAL MRF: Index Error at index %d", end_idx);
915 0 : delete poColorTable;
916 0 : return CE_Failure;
917 : }
918 255 : poColorTable->CreateColorRamp(start_idx, &ce_start, end_idx,
919 : &ce_end);
920 255 : ce_start = ce_end;
921 255 : start_idx = end_idx;
922 : }
923 : }
924 :
925 1 : ds->SetColorTable(poColorTable);
926 : }
927 : else
928 : {
929 0 : CPLError(CE_Failure, CPLE_IllegalArg,
930 : "GDAL MRF: Palette definition error");
931 0 : return CE_Failure;
932 : }
933 : }
934 :
935 : // Order of increment
936 363 : if (image.pagesize.c != image.size.c && image.pagesize.c != 1)
937 : {
938 : // Fixes heap buffer overflow in
939 : // GDALMRFRasterBand::ReadInterleavedBlock() See
940 : // https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=2884
941 0 : CPLError(CE_Failure, CPLE_NotSupported,
942 : "GDAL MRF: image.pagesize.c = %d and image.size.c = %d",
943 : image.pagesize.c, image.size.c);
944 0 : return CE_Failure;
945 : }
946 :
947 363 : image.order = OrderToken(
948 : CPLGetXMLValue(defimage, "Order",
949 363 : (image.pagesize.c != image.size.c) ? "BAND" : "PIXEL"));
950 363 : if (image.order == IL_ERR_ORD)
951 : {
952 0 : CPLError(CE_Failure, CPLE_AppDefined, "GDAL MRF: Order %s is unknown",
953 : CPLGetXMLValue(defimage, "Order", nullptr));
954 0 : return CE_Failure;
955 : }
956 :
957 363 : const char *photo_val = CPLGetXMLValue(defimage, "Photometric", nullptr);
958 363 : if (photo_val)
959 5 : ds->SetPhotometricInterpretation(photo_val);
960 :
961 363 : image.quality = atoi(CPLGetXMLValue(defimage, "Quality", "85"));
962 363 : if (image.quality < 0 || image.quality > 99)
963 : {
964 0 : CPLError(CE_Warning, CPLE_AppDefined,
965 : "GDAL MRF: Quality setting error, using default of 85");
966 0 : image.quality = 85;
967 : }
968 :
969 : // Data Type, use GDAL Names
970 363 : image.dt = GDALGetDataTypeByName(
971 : CPLGetXMLValue(defimage, "DataType", GDALGetDataTypeName(image.dt)));
972 363 : if (image.dt == GDT_Unknown || GDALGetDataTypeSize(image.dt) == 0)
973 : {
974 0 : CPLError(CE_Failure, CPLE_AppDefined, "GDAL MRF: Unsupported type");
975 0 : return CE_Failure;
976 : }
977 :
978 : // Check the endianness if needed, assume host order
979 363 : if (is_Endianness_Dependent(image.dt, image.comp))
980 69 : image.nbo = on(CPLGetXMLValue(defimage, "NetByteOrder", "No"));
981 :
982 363 : CPLXMLNode *DataValues = CPLGetXMLNode(defimage, "DataValues");
983 363 : if (nullptr != DataValues)
984 : {
985 27 : const char *pszValue = CPLGetXMLValue(DataValues, "NoData", nullptr);
986 27 : if (pszValue)
987 20 : ds->SetNoDataValue(pszValue);
988 27 : pszValue = CPLGetXMLValue(DataValues, "min", nullptr);
989 27 : if (pszValue)
990 7 : ds->SetMinValue(pszValue);
991 27 : pszValue = CPLGetXMLValue(DataValues, "max", nullptr);
992 27 : if (pszValue)
993 7 : ds->SetMaxValue(pszValue);
994 : }
995 :
996 : // Calculate the page size in bytes
997 363 : const int nDTSize = GDALGetDataTypeSizeBytes(image.dt);
998 363 : if (nDTSize <= 0 || image.pagesize.z <= 0 ||
999 363 : image.pagesize.x > INT_MAX / image.pagesize.y ||
1000 363 : image.pagesize.x * image.pagesize.y > INT_MAX / image.pagesize.z ||
1001 363 : image.pagesize.x * image.pagesize.y * image.pagesize.z >
1002 363 : INT_MAX / image.pagesize.c ||
1003 363 : image.pagesize.x * image.pagesize.y * image.pagesize.z *
1004 363 : image.pagesize.c >
1005 363 : INT_MAX / nDTSize)
1006 : {
1007 0 : CPLError(CE_Failure, CPLE_AppDefined, "MRF page size too big");
1008 0 : return CE_Failure;
1009 : }
1010 363 : image.pageSizeBytes = nDTSize * image.pagesize.x * image.pagesize.y *
1011 363 : image.pagesize.z * image.pagesize.c;
1012 :
1013 : // Calculate the page count, including the total for the level
1014 363 : image.pagecount = pcount(image.size, image.pagesize);
1015 363 : if (image.pagecount.l < 0)
1016 : {
1017 0 : return CE_Failure;
1018 : }
1019 :
1020 : // Data File Name and base offset
1021 : image.datfname =
1022 363 : getFname(defimage, "DataFile", ds->GetFname(), ILComp_Ext[image.comp]);
1023 363 : image.dataoffset = static_cast<int>(
1024 363 : getXMLNum(CPLGetXMLNode(defimage, "DataFile"), "offset", 0.0));
1025 :
1026 : // Index File Name and base offset
1027 363 : image.idxfname = getFname(defimage, "IndexFile", ds->GetFname(), ".idx");
1028 363 : image.idxoffset = static_cast<int>(
1029 363 : getXMLNum(CPLGetXMLNode(defimage, "IndexFile"), "offset", 0.0));
1030 :
1031 363 : return CE_None;
1032 : }
1033 :
1034 160 : char **MRFDataset::GetFileList()
1035 : {
1036 160 : char **papszFileList = nullptr;
1037 :
1038 160 : string usename = fname;
1039 160 : if (!publicname.empty())
1040 0 : usename = publicname;
1041 : // Add the header file name if it is real
1042 : VSIStatBufL sStat;
1043 160 : if (VSIStatExL(usename.c_str(), &sStat, VSI_STAT_EXISTS_FLAG) == 0)
1044 160 : papszFileList = CSLAddString(papszFileList, usename.c_str());
1045 :
1046 : // These two should be real
1047 : // We don't really want to add these files, since they will be erased when
1048 : // an mrf is overwritten This collides with the concept that the data file
1049 : // never shrinks. Same goes with the index, in case we just want to add
1050 : // things to it.
1051 : // papszFileList = CSLAddString( papszFileList, full.datfname);
1052 : // papszFileList = CSLAddString( papszFileList, full.idxfname);
1053 : // if (!source.empty())
1054 : // papszFileList = CSLAddString( papszFileList, source);
1055 :
1056 320 : return papszFileList;
1057 : }
1058 :
1059 : // Try to create all the folders in the path in sequence, ignore errors
1060 4 : static void mkdir_r(string const &fname)
1061 : {
1062 4 : size_t loc = fname.find_first_of("\\/");
1063 4 : if (loc == string::npos)
1064 0 : return;
1065 : while (true)
1066 : {
1067 5 : ++loc;
1068 5 : loc = fname.find_first_of("\\/", loc);
1069 5 : if (loc == string::npos)
1070 4 : break;
1071 1 : VSIMkdir(fname.substr(0, loc).c_str(), 0);
1072 : }
1073 : }
1074 :
1075 : // Returns the dataset index file or null
1076 7265 : VSILFILE *MRFDataset::IdxFP()
1077 : {
1078 7265 : if (ifp.FP != nullptr)
1079 6998 : return ifp.FP;
1080 :
1081 : // If missing is set, we already checked, there is no index
1082 267 : if (missing)
1083 0 : return nullptr;
1084 :
1085 : // If name starts with '(' it is not a real file name
1086 267 : if (current.idxfname[0] == '(')
1087 0 : return nullptr;
1088 :
1089 267 : const char *mode = "rb";
1090 267 : ifp.acc = GF_Read;
1091 :
1092 267 : if (eAccess == GA_Update || !source.empty())
1093 : {
1094 158 : mode = "r+b";
1095 158 : ifp.acc = GF_Write;
1096 : }
1097 :
1098 267 : ifp.FP = VSIFOpenL(current.idxfname, mode);
1099 :
1100 : // If file didn't open for reading and no_errors is set, just return null
1101 : // and make a note
1102 267 : if (ifp.FP == nullptr && eAccess == GA_ReadOnly && no_errors)
1103 : {
1104 0 : missing = 1;
1105 0 : return nullptr;
1106 : }
1107 :
1108 : // need to create the index file
1109 367 : if (ifp.FP == nullptr && !bCrystalized &&
1110 100 : (eAccess == GA_Update || !source.empty()))
1111 : {
1112 100 : mode = "w+b";
1113 100 : ifp.FP = VSIFOpenL(current.idxfname, mode);
1114 : }
1115 :
1116 267 : if (nullptr == ifp.FP && !source.empty())
1117 : {
1118 : // caching and cloning, try making the folder and attempt again
1119 4 : mkdir_r(current.idxfname);
1120 4 : ifp.FP = VSIFOpenL(current.idxfname, mode);
1121 : }
1122 :
1123 267 : GIntBig expected_size = idxSize;
1124 267 : if (clonedSource)
1125 2 : expected_size *= 2;
1126 :
1127 267 : if (nullptr != ifp.FP)
1128 : {
1129 406 : if (!bCrystalized &&
1130 149 : !CheckFileSize(current.idxfname, expected_size, GA_Update))
1131 : {
1132 0 : CPLError(CE_Failure, CPLE_FileIO,
1133 : "MRF: Can't extend the cache index file %s",
1134 : current.idxfname.c_str());
1135 0 : return nullptr;
1136 : }
1137 :
1138 257 : if (source.empty())
1139 253 : return ifp.FP;
1140 :
1141 : // Make sure the index is large enough before proceeding
1142 : // Timeout in .1 seconds, can't really guarantee the accuracy
1143 : // So this is about half second, should be sufficient
1144 4 : int timeout = 5;
1145 0 : do
1146 : {
1147 4 : if (CheckFileSize(current.idxfname, expected_size, GA_ReadOnly))
1148 4 : return ifp.FP;
1149 0 : CPLSleep(0.100); /* 100 ms */
1150 0 : } while (--timeout);
1151 :
1152 : // If we get here it is a time-out
1153 0 : CPLError(CE_Failure, CPLE_AppDefined,
1154 : "GDAL MRF: Timeout on fetching cloned index file %s\n",
1155 : current.idxfname.c_str());
1156 0 : return nullptr;
1157 : }
1158 :
1159 : // If single tile, and no index file, let the caller figure it out
1160 10 : if (IsSingleTile())
1161 6 : return nullptr;
1162 :
1163 : // Error if this is not a caching MRF
1164 4 : if (source.empty())
1165 : {
1166 0 : CPLError(CE_Failure, CPLE_AppDefined,
1167 : "GDAL MRF: Can't open index file %s\n",
1168 : current.idxfname.c_str());
1169 0 : return nullptr;
1170 : }
1171 :
1172 : // Caching/Cloning MRF and index could be read only
1173 : // Is this actually works, we should try again, maybe somebody else just
1174 : // created the file?
1175 4 : mode = "rb";
1176 4 : ifp.acc = GF_Read;
1177 4 : ifp.FP = VSIFOpenL(current.idxfname, mode);
1178 4 : if (nullptr != ifp.FP)
1179 0 : return ifp.FP;
1180 :
1181 : // Caching and index file absent, create it
1182 : // Due to a race, multiple processes might do this at the same time, but
1183 : // that is fine
1184 4 : ifp.FP = VSIFOpenL(current.idxfname, "wb");
1185 4 : if (nullptr == ifp.FP)
1186 : {
1187 0 : CPLError(CE_Failure, CPLE_AppDefined,
1188 : "Can't create the MRF cache index file %s",
1189 : current.idxfname.c_str());
1190 0 : return nullptr;
1191 : }
1192 4 : VSIFCloseL(ifp.FP);
1193 4 : ifp.FP = nullptr;
1194 :
1195 : // Make it large enough for caching and for cloning
1196 4 : if (!CheckFileSize(current.idxfname, expected_size, GA_Update))
1197 : {
1198 0 : CPLError(CE_Failure, CPLE_AppDefined,
1199 : "Can't extend the cache index file %s",
1200 : current.idxfname.c_str());
1201 0 : return nullptr;
1202 : }
1203 :
1204 : // Try opening it again in rw mode so we can read and write
1205 4 : mode = "r+b";
1206 4 : ifp.acc = GF_Write;
1207 4 : ifp.FP = VSIFOpenL(current.idxfname.c_str(), mode);
1208 :
1209 4 : if (nullptr == ifp.FP)
1210 : {
1211 0 : CPLError(CE_Failure, CPLE_AppDefined,
1212 : "GDAL MRF: Can't reopen cache index file %s\n",
1213 : full.idxfname.c_str());
1214 0 : return nullptr;
1215 : }
1216 4 : return ifp.FP;
1217 : }
1218 :
1219 : //
1220 : // Returns the dataset data file or null
1221 : // Data file is opened either in Read or Append mode, never in straight write
1222 : //
1223 7140 : VSILFILE *MRFDataset::DataFP()
1224 : {
1225 7140 : if (dfp.FP != nullptr)
1226 6876 : return dfp.FP;
1227 264 : const char *mode = "rb";
1228 264 : dfp.acc = GF_Read;
1229 :
1230 : // Open it for writing if updating or if caching
1231 264 : if (eAccess == GA_Update || !source.empty())
1232 : {
1233 157 : mode = "a+b";
1234 157 : dfp.acc = GF_Write;
1235 : }
1236 :
1237 264 : dfp.FP = VSIFOpenL(current.datfname, mode);
1238 264 : if (dfp.FP)
1239 264 : return dfp.FP;
1240 :
1241 : // It could be a caching MRF
1242 0 : if (source.empty())
1243 0 : goto io_error;
1244 :
1245 : // May be there but read only, remember that it was open that way
1246 0 : mode = "rb";
1247 0 : dfp.acc = GF_Read;
1248 0 : dfp.FP = VSIFOpenL(current.datfname, mode);
1249 0 : if (nullptr != dfp.FP)
1250 : {
1251 0 : CPLDebug("MRF_IO", "Opened %s RO mode %s\n", current.datfname.c_str(),
1252 : mode);
1253 0 : return dfp.FP;
1254 : }
1255 :
1256 0 : if (source.empty())
1257 0 : goto io_error;
1258 :
1259 : // caching, maybe the folder didn't exist
1260 0 : mkdir_r(current.datfname);
1261 0 : mode = "a+b";
1262 0 : dfp.acc = GF_Write;
1263 0 : dfp.FP = VSIFOpenL(current.datfname, mode);
1264 0 : if (dfp.FP)
1265 0 : return dfp.FP;
1266 :
1267 0 : io_error:
1268 0 : dfp.FP = nullptr;
1269 0 : CPLError(CE_Failure, CPLE_FileIO, "GDAL MRF: %s : %s", strerror(errno),
1270 : current.datfname.c_str());
1271 0 : return nullptr;
1272 : }
1273 :
1274 : // Builds an XML tree from the current MRF. If written to a file it becomes an
1275 : // MRF
1276 332 : CPLXMLNode *MRFDataset::BuildConfig()
1277 : {
1278 332 : CPLXMLNode *config = CPLCreateXMLNode(nullptr, CXT_Element, "MRF_META");
1279 :
1280 332 : if (!source.empty())
1281 : {
1282 : CPLXMLNode *psCachedSource =
1283 4 : CPLCreateXMLNode(config, CXT_Element, "CachedSource");
1284 : // Should wrap the string in CDATA, in case it is XML
1285 : CPLXMLNode *psSource =
1286 4 : CPLCreateXMLElementAndValue(psCachedSource, "Source", source);
1287 4 : if (clonedSource)
1288 0 : CPLSetXMLValue(psSource, "#clone", "true");
1289 : }
1290 :
1291 : // Use the full size
1292 332 : CPLXMLNode *raster = CPLCreateXMLNode(config, CXT_Element, "Raster");
1293 :
1294 : // Preserve the file names if not the default ones
1295 332 : if (full.datfname != getFname(GetFname(), ILComp_Ext[full.comp]))
1296 0 : CPLCreateXMLElementAndValue(raster, "DataFile", full.datfname.c_str());
1297 332 : if (full.idxfname != getFname(GetFname(), ".idx"))
1298 0 : CPLCreateXMLElementAndValue(raster, "IndexFile", full.idxfname.c_str());
1299 332 : if (spacing != 0)
1300 0 : XMLSetAttributeVal(raster, "Spacing", static_cast<double>(spacing),
1301 : "%.0f");
1302 :
1303 332 : XMLSetAttributeVal(raster, "Size", full.size, "%.0f");
1304 332 : XMLSetAttributeVal(raster, "PageSize", full.pagesize, "%.0f");
1305 :
1306 : #ifdef HAVE_PNG
1307 332 : if (full.comp != IL_PNG)
1308 : #endif
1309 : {
1310 192 : CPLCreateXMLElementAndValue(raster, "Compression", CompName(full.comp));
1311 : }
1312 :
1313 332 : if (full.dt != GDT_Byte)
1314 202 : CPLCreateXMLElementAndValue(raster, "DataType",
1315 : GDALGetDataTypeName(full.dt));
1316 :
1317 : // special photometric interpretation
1318 332 : if (!photometric.empty())
1319 4 : CPLCreateXMLElementAndValue(raster, "Photometric", photometric);
1320 :
1321 332 : if (!vNoData.empty() || !vMin.empty() || !vMax.empty())
1322 : {
1323 : CPLXMLNode *values =
1324 43 : CPLCreateXMLNode(raster, CXT_Element, "DataValues");
1325 43 : XMLSetAttributeVal(values, "NoData", vNoData);
1326 43 : XMLSetAttributeVal(values, "min", vMin);
1327 43 : XMLSetAttributeVal(values, "max", vMax);
1328 : }
1329 :
1330 : // palette, if we have one
1331 332 : if (poColorTable != nullptr)
1332 : {
1333 1 : const char *pfrmt = "%.0f";
1334 1 : CPLXMLNode *pal = CPLCreateXMLNode(raster, CXT_Element, "Palette");
1335 1 : int sz = poColorTable->GetColorEntryCount();
1336 1 : if (sz != 256)
1337 0 : XMLSetAttributeVal(pal, "Size", poColorTable->GetColorEntryCount());
1338 : // RGB or RGBA for now
1339 257 : for (int i = 0; i < sz; i++)
1340 : {
1341 256 : CPLXMLNode *entry = CPLCreateXMLNode(pal, CXT_Element, "Entry");
1342 256 : const GDALColorEntry *ent = poColorTable->GetColorEntry(i);
1343 : // No need to set the index, it is always from 0 no size-1
1344 256 : XMLSetAttributeVal(entry, "c1", ent->c1, pfrmt);
1345 256 : XMLSetAttributeVal(entry, "c2", ent->c2, pfrmt);
1346 256 : XMLSetAttributeVal(entry, "c3", ent->c3, pfrmt);
1347 256 : if (ent->c4 != 255)
1348 0 : XMLSetAttributeVal(entry, "c4", ent->c4, pfrmt);
1349 : }
1350 : }
1351 :
1352 332 : if (is_Endianness_Dependent(full.dt, full.comp)) // Need to set the order
1353 62 : CPLCreateXMLElementAndValue(raster, "NetByteOrder",
1354 62 : (full.nbo || NET_ORDER) ? "TRUE" : "FALSE");
1355 :
1356 332 : if (full.quality > 0 && full.quality != 85)
1357 12 : CPLCreateXMLElementAndValue(raster, "Quality",
1358 24 : CPLOPrintf("%d", full.quality));
1359 :
1360 : // Done with the raster node
1361 :
1362 332 : if (scale != 0.0)
1363 : {
1364 0 : CPLCreateXMLNode(config, CXT_Element, "Rsets");
1365 0 : CPLSetXMLValue(config, "Rsets.#model", "uniform");
1366 0 : CPLSetXMLValue(config, "Rsets.#scale", PrintDouble(scale));
1367 : }
1368 332 : CPLXMLNode *gtags = CPLCreateXMLNode(config, CXT_Element, "GeoTags");
1369 :
1370 : // Do we have an affine transform different from identity?
1371 : double gt[6];
1372 664 : if ((MRFDataset::GetGeoTransform(gt) == CE_None) &&
1373 332 : (gt[0] != 0 || gt[1] != 1 || gt[2] != 0 || gt[3] != 0 || gt[4] != 0 ||
1374 196 : gt[5] != 1))
1375 : {
1376 136 : double minx = gt[0];
1377 136 : double maxx = gt[1] * full.size.x + minx;
1378 136 : double maxy = gt[3];
1379 136 : double miny = gt[5] * full.size.y + maxy;
1380 136 : CPLXMLNode *bbox = CPLCreateXMLNode(gtags, CXT_Element, "BoundingBox");
1381 136 : XMLSetAttributeVal(bbox, "minx", minx);
1382 136 : XMLSetAttributeVal(bbox, "miny", miny);
1383 136 : XMLSetAttributeVal(bbox, "maxx", maxx);
1384 136 : XMLSetAttributeVal(bbox, "maxy", maxy);
1385 : }
1386 :
1387 332 : const char *pszProj = GetProjectionRef();
1388 332 : if (pszProj && (!EQUAL(pszProj, "")))
1389 137 : CPLCreateXMLElementAndValue(gtags, "Projection", pszProj);
1390 :
1391 332 : if (optlist.Count() != 0)
1392 : {
1393 48 : CPLString options;
1394 50 : for (int i = 0; i < optlist.size(); i++)
1395 : {
1396 26 : options += optlist[i];
1397 26 : options += ' ';
1398 : }
1399 24 : options.pop_back();
1400 24 : CPLCreateXMLElementAndValue(config, "Options", options);
1401 : }
1402 :
1403 332 : return config;
1404 : }
1405 :
1406 : /**
1407 : * \brief Populates the dataset variables from the XML definition
1408 : *
1409 : *
1410 : */
1411 363 : CPLErr MRFDataset::Initialize(CPLXMLNode *config)
1412 : {
1413 : // We only need a basic initialization here, usually gets overwritten by the
1414 : // image params
1415 363 : full.dt = GDT_Byte;
1416 363 : full.hasNoData = false;
1417 363 : full.NoDataValue = 0;
1418 363 : Quality = 85;
1419 :
1420 363 : CPLErr ret = Init_Raster(full, this, CPLGetXMLNode(config, "Raster"));
1421 363 : if (CE_None != ret)
1422 0 : return ret;
1423 :
1424 363 : hasVersions = on(CPLGetXMLValue(config, "Raster.versioned", "no"));
1425 363 : mp_safe = on(CPLGetXMLValue(config, "Raster.mp_safe", "no"));
1426 363 : spacing = atoi(CPLGetXMLValue(config, "Raster.Spacing", "0"));
1427 :
1428 : // The zslice defined in the file wins over the oo or the file argument
1429 363 : if (CPLGetXMLNode(config, "Raster.zslice"))
1430 0 : zslice = atoi(CPLGetXMLValue(config, "Raster.zslice", "0"));
1431 :
1432 363 : Quality = full.quality;
1433 :
1434 : // Bounding box
1435 363 : CPLXMLNode *bbox = CPLGetXMLNode(config, "GeoTags.BoundingBox");
1436 363 : if (nullptr != bbox)
1437 : {
1438 : double x0, x1, y0, y1;
1439 :
1440 159 : x0 = atof(CPLGetXMLValue(bbox, "minx", "0"));
1441 159 : x1 = atof(CPLGetXMLValue(bbox, "maxx", "1"));
1442 159 : y1 = atof(CPLGetXMLValue(bbox, "maxy", "1"));
1443 159 : y0 = atof(CPLGetXMLValue(bbox, "miny", "0"));
1444 :
1445 159 : GeoTransform[0] = x0;
1446 159 : GeoTransform[1] = (x1 - x0) / full.size.x;
1447 159 : GeoTransform[2] = 0;
1448 159 : GeoTransform[3] = y1;
1449 159 : GeoTransform[4] = 0;
1450 159 : GeoTransform[5] = (y0 - y1) / full.size.y;
1451 159 : bGeoTransformValid = TRUE;
1452 : }
1453 :
1454 : const char *pszRawProjFromXML =
1455 363 : CPLGetXMLValue(config, "GeoTags.Projection", "");
1456 363 : if (strlen(pszRawProjFromXML) != 0)
1457 160 : m_oSRS.SetFromUserInput(
1458 : pszRawProjFromXML,
1459 : OGRSpatialReference::SET_FROM_USER_INPUT_LIMITATIONS_get());
1460 :
1461 : // Copy the full size to current, data and index are not yet open
1462 363 : current = full;
1463 363 : if (current.size.z != 1)
1464 : {
1465 0 : SetMetadataItem("ZSIZE", CPLOPrintf("%d", current.size.z),
1466 0 : "IMAGE_STRUCTURE");
1467 0 : SetMetadataItem("ZSLICE", CPLOPrintf("%d", zslice), "IMAGE_STRUCTURE");
1468 : // Capture the zslice in pagesize.l
1469 0 : current.pagesize.l = zslice;
1470 : // Adjust offset for base image
1471 0 : if (full.size.z <= 0)
1472 : {
1473 0 : CPLError(CE_Failure, CPLE_AppDefined,
1474 : "GDAL MRF: Invalid Raster.z value");
1475 0 : return CE_Failure;
1476 : }
1477 0 : if (zslice >= full.size.z)
1478 : {
1479 0 : CPLError(CE_Failure, CPLE_AppDefined, "GDAL MRF: Invalid slice");
1480 0 : return CE_Failure;
1481 : }
1482 :
1483 0 : current.idxoffset +=
1484 0 : (current.pagecount.l / full.size.z) * zslice * sizeof(ILIdx);
1485 : }
1486 :
1487 : // Dataset metadata setup
1488 363 : SetMetadataItem("INTERLEAVE", OrderName(current.order), "IMAGE_STRUCTURE");
1489 363 : SetMetadataItem("COMPRESSION", CompName(current.comp), "IMAGE_STRUCTURE");
1490 :
1491 363 : if (is_Endianness_Dependent(current.dt, current.comp))
1492 69 : SetMetadataItem("NETBYTEORDER", current.nbo ? "TRUE" : "FALSE",
1493 69 : "IMAGE_STRUCTURE");
1494 :
1495 : // Open the files for the current image, either RW or RO
1496 363 : nRasterXSize = current.size.x;
1497 363 : nRasterYSize = current.size.y;
1498 363 : nBands = current.size.c;
1499 :
1500 363 : if (!nBands || !nRasterXSize || !nRasterYSize)
1501 : {
1502 0 : CPLError(CE_Failure, CPLE_AppDefined, "GDAL MRF: Image size missing");
1503 0 : return CE_Failure;
1504 : }
1505 :
1506 : // Pick up the source data image, if there is one
1507 363 : source = CPLGetXMLValue(config, "CachedSource.Source", "");
1508 : // Is it a clone?
1509 363 : clonedSource =
1510 363 : on(CPLGetXMLValue(config, "CachedSource.Source.clone", "no"));
1511 : // Pick up the options, if any
1512 : optlist.Assign(CSLTokenizeString2(
1513 : CPLGetXMLValue(config, "Options", nullptr), " \t\n\r",
1514 363 : CSLT_STRIPLEADSPACES | CSLT_STRIPENDSPACES));
1515 :
1516 : // Load all the options in the IMAGE_STRUCTURE metadata
1517 391 : for (int i = 0; i < optlist.Count(); i++)
1518 : {
1519 56 : CPLString s(optlist[i]);
1520 28 : size_t nSepPos = s.find_first_of(":=");
1521 28 : if (std::string::npos != nSepPos)
1522 : {
1523 28 : s.resize(nSepPos);
1524 28 : SetMetadataItem(s, optlist.FetchNameValue(s), "IMAGE_STRUCTURE");
1525 : }
1526 : }
1527 :
1528 : // We have the options, so we can call rasterband
1529 786 : for (int i = 1; i <= nBands; i++)
1530 : {
1531 : // The overviews are low resolution copies of the current one.
1532 475 : MRFRasterBand *band = newMRFRasterBand(this, current, i);
1533 475 : if (!band)
1534 52 : return CE_Failure;
1535 :
1536 423 : GDALColorInterp ci = GCI_Undefined;
1537 :
1538 : // Default color interpretation
1539 423 : switch (nBands)
1540 : {
1541 264 : case 1:
1542 : case 2:
1543 264 : ci = (i == 1) ? GCI_GrayIndex : GCI_AlphaBand;
1544 264 : break;
1545 144 : case 3:
1546 : case 4:
1547 144 : if (i < 3)
1548 94 : ci = (i == 1) ? GCI_RedBand : GCI_GreenBand;
1549 : else
1550 50 : ci = (i == 3) ? GCI_BlueBand : GCI_AlphaBand;
1551 : }
1552 :
1553 423 : if (GetColorTable())
1554 1 : ci = GCI_PaletteIndex;
1555 :
1556 : // Legacy, deprecated
1557 423 : if (optlist.FetchBoolean("MULTISPECTRAL", FALSE))
1558 0 : ci = GCI_Undefined;
1559 :
1560 : // New style
1561 423 : if (!photometric.empty())
1562 : {
1563 15 : if ("MULTISPECTRAL" == photometric)
1564 0 : ci = GCI_Undefined;
1565 : }
1566 :
1567 423 : band->SetColorInterpretation(ci);
1568 423 : SetBand(i, band);
1569 : }
1570 :
1571 311 : CPLXMLNode *rsets = CPLGetXMLNode(config, "Rsets");
1572 311 : if (nullptr != rsets && nullptr != rsets->psChild)
1573 : {
1574 : // We have rsets
1575 :
1576 : // Regular spaced overlays, until everything fits in a single tile
1577 32 : if (EQUAL("uniform", CPLGetXMLValue(rsets, "model", "uniform")))
1578 : {
1579 32 : scale = getXMLNum(rsets, "scale", 2.0);
1580 32 : if (scale <= 1)
1581 : {
1582 0 : CPLError(CE_Failure, CPLE_AppDefined,
1583 : "MRF: zoom factor less than unit not allowed");
1584 0 : return CE_Failure;
1585 : }
1586 : // Looks like there are overlays
1587 32 : AddOverviews(int(scale));
1588 : }
1589 : else
1590 : {
1591 0 : CPLError(CE_Failure, CPLE_AppDefined, "Unknown Rset definition");
1592 0 : return CE_Failure;
1593 : }
1594 : }
1595 :
1596 311 : idxSize = IdxSize(full, int(scale));
1597 311 : if (idxSize == 0)
1598 0 : return CE_Failure;
1599 :
1600 : // If not set by the bands, get a pageSizeBytes buffer
1601 311 : if (GetPBufferSize() == 0 && !SetPBuffer(current.pageSizeBytes))
1602 0 : return CE_Failure;
1603 :
1604 311 : if (hasVersions)
1605 : { // It has versions, but how many?
1606 5 : verCount = 0; // Assume it only has one
1607 : VSIStatBufL statb;
1608 : // If the file exists, compute the last version number
1609 5 : if (0 == VSIStatL(full.idxfname, &statb))
1610 5 : verCount = int(statb.st_size / idxSize - 1);
1611 : }
1612 :
1613 311 : return CE_None;
1614 : }
1615 :
1616 0 : static inline bool has_path(const CPLString &name)
1617 : {
1618 0 : return name.find_first_of("/\\") != string::npos;
1619 : }
1620 :
1621 : // Does name look like an absolute gdal file name?
1622 4 : static inline bool is_absolute(const CPLString &name)
1623 : {
1624 4 : return (name.find_first_of("/\\") == 0) // Starts with root
1625 4 : || (name.size() > 1 && name[1] == ':' &&
1626 0 : isalpha(static_cast<unsigned char>(
1627 0 : name[0]))) // Starts with drive letter
1628 8 : || (name[0] == '<'); // Maybe it is XML
1629 : }
1630 :
1631 : // Add the dirname of path to the beginning of name, if it is relative
1632 : // returns true if name was modified
1633 4 : static inline bool make_absolute(CPLString &name, const CPLString &path)
1634 : {
1635 4 : if (!is_absolute(name) && (path.find_first_of("/\\") != string::npos))
1636 : {
1637 4 : name = path.substr(0, path.find_last_of("/\\") + 1) + name;
1638 4 : return true;
1639 : }
1640 0 : return false;
1641 : }
1642 :
1643 : /**
1644 : *\brief Get the source dataset, open it if necessary
1645 : */
1646 5 : GDALDataset *MRFDataset::GetSrcDS()
1647 : {
1648 5 : if (poSrcDS)
1649 1 : return poSrcDS;
1650 4 : if (source.empty())
1651 0 : return nullptr;
1652 :
1653 : // Stub out the error handler
1654 4 : CPLPushErrorHandler(CPLQuietErrorHandler);
1655 : // Try open the source dataset as is
1656 4 : poSrcDS =
1657 4 : GDALDataset::FromHandle(GDALOpenShared(source.c_str(), GA_ReadOnly));
1658 4 : CPLPopErrorHandler();
1659 :
1660 : // It the open fails, try again with the current dataset path prepended
1661 4 : if (!poSrcDS && make_absolute(source, fname))
1662 4 : poSrcDS = GDALDataset::FromHandle(
1663 : GDALOpenShared(source.c_str(), GA_ReadOnly));
1664 :
1665 4 : if (0 == source.find("<MRF_META>") && has_path(fname))
1666 : {
1667 : // MRF XML source, might need to patch the file names with the current
1668 : // one
1669 0 : MRFDataset *poMRFDS = dynamic_cast<MRFDataset *>(poSrcDS);
1670 0 : if (!poMRFDS)
1671 : {
1672 0 : delete poSrcDS;
1673 0 : poSrcDS = nullptr;
1674 0 : return nullptr;
1675 : }
1676 0 : make_absolute(poMRFDS->current.datfname, fname);
1677 0 : make_absolute(poMRFDS->current.idxfname, fname);
1678 : }
1679 4 : mp_safe = true; // Turn on MP safety
1680 4 : return poSrcDS;
1681 : }
1682 :
1683 : /**
1684 : *\brief Add or verify that all overlays exits
1685 : *
1686 : * @return size of the index file
1687 : */
1688 :
1689 62 : GIntBig MRFDataset::AddOverviews(int scaleIn)
1690 : {
1691 : // Fit the overlays
1692 62 : ILImage img = current;
1693 134 : while (1 != img.pagecount.x * img.pagecount.y)
1694 : {
1695 : // Adjust raster data for next level
1696 : // Adjust the offsets for indices left at this level
1697 72 : img.idxoffset += sizeof(ILIdx) * img.pagecount.l / img.size.z *
1698 72 : (img.size.z - zslice);
1699 :
1700 : // Next overview size
1701 72 : img.size.x = pcount(img.size.x, scaleIn);
1702 72 : img.size.y = pcount(img.size.y, scaleIn);
1703 72 : img.size.l++; // Increment the level
1704 72 : img.pagecount = pcount(img.size, img.pagesize);
1705 :
1706 : // And adjust the offset again, within next level
1707 72 : img.idxoffset += sizeof(ILIdx) * img.pagecount.l / img.size.z * zslice;
1708 72 : int l = static_cast<int>(img.size.l);
1709 : // Create and register the overviews for each band
1710 144 : for (int i = 1; i <= nBands; i++)
1711 : {
1712 : MRFRasterBand *b =
1713 72 : reinterpret_cast<MRFRasterBand *>(GetRasterBand(i));
1714 72 : if (!(b->GetOverview(l - 1)))
1715 72 : b->AddOverview(newMRFRasterBand(this, img, i, l));
1716 : }
1717 : }
1718 :
1719 : // Last adjustment, should be a single set of c and leftover z tiles
1720 62 : return img.idxoffset +
1721 124 : sizeof(ILIdx) * img.pagecount.l / img.size.z * (img.size.z - zslice);
1722 : }
1723 :
1724 : //
1725 : // set an entry if it doesn't already exist
1726 : //
1727 134 : static char **CSLAddIfMissing(char **papszList, const char *pszName,
1728 : const char *pszValue)
1729 : {
1730 134 : if (CSLFetchNameValue(papszList, pszName))
1731 13 : return papszList;
1732 121 : return CSLSetNameValue(papszList, pszName, pszValue);
1733 : }
1734 :
1735 : // CreateCopy implemented based on Create
1736 133 : GDALDataset *MRFDataset::CreateCopy(const char *pszFilename,
1737 : GDALDataset *poSrcDS, int /*bStrict*/,
1738 : char **papszOptions,
1739 : GDALProgressFunc pfnProgress,
1740 : void *pProgressData)
1741 : {
1742 266 : ILImage img;
1743 :
1744 133 : int x = poSrcDS->GetRasterXSize();
1745 133 : int y = poSrcDS->GetRasterYSize();
1746 133 : int nBands = poSrcDS->GetRasterCount();
1747 133 : if (nBands == 0)
1748 : {
1749 1 : CPLError(CE_Failure, CPLE_NotSupported, "nBands == 0 not supported");
1750 1 : return nullptr;
1751 : }
1752 132 : GDALRasterBand *poSrcBand1 = poSrcDS->GetRasterBand(1);
1753 :
1754 132 : GDALDataType dt = poSrcBand1->GetRasterDataType();
1755 : // Have our own options, to modify as we want
1756 132 : char **options = CSLDuplicate(papszOptions);
1757 :
1758 : const char *pszValue =
1759 132 : poSrcDS->GetMetadataItem("INTERLEAVE", "IMAGE_STRUCTURE");
1760 : options =
1761 132 : CSLAddIfMissing(options, "INTERLEAVE", pszValue ? pszValue : "PIXEL");
1762 : int xb, yb;
1763 132 : poSrcBand1->GetBlockSize(&xb, &yb);
1764 :
1765 : // Keep input block size if it exists and not explicitly set
1766 133 : if (CSLFetchNameValue(options, "BLOCKSIZE") == nullptr && xb != x &&
1767 1 : yb != y)
1768 : {
1769 1 : options = CSLAddIfMissing(options, "BLOCKXSIZE",
1770 2 : PrintDouble(xb, "%d").c_str());
1771 1 : options = CSLAddIfMissing(options, "BLOCKYSIZE",
1772 2 : PrintDouble(yb, "%d").c_str());
1773 : }
1774 :
1775 132 : MRFDataset *poDS = nullptr;
1776 : try
1777 : {
1778 : poDS = reinterpret_cast<MRFDataset *>(
1779 132 : Create(pszFilename, x, y, nBands, dt, options));
1780 :
1781 132 : if (poDS == nullptr || poDS->bCrystalized)
1782 11 : throw CPLOPrintf("MRF: Can't create %s", pszFilename);
1783 :
1784 121 : img = poDS->current; // Deal with the current one here
1785 :
1786 : // Copy data values from source
1787 280 : for (int i = 0; i < poDS->nBands; i++)
1788 : {
1789 : int bHas;
1790 : double dfData;
1791 159 : GDALRasterBand *srcBand = poSrcDS->GetRasterBand(i + 1);
1792 159 : GDALRasterBand *mBand = poDS->GetRasterBand(i + 1);
1793 159 : dfData = srcBand->GetNoDataValue(&bHas);
1794 159 : if (bHas)
1795 : {
1796 16 : poDS->vNoData.push_back(dfData);
1797 16 : mBand->SetNoDataValue(dfData);
1798 : }
1799 159 : dfData = srcBand->GetMinimum(&bHas);
1800 159 : if (bHas)
1801 17 : poDS->vMin.push_back(dfData);
1802 159 : dfData = srcBand->GetMaximum(&bHas);
1803 159 : if (bHas)
1804 17 : poDS->vMax.push_back(dfData);
1805 :
1806 : // Copy the band metadata, PAM will handle it
1807 159 : char **meta = srcBand->GetMetadata("IMAGE_STRUCTURE");
1808 159 : if (CSLCount(meta))
1809 7 : mBand->SetMetadata(meta, "IMAGE_STRUCTURE");
1810 :
1811 159 : meta = srcBand->GetMetadata();
1812 159 : if (CSLCount(meta))
1813 18 : mBand->SetMetadata(meta);
1814 : }
1815 :
1816 : // Geotags
1817 : double gt[6];
1818 121 : if (CE_None == poSrcDS->GetGeoTransform(gt))
1819 118 : poDS->SetGeoTransform(gt);
1820 :
1821 121 : const auto poSRS = poSrcDS->GetSpatialRef();
1822 121 : if (poSRS)
1823 117 : poDS->m_oSRS = *poSRS;
1824 :
1825 : // Color palette if we only have one band
1826 224 : if (1 == nBands &&
1827 103 : GCI_PaletteIndex == poSrcBand1->GetColorInterpretation())
1828 1 : poDS->SetColorTable(poSrcBand1->GetColorTable()->Clone());
1829 :
1830 : // Finally write the XML in the right file name
1831 121 : if (!poDS->Crystalize())
1832 10 : throw CPLString("MRF: Error creating files");
1833 : }
1834 21 : catch (const CPLString &e)
1835 : {
1836 21 : if (nullptr != poDS)
1837 10 : delete poDS;
1838 21 : CPLError(CE_Failure, CPLE_ObjectNull, "%s", e.c_str());
1839 21 : poDS = nullptr;
1840 : }
1841 :
1842 132 : CSLDestroy(options);
1843 132 : if (nullptr == poDS)
1844 21 : return nullptr;
1845 :
1846 111 : char **papszFileList = poDS->GetFileList();
1847 111 : poDS->oOvManager.Initialize(poDS, poDS->GetPhysicalFilename(),
1848 : papszFileList);
1849 111 : CSLDestroy(papszFileList);
1850 :
1851 111 : CPLErr err = CE_None;
1852 : // Have PAM copy all, but skip the mask
1853 111 : int nCloneFlags = GCIF_PAM_DEFAULT & ~GCIF_MASK;
1854 :
1855 : // If copy is disabled, we're done, we just created an empty MRF
1856 111 : if (!on(CSLFetchNameValue(papszOptions, "NOCOPY")))
1857 : {
1858 : // Use the GDAL copy call
1859 : // Need to flag the dataset as compressed (COMPRESSED=TRUE) to force
1860 : // block writes This might not be what we want, if the input and out
1861 : // order is truly separate
1862 109 : nCloneFlags |= GCIF_MASK; // We do copy the data, so copy the mask too
1863 : // if necessary
1864 109 : char **papszCWROptions = nullptr;
1865 : papszCWROptions =
1866 109 : CSLAddNameValue(papszCWROptions, "COMPRESSED", "TRUE");
1867 :
1868 : #ifdef HAVE_JPEG
1869 : // Use the Zen version of the CopyWholeRaster if input has a dataset
1870 : // mask and JPEGs are generated
1871 111 : if (GMF_PER_DATASET == poSrcDS->GetRasterBand(1)->GetMaskFlags() &&
1872 2 : (poDS->current.comp == IL_JPEG
1873 : #ifdef HAVE_PNG
1874 0 : || poDS->current.comp == IL_JPNG
1875 : #endif
1876 : ))
1877 : {
1878 2 : err = poDS->ZenCopy(poSrcDS, pfnProgress, pProgressData);
1879 2 : nCloneFlags ^= GCIF_MASK; // Turn the external mask off
1880 : }
1881 : else
1882 : #endif
1883 : {
1884 107 : err = GDALDatasetCopyWholeRaster(
1885 : (GDALDatasetH)poSrcDS, (GDALDatasetH)poDS, papszCWROptions,
1886 : pfnProgress, pProgressData);
1887 : }
1888 :
1889 109 : CSLDestroy(papszCWROptions);
1890 : }
1891 :
1892 111 : if (CE_None == err)
1893 111 : err = poDS->CloneInfo(poSrcDS, nCloneFlags);
1894 :
1895 111 : if (CE_Failure == err)
1896 : {
1897 0 : delete poDS;
1898 0 : return nullptr;
1899 : }
1900 :
1901 111 : return poDS;
1902 : }
1903 :
1904 : // Prepares the data so it is suitable for Zen JPEG encoding, based on input
1905 : // mask If bFBO is set, only the values of the first band are set non-zero when
1906 : // needed
1907 : template <typename T>
1908 2 : static void ZenFilter(T *buffer, GByte *mask, int nPixels, int nBands,
1909 : bool bFBO)
1910 : {
1911 524290 : for (int i = 0; i < nPixels; i++)
1912 : {
1913 524288 : if (mask[i] == 0)
1914 : { // enforce zero values
1915 516096 : for (int b = 0; b < nBands; b++)
1916 387072 : buffer[nBands * i + b] = 0;
1917 : }
1918 : else
1919 : { // enforce non-zero
1920 395264 : if (bFBO)
1921 : { // First band only
1922 197632 : bool f = true;
1923 790528 : for (int b = 0; b < nBands; b++)
1924 : {
1925 592896 : if (0 == buffer[nBands * i + b])
1926 : {
1927 0 : f = false;
1928 0 : break;
1929 : }
1930 : }
1931 197632 : if (f)
1932 197632 : buffer[nBands * i] = 1;
1933 : }
1934 : else
1935 : { // Every band
1936 790528 : for (int b = 0; b < nBands; b++)
1937 592896 : if (0 == buffer[nBands * i + b])
1938 0 : buffer[nBands * i + b] = 1;
1939 : }
1940 : }
1941 : }
1942 2 : }
1943 :
1944 : // Custom CopyWholeRaster for Zen JPEG, called when the input has a PER_DATASET
1945 : // mask Works like GDALDatasetCopyWholeRaster, but it does filter the input data
1946 : // based on the mask
1947 : //
1948 2 : CPLErr MRFDataset::ZenCopy(GDALDataset *poSrc, GDALProgressFunc pfnProgress,
1949 : void *pProgressData)
1950 : {
1951 2 : VALIDATE_POINTER1(poSrc, "MRF:ZenCopy", CE_Failure);
1952 :
1953 2 : if (!pfnProgress)
1954 0 : pfnProgress = GDALDummyProgress;
1955 :
1956 : /* -------------------------------------------------------------------- */
1957 : /* Confirm the datasets match in size and band counts. */
1958 : /* -------------------------------------------------------------------- */
1959 2 : const int nXSize = GetRasterXSize();
1960 2 : const int nYSize = GetRasterYSize();
1961 2 : const int nBandCount = GetRasterCount();
1962 :
1963 2 : if (poSrc->GetRasterXSize() != nXSize ||
1964 4 : poSrc->GetRasterYSize() != nYSize ||
1965 2 : poSrc->GetRasterCount() != nBandCount)
1966 : {
1967 0 : CPLError(CE_Failure, CPLE_AppDefined,
1968 : "Input and output dataset sizes or band counts do not\n"
1969 : "match in GDALDatasetCopyWholeRaster()");
1970 0 : return CE_Failure;
1971 : }
1972 :
1973 : /* -------------------------------------------------------------------- */
1974 : /* Get our prototype band, and assume the others are similarly */
1975 : /* configured. Also get the per_dataset mask */
1976 : /* -------------------------------------------------------------------- */
1977 2 : GDALRasterBand *poSrcPrototypeBand = poSrc->GetRasterBand(1);
1978 2 : GDALRasterBand *poDstPrototypeBand = GetRasterBand(1);
1979 2 : GDALRasterBand *poSrcMask = poSrcPrototypeBand->GetMaskBand();
1980 :
1981 2 : const int nPageXSize = current.pagesize.x;
1982 2 : const int nPageYSize = current.pagesize.y;
1983 2 : const double nTotalBlocks =
1984 2 : static_cast<double>(DIV_ROUND_UP(nYSize, nPageYSize)) *
1985 2 : static_cast<double>(DIV_ROUND_UP(nXSize, nPageXSize));
1986 2 : const GDALDataType eDT = poDstPrototypeBand->GetRasterDataType();
1987 :
1988 : // All the bands are done per block
1989 : // this flag tells us to apply the Zen filter to the first band only
1990 2 : const bool bFirstBandOnly = (current.order == IL_Interleaved);
1991 :
1992 2 : if (!pfnProgress(0.0, nullptr, pProgressData))
1993 : {
1994 0 : CPLError(CE_Failure, CPLE_UserInterrupt,
1995 : "User terminated CreateCopy()");
1996 0 : return CE_Failure;
1997 : }
1998 :
1999 2 : const int nPixelCount = nPageXSize * nPageYSize;
2000 2 : const int dts = GDALGetDataTypeSizeBytes(eDT);
2001 2 : void *buffer = VSI_MALLOC3_VERBOSE(nPixelCount, nBandCount, dts);
2002 2 : GByte *buffer_mask = nullptr;
2003 2 : if (buffer)
2004 : buffer_mask =
2005 2 : reinterpret_cast<GByte *>(VSI_MALLOC_VERBOSE(nPixelCount));
2006 :
2007 2 : if (!buffer || !buffer_mask)
2008 : {
2009 : // Just in case buffers did get allocated
2010 0 : CPLFree(buffer);
2011 0 : CPLFree(buffer_mask);
2012 0 : CPLError(CE_Failure, CPLE_OutOfMemory, "Can't allocate copy buffer");
2013 0 : return CE_Failure;
2014 : }
2015 :
2016 2 : int nBlocksDone = 0;
2017 2 : CPLErr eErr = CE_None;
2018 : // Advise the source that a complete read will be done
2019 2 : poSrc->AdviseRead(0, 0, nXSize, nYSize, nXSize, nYSize, eDT, nBandCount,
2020 2 : nullptr, nullptr);
2021 :
2022 : // For every block, break on error
2023 4 : for (int row = 0; row < nYSize && eErr == CE_None; row += nPageYSize)
2024 : {
2025 2 : int nRows = std::min(nPageYSize, nYSize - row);
2026 4 : for (int col = 0; col < nXSize && eErr == CE_None; col += nPageXSize)
2027 : {
2028 2 : int nCols = std::min(nPageXSize, nXSize - col);
2029 :
2030 : // Report
2031 2 : if (eErr == CE_None && !pfnProgress(nBlocksDone++ / nTotalBlocks,
2032 : nullptr, pProgressData))
2033 : {
2034 0 : eErr = CE_Failure;
2035 0 : CPLError(CE_Failure, CPLE_UserInterrupt,
2036 : "User terminated CreateCopy()");
2037 0 : break;
2038 : }
2039 :
2040 : // Get the data mask as byte
2041 2 : eErr = poSrcMask->RasterIO(GF_Read, col, row, nCols, nRows,
2042 : buffer_mask, nCols, nRows, GDT_Byte, 0,
2043 : 0, nullptr);
2044 :
2045 2 : if (eErr != CE_None)
2046 0 : break;
2047 :
2048 : // If there is no data at all, skip this block
2049 2 : if (MatchCount(buffer_mask, nPixelCount, static_cast<GByte>(0)) ==
2050 : nPixelCount)
2051 0 : continue;
2052 :
2053 : // get the data in the buffer, interleaved
2054 4 : eErr = poSrc->RasterIO(
2055 : GF_Read, col, row, nCols, nRows, buffer, nCols, nRows, eDT,
2056 2 : nBandCount, nullptr, static_cast<GSpacing>(nBands) * dts,
2057 2 : static_cast<GSpacing>(nBands) * dts * nCols, dts, nullptr);
2058 :
2059 2 : if (eErr != CE_None)
2060 0 : break;
2061 :
2062 : // This is JPEG, only 8 and 12(16) bits unsigned integer types are
2063 : // valid
2064 2 : switch (eDT)
2065 : {
2066 2 : case GDT_Byte:
2067 2 : ZenFilter(reinterpret_cast<GByte *>(buffer), buffer_mask,
2068 : nPixelCount, nBandCount, bFirstBandOnly);
2069 2 : break;
2070 0 : case GDT_UInt16:
2071 0 : ZenFilter(reinterpret_cast<GUInt16 *>(buffer), buffer_mask,
2072 : nPixelCount, nBandCount, bFirstBandOnly);
2073 0 : break;
2074 0 : default:
2075 0 : CPLError(CE_Failure, CPLE_AppDefined,
2076 : "Unsupported data type for Zen filter");
2077 0 : eErr = CE_Failure;
2078 0 : break;
2079 : }
2080 :
2081 : // Write
2082 2 : if (eErr == CE_None)
2083 2 : eErr = RasterIO(
2084 : GF_Write, col, row, nCols, nRows, buffer, nCols, nRows, eDT,
2085 2 : nBandCount, nullptr, static_cast<GSpacing>(nBands) * dts,
2086 2 : static_cast<GSpacing>(nBands) * dts * nCols, dts, nullptr);
2087 :
2088 : } // Columns
2089 2 : if (eErr != CE_None)
2090 0 : break;
2091 :
2092 : } // Rows
2093 :
2094 : // Cleanup
2095 2 : CPLFree(buffer);
2096 2 : CPLFree(buffer_mask);
2097 :
2098 : // Final report
2099 2 : if (eErr == CE_None && !pfnProgress(1.0, nullptr, pProgressData))
2100 : {
2101 0 : eErr = CE_Failure;
2102 0 : CPLError(CE_Failure, CPLE_UserInterrupt,
2103 : "User terminated CreateCopy()");
2104 : }
2105 :
2106 2 : return eErr;
2107 : }
2108 :
2109 : // Apply open options to the current dataset
2110 : // Called before the configuration is read
2111 202 : void MRFDataset::ProcessOpenOptions(char **papszOptions)
2112 : {
2113 404 : CPLStringList opt(papszOptions, FALSE);
2114 202 : no_errors = opt.FetchBoolean("NOERRORS", FALSE);
2115 202 : const char *val = opt.FetchNameValue("ZSLICE");
2116 202 : if (val)
2117 0 : zslice = atoi(val);
2118 202 : }
2119 :
2120 : // Apply create options to the current dataset, only valid during creation
2121 161 : void MRFDataset::ProcessCreateOptions(char **papszOptions)
2122 : {
2123 161 : assert(!bCrystalized);
2124 322 : CPLStringList opt(papszOptions, FALSE);
2125 161 : ILImage &img(full);
2126 :
2127 161 : const char *val = opt.FetchNameValue("COMPRESS");
2128 161 : if (val && IL_ERR_COMP == (img.comp = CompToken(val)))
2129 0 : throw CPLString("GDAL MRF: Error setting compression");
2130 :
2131 161 : val = opt.FetchNameValue("INTERLEAVE");
2132 161 : if (val && IL_ERR_ORD == (img.order = OrderToken(val)))
2133 0 : throw CPLString("GDAL MRF: Error setting interleave");
2134 :
2135 161 : val = opt.FetchNameValue("QUALITY");
2136 161 : if (val)
2137 6 : img.quality = atoi(val);
2138 :
2139 161 : val = opt.FetchNameValue("ZSIZE");
2140 161 : if (val)
2141 0 : img.size.z = atoi(val);
2142 :
2143 161 : val = opt.FetchNameValue("BLOCKXSIZE");
2144 161 : if (val)
2145 1 : img.pagesize.x = atoi(val);
2146 :
2147 161 : val = opt.FetchNameValue("BLOCKYSIZE");
2148 161 : if (val)
2149 1 : img.pagesize.y = atoi(val);
2150 :
2151 161 : val = opt.FetchNameValue("BLOCKSIZE");
2152 161 : if (val)
2153 30 : img.pagesize.x = img.pagesize.y = atoi(val);
2154 :
2155 161 : img.nbo = opt.FetchBoolean("NETBYTEORDER", FALSE) != FALSE;
2156 :
2157 161 : val = opt.FetchNameValue("CACHEDSOURCE");
2158 161 : if (val)
2159 : {
2160 2 : source = val;
2161 2 : nocopy = opt.FetchBoolean("NOCOPY", FALSE);
2162 : }
2163 :
2164 161 : val = opt.FetchNameValue("UNIFORM_SCALE");
2165 161 : if (val)
2166 0 : scale = atoi(val);
2167 :
2168 161 : val = opt.FetchNameValue("PHOTOMETRIC");
2169 161 : if (val)
2170 2 : photometric = val;
2171 :
2172 161 : val = opt.FetchNameValue("DATANAME");
2173 161 : if (val)
2174 0 : img.datfname = val;
2175 :
2176 161 : val = opt.FetchNameValue("INDEXNAME");
2177 161 : if (val)
2178 0 : img.idxfname = val;
2179 :
2180 161 : val = opt.FetchNameValue("SPACING");
2181 161 : if (val)
2182 0 : spacing = atoi(val);
2183 :
2184 : optlist.Assign(
2185 : CSLTokenizeString2(opt.FetchNameValue("OPTIONS"), " \t\n\r",
2186 161 : CSLT_STRIPLEADSPACES | CSLT_STRIPENDSPACES));
2187 :
2188 : // General Fixups
2189 161 : if (img.order == IL_Interleaved)
2190 40 : img.pagesize.c = img.size.c;
2191 :
2192 : // Compression dependent fixups
2193 161 : }
2194 :
2195 : /**
2196 : *\brief Create an MRF dataset, some settings can be changed later
2197 : * papszOptions might be anything that an MRF might take
2198 : * Still missing are the georeference ...
2199 : *
2200 : */
2201 :
2202 165 : GDALDataset *MRFDataset::Create(const char *pszName, int nXSize, int nYSize,
2203 : int nBandsIn, GDALDataType eType,
2204 : char **papszOptions)
2205 : {
2206 165 : if (nBandsIn == 0)
2207 : {
2208 1 : CPLError(CE_Failure, CPLE_NotSupported, "No bands defined");
2209 1 : return nullptr;
2210 : }
2211 :
2212 164 : MRFDataset *poDS = new MRFDataset();
2213 164 : CPLErr err = CE_None;
2214 164 : poDS->fname = pszName;
2215 164 : poDS->nBands = nBandsIn;
2216 :
2217 : // Don't know what to do with these in this call
2218 : // int level = -1;
2219 : // int version = 0;
2220 :
2221 164 : size_t pos = poDS->fname.find(":MRF:");
2222 164 : if (string::npos != pos)
2223 : { // Tokenize and pick known options
2224 0 : vector<string> tokens;
2225 0 : stringSplit(tokens, poDS->fname, pos + 5, ':');
2226 : // level = getnum(tokens, 'L', -1);
2227 : // version = getnum(tokens, 'V', 0);
2228 0 : poDS->zslice = getnum(tokens, 'Z', 0);
2229 0 : poDS->fname.resize(pos); // Cut the ornamentations
2230 : }
2231 :
2232 : // Try creating the mrf file early, to avoid failing on Crystalize later
2233 164 : if (!STARTS_WITH(poDS->fname.c_str(), "<MRF_META>"))
2234 : {
2235 : // Try opening it first, even though we still clobber it later
2236 164 : VSILFILE *mainfile = VSIFOpenL(poDS->fname.c_str(), "r+b");
2237 164 : if (!mainfile)
2238 : { // Then try creating it
2239 115 : mainfile = VSIFOpenL(poDS->fname.c_str(), "w+b");
2240 115 : if (!mainfile)
2241 : {
2242 3 : CPLError(CE_Failure, CPLE_OpenFailed,
2243 : "MRF: Can't open %s for writing", poDS->fname.c_str());
2244 3 : delete poDS;
2245 3 : return nullptr;
2246 : }
2247 : }
2248 161 : VSIFCloseL(mainfile);
2249 : }
2250 :
2251 : // Use the full, set some initial parameters
2252 161 : ILImage &img = poDS->full;
2253 161 : img.size = ILSize(nXSize, nYSize, 1, nBandsIn);
2254 : #ifdef HAVE_PNG
2255 161 : img.comp = IL_PNG;
2256 : #else
2257 : img.comp = IL_NONE;
2258 : #endif
2259 161 : img.order = (nBandsIn < 5) ? IL_Interleaved : IL_Separate;
2260 161 : img.pagesize = ILSize(512, 512, 1, 1);
2261 161 : img.quality = 85;
2262 161 : img.dt = eType;
2263 161 : img.dataoffset = 0;
2264 161 : img.idxoffset = 0;
2265 161 : img.hasNoData = false;
2266 161 : img.nbo = false;
2267 :
2268 : // Set the guard that tells us it needs saving before IO can take place
2269 161 : poDS->bCrystalized = FALSE;
2270 :
2271 : // Process the options, anything that an MRF might take
2272 :
2273 : try
2274 : {
2275 : // Adjust the dataset and the full image
2276 161 : poDS->ProcessCreateOptions(papszOptions);
2277 :
2278 : // Set default file names
2279 161 : if (img.datfname.empty())
2280 161 : img.datfname = getFname(poDS->GetFname(), ILComp_Ext[img.comp]);
2281 161 : if (img.idxfname.empty())
2282 161 : img.idxfname = getFname(poDS->GetFname(), ".idx");
2283 :
2284 161 : poDS->eAccess = GA_Update;
2285 : }
2286 :
2287 0 : catch (const CPLString &e)
2288 : {
2289 0 : CPLError(CE_Failure, CPLE_OpenFailed, "%s", e.c_str());
2290 0 : delete poDS;
2291 0 : return nullptr;
2292 : }
2293 :
2294 161 : poDS->current = poDS->full;
2295 161 : poDS->SetDescription(poDS->GetFname());
2296 :
2297 : // Build a MRF XML and initialize from it, this creates the bands
2298 161 : CPLXMLNode *config = poDS->BuildConfig();
2299 161 : err = poDS->Initialize(config);
2300 161 : CPLDestroyXMLNode(config);
2301 :
2302 161 : if (CPLE_None != err)
2303 : {
2304 30 : delete poDS;
2305 30 : return nullptr;
2306 : }
2307 :
2308 : // If not set by the band, get a pageSizeBytes buffer
2309 131 : if (poDS->GetPBufferSize() == 0 &&
2310 0 : !poDS->SetPBuffer(poDS->current.pageSizeBytes))
2311 : {
2312 0 : delete poDS;
2313 0 : return nullptr;
2314 : }
2315 :
2316 : // Tell PAM what our real file name is, to help it find the aux.xml
2317 131 : poDS->SetPhysicalFilename(poDS->GetFname());
2318 131 : return poDS;
2319 : }
2320 :
2321 171 : int MRFDataset::Crystalize()
2322 : {
2323 171 : if (bCrystalized || eAccess != GA_Update)
2324 : {
2325 0 : bCrystalized = TRUE;
2326 0 : return TRUE;
2327 : }
2328 :
2329 : // No need to write to disk if there is no filename. This is a
2330 : // memory only dataset.
2331 342 : if (strlen(GetDescription()) == 0 ||
2332 171 : EQUALN(GetDescription(), "<MRF_META>", 10))
2333 : {
2334 0 : bCrystalized = TRUE;
2335 0 : return TRUE;
2336 : }
2337 :
2338 171 : CPLXMLNode *config = BuildConfig();
2339 171 : if (!WriteConfig(config))
2340 20 : return FALSE;
2341 151 : CPLDestroyXMLNode(config);
2342 151 : if (!nocopy && (!IdxFP() || !DataFP()))
2343 0 : return FALSE;
2344 151 : bCrystalized = TRUE;
2345 151 : return TRUE;
2346 : }
2347 :
2348 : // Copy the first index at the end of the file and bump the version count
2349 1 : CPLErr MRFDataset::AddVersion()
2350 : {
2351 1 : VSILFILE *l_ifp = IdxFP();
2352 1 : void *tbuff = CPLMalloc(static_cast<size_t>(idxSize));
2353 1 : VSIFSeekL(l_ifp, 0, SEEK_SET);
2354 1 : VSIFReadL(tbuff, 1, static_cast<size_t>(idxSize), l_ifp);
2355 1 : verCount++; // The one we write
2356 1 : VSIFSeekL(l_ifp, idxSize * verCount,
2357 : SEEK_SET); // At the end, this can mess things up royally
2358 1 : VSIFWriteL(tbuff, 1, static_cast<size_t>(idxSize), l_ifp);
2359 1 : CPLFree(tbuff);
2360 1 : return CE_None;
2361 : }
2362 :
2363 : //
2364 : // Write a tile at the end of the data file
2365 : // If buff and size are zero, it is equivalent to erasing the tile
2366 : // If only size is zero, it is a special empty tile,
2367 : // when used for caching, offset should be 1
2368 : //
2369 : // To make it multi-processor safe, open the file in append mode
2370 : // and verify after write
2371 : //
2372 5095 : CPLErr MRFDataset::WriteTile(void *buff, GUIntBig infooffset, GUIntBig size)
2373 : {
2374 5095 : CPLErr ret = CE_None;
2375 5095 : ILIdx tinfo = {0, 0};
2376 :
2377 5095 : VSILFILE *l_dfp = DataFP();
2378 5095 : VSILFILE *l_ifp = IdxFP();
2379 :
2380 : // Verify buffer
2381 10190 : std::vector<GByte> tbuff;
2382 :
2383 5095 : if (l_ifp == nullptr || l_dfp == nullptr)
2384 0 : return CE_Failure;
2385 :
2386 : // Flag that versioned access requires a write even if empty
2387 5095 : int new_tile = false;
2388 : // If it has versions, might need to start a new one
2389 5095 : if (hasVersions)
2390 : {
2391 1 : int new_version = false; // Assume no need to build new version
2392 :
2393 : // Read the current tile info
2394 1 : VSIFSeekL(l_ifp, infooffset, SEEK_SET);
2395 1 : VSIFReadL(&tinfo, 1, sizeof(ILIdx), l_ifp);
2396 :
2397 1 : if (verCount == 0)
2398 1 : new_version = true; // No previous yet, might create a new version
2399 : else
2400 : { // We need at least two versions before we can test for changes
2401 0 : ILIdx prevtinfo = {0, 0};
2402 :
2403 : // Read the previous one
2404 0 : VSIFSeekL(l_ifp, infooffset + verCount * idxSize, SEEK_SET);
2405 0 : VSIFReadL(&prevtinfo, 1, sizeof(ILIdx), l_ifp);
2406 :
2407 : // current and previous tiles are different, might create version
2408 0 : if (tinfo.size != prevtinfo.size ||
2409 0 : tinfo.offset != prevtinfo.offset)
2410 0 : new_version = true;
2411 : }
2412 :
2413 : // tinfo contains the current info or 0,0
2414 1 : if (tinfo.size == GIntBig(net64(size)))
2415 : { // Might be identical
2416 0 : if (size != 0)
2417 : {
2418 : // Use the temporary buffer
2419 0 : tbuff.resize(static_cast<size_t>(size));
2420 0 : VSIFSeekL(l_dfp, infooffset, SEEK_SET);
2421 0 : VSIFReadL(tbuff.data(), 1, tbuff.size(), l_dfp);
2422 : // Need to write it if not the same
2423 0 : new_tile = !std::equal(tbuff.begin(), tbuff.end(),
2424 : static_cast<GByte *>(buff));
2425 0 : tbuff.clear();
2426 : }
2427 : else
2428 : {
2429 : // Writing a null tile on top of a null tile, does it count?
2430 0 : if (tinfo.offset != GIntBig(net64(GUIntBig(buff))))
2431 0 : new_tile = true;
2432 : }
2433 : }
2434 : else
2435 : {
2436 1 : new_tile = true; // Need to write it because it is different
2437 1 : if (verCount == 0 && tinfo.size == 0)
2438 0 : new_version = false; // Don't create a version if current is
2439 : // empty and there is no previous
2440 : }
2441 :
2442 1 : if (!new_tile)
2443 0 : return CE_None; // No reason to write
2444 :
2445 : // Do we need to start a new version before writing the tile?
2446 1 : if (new_version)
2447 1 : AddVersion();
2448 : }
2449 :
2450 5095 : bool same = true;
2451 5095 : if (size)
2452 0 : do
2453 : {
2454 : // start of critical MP section
2455 5084 : VSIFSeekL(l_dfp, 0, SEEK_END);
2456 5084 : GUIntBig offset = VSIFTellL(l_dfp) + spacing;
2457 :
2458 : // Spacing should be 0 in MP safe mode, this doesn't have much of
2459 : // effect Use the existing data, spacing content is not guaranteed
2460 5084 : for (GUIntBig pending = spacing; pending != 0;
2461 0 : pending -= std::min(pending, size))
2462 0 : VSIFWriteL(buff, 1,
2463 0 : static_cast<size_t>(std::min(pending, size)),
2464 : l_dfp); // Usually only once
2465 :
2466 5084 : if (static_cast<size_t>(size) !=
2467 5084 : VSIFWriteL(buff, 1, static_cast<size_t>(size), l_dfp))
2468 0 : ret = CE_Failure;
2469 : // End of critical section
2470 :
2471 5084 : tinfo.offset = net64(offset);
2472 : //
2473 : // For MP ops, check that we can read the same content, otherwise
2474 : // try again This makes the caching MRF MP safe on file systems that
2475 : // implement append mode fully, without using explicit locks
2476 : //
2477 5084 : if (CE_None == ret && mp_safe)
2478 : { // readback and check
2479 3 : if (tbuff.size() < size)
2480 3 : tbuff.resize(static_cast<size_t>(size));
2481 3 : VSIFSeekL(l_dfp, offset, SEEK_SET);
2482 3 : VSIFReadL(tbuff.data(), 1, tbuff.size(), l_dfp);
2483 3 : same = std::equal(tbuff.begin(), tbuff.end(),
2484 : static_cast<GByte *>(buff));
2485 : }
2486 5084 : } while (CE_None == ret && mp_safe && !same);
2487 :
2488 5095 : if (CE_None != ret)
2489 : {
2490 0 : CPLError(CE_Failure, CPLE_AppDefined, "MRF: Tile write failed");
2491 0 : return ret;
2492 : }
2493 :
2494 : // Convert index to net format, offset is set already
2495 5095 : tinfo.size = net64(size);
2496 : // Do nothing if the tile is empty and the file record is also empty
2497 5095 : if (!new_tile && 0 == size && nullptr == buff)
2498 : {
2499 10 : VSIFSeekL(l_ifp, infooffset, SEEK_SET);
2500 10 : VSIFReadL(&tinfo, 1, sizeof(ILIdx), l_ifp);
2501 10 : if (0 == tinfo.offset && 0 == tinfo.size)
2502 10 : return ret;
2503 : }
2504 :
2505 : // Special case, any non-zero offset will do
2506 5085 : if (nullptr != buff && 0 == size)
2507 0 : tinfo.offset = ~GUIntBig(0);
2508 :
2509 5085 : VSIFSeekL(l_ifp, infooffset, SEEK_SET);
2510 5085 : if (sizeof(tinfo) != VSIFWriteL(&tinfo, 1, sizeof(tinfo), l_ifp))
2511 : {
2512 0 : CPLError(CE_Failure, CPLE_AppDefined, "MRF: Index write failed");
2513 0 : ret = CE_Failure;
2514 : }
2515 :
2516 5085 : return ret;
2517 : }
2518 :
2519 127 : CPLErr MRFDataset::SetGeoTransform(double *gt)
2520 : {
2521 127 : if (GetAccess() != GA_Update || bCrystalized)
2522 : {
2523 0 : CPLError(CE_Failure, CPLE_NotSupported,
2524 : "SetGeoTransform only works during Create call");
2525 0 : return CE_Failure;
2526 : }
2527 127 : memcpy(GeoTransform, gt, 6 * sizeof(double));
2528 127 : bGeoTransformValid = TRUE;
2529 127 : return CE_None;
2530 : }
2531 :
2532 16 : bool MRFDataset::IsSingleTile()
2533 : {
2534 16 : if (current.pagecount.l != 1 || !source.empty() || nullptr == DataFP())
2535 4 : return FALSE;
2536 12 : return 0 == reinterpret_cast<MRFRasterBand *>(GetRasterBand(1))
2537 12 : ->GetOverviewCount();
2538 : }
2539 :
2540 : /*
2541 : * Returns 0,1,0,0,0,1 even if it was not set
2542 : */
2543 452 : CPLErr MRFDataset::GetGeoTransform(double *gt)
2544 : {
2545 452 : memcpy(gt, GeoTransform, 6 * sizeof(double));
2546 452 : if (GetMetadata("RPC") || GetGCPCount())
2547 0 : bGeoTransformValid = FALSE;
2548 452 : if (!bGeoTransformValid)
2549 0 : return CE_Failure;
2550 452 : return CE_None;
2551 : }
2552 :
2553 : /**
2554 : *\brief Read a tile index
2555 : *
2556 : * It handles the non-existent index case, for no compression
2557 : * The bias is non-zero only when the cloned index is read
2558 : */
2559 :
2560 2019 : CPLErr MRFDataset::ReadTileIdx(ILIdx &tinfo, const ILSize &pos,
2561 : const ILImage &img, const GIntBig bias)
2562 : {
2563 2019 : VSILFILE *l_ifp = IdxFP();
2564 :
2565 : // Initialize the tinfo structure, in case the files are missing
2566 2019 : if (missing)
2567 0 : return CE_None;
2568 :
2569 2019 : GIntBig offset = bias + IdxOffset(pos, img);
2570 2019 : if (l_ifp == nullptr && img.comp == IL_NONE)
2571 : {
2572 0 : tinfo.size = current.pageSizeBytes;
2573 0 : tinfo.offset = offset * tinfo.size;
2574 0 : return CE_None;
2575 : }
2576 :
2577 2019 : if (l_ifp == nullptr && IsSingleTile())
2578 : {
2579 6 : tinfo.offset = 0;
2580 6 : VSILFILE *l_dfp = DataFP(); // IsSingleTile() checks that fp is valid
2581 6 : VSIFSeekL(l_dfp, 0, SEEK_END);
2582 6 : tinfo.size = VSIFTellL(l_dfp);
2583 :
2584 : // It should be less than the pagebuffer
2585 6 : tinfo.size = std::min(tinfo.size, static_cast<GIntBig>(pbsize));
2586 6 : return CE_None;
2587 : }
2588 :
2589 2013 : if (l_ifp == nullptr)
2590 : {
2591 0 : CPLError(CE_Failure, CPLE_FileIO, "Can't open index file");
2592 0 : return CE_Failure;
2593 : }
2594 :
2595 2013 : VSIFSeekL(l_ifp, offset, SEEK_SET);
2596 2013 : if (1 != VSIFReadL(&tinfo, sizeof(ILIdx), 1, l_ifp))
2597 0 : return CE_Failure;
2598 : // Convert them to native form
2599 2013 : tinfo.offset = net64(tinfo.offset);
2600 2013 : tinfo.size = net64(tinfo.size);
2601 :
2602 2013 : if (0 == bias || 0 != tinfo.size || 0 != tinfo.offset)
2603 2012 : return CE_None;
2604 :
2605 : // zero size and zero offset in sourced index means that this portion is
2606 : // un-initialized
2607 :
2608 : // Should be cloned and the offset within the cloned index
2609 1 : offset -= bias;
2610 1 : assert(offset < bias);
2611 1 : assert(clonedSource);
2612 :
2613 : // Read this block from the remote index, prepare it and store it in the
2614 : // right place The block size in bytes, should be a multiple of 16, to have
2615 : // full index entries
2616 1 : const int CPYSZ = 32768;
2617 : // Adjust offset to the start of the block
2618 1 : offset = (offset / CPYSZ) * CPYSZ;
2619 1 : GIntBig size = std::min(size_t(CPYSZ), size_t(bias - offset));
2620 1 : size /= sizeof(ILIdx); // In records
2621 2 : vector<ILIdx> buf(static_cast<size_t>(size));
2622 1 : ILIdx *buffer = &buf[0]; // Buffer to copy the source to the clone index
2623 :
2624 : // Fetch the data from the cloned index
2625 1 : MRFDataset *pSrc = static_cast<MRFDataset *>(GetSrcDS());
2626 1 : if (nullptr == pSrc)
2627 : {
2628 0 : CPLError(CE_Failure, CPLE_FileIO, "Can't open cloned source index");
2629 0 : return CE_Failure; // Source reported the error
2630 : }
2631 :
2632 1 : VSILFILE *srcidx = pSrc->IdxFP();
2633 1 : if (nullptr == srcidx)
2634 : {
2635 0 : CPLError(CE_Failure, CPLE_FileIO, "Can't open cloned source index");
2636 0 : return CE_Failure; // Source reported the error
2637 : }
2638 :
2639 1 : VSIFSeekL(srcidx, offset, SEEK_SET);
2640 1 : size = VSIFReadL(buffer, sizeof(ILIdx), static_cast<size_t>(size), srcidx);
2641 1 : if (size != GIntBig(buf.size()))
2642 : {
2643 0 : CPLError(CE_Failure, CPLE_FileIO, "Can't read cloned source index");
2644 0 : return CE_Failure; // Source reported the error
2645 : }
2646 :
2647 : // Mark the empty records as checked, by making the offset non-zero
2648 2 : for (vector<ILIdx>::iterator it = buf.begin(); it != buf.end(); ++it)
2649 : {
2650 1 : if (it->offset == 0 && it->size == 0)
2651 0 : it->offset = net64(1);
2652 : }
2653 :
2654 : // Write it in the right place in the local index file
2655 1 : VSIFSeekL(l_ifp, bias + offset, SEEK_SET);
2656 1 : size = VSIFWriteL(&buf[0], sizeof(ILIdx), static_cast<size_t>(size), l_ifp);
2657 1 : if (size != GIntBig(buf.size()))
2658 : {
2659 0 : CPLError(CE_Failure, CPLE_FileIO, "Can't write to cloning MRF index");
2660 0 : return CE_Failure; // Source reported the error
2661 : }
2662 :
2663 : // Cloned index updated, restart this function, it will work now
2664 1 : return ReadTileIdx(tinfo, pos, img, bias);
2665 : }
2666 :
2667 : NAMESPACE_MRF_END
|
