Line data Source code
1 : /*
2 : * Copyright (c) 2002-2012, California Institute of Technology.
3 : * All rights reserved. Based on Government Sponsored Research under contracts
4 : * NAS7-1407 and/or NAS7-03001.
5 : *
6 : * Redistribution and use in source and binary forms, with or without
7 : * modification, are permitted provided that the following conditions are met:
8 : * 1. Redistributions of source code must retain the above copyright notice,
9 : * this list of conditions and the following disclaimer.
10 : * 2. Redistributions in binary form must reproduce the above copyright
11 : * notice, this list of conditions and the following disclaimer in the
12 : * documentation and/or other materials provided with the distribution.
13 : * 3. Neither the name of the California Institute of Technology (Caltech),
14 : * its operating division the Jet Propulsion Laboratory (JPL), the National
15 : * Aeronautics and Space Administration (NASA), nor the names of its
16 : * contributors may be used to endorse or promote products derived from this
17 : * software without specific prior written permission.
18 : *
19 : * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20 : * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 : * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 : * ARE DISCLAIMED. IN NO EVENT SHALL THE CALIFORNIA INSTITUTE OF TECHNOLOGY BE
23 : * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 : * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 : * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 : * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 : * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 : * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 : * POSSIBILITY OF SUCH DAMAGE.
30 : *
31 : * Portions copyright 2014-2021 Esri
32 : *
33 : * Licensed under the Apache License, Version 2.0 (the "License");
34 : * you may not use this file except in compliance with the License.
35 : * You may obtain a copy of the License at
36 : *
37 : * http://www.apache.org/licenses/LICENSE-2.0
38 : *
39 : * Unless required by applicable law or agreed to in writing, software
40 : * distributed under the License is distributed on an "AS IS" BASIS,
41 : * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
42 : * See the License for the specific language governing permissions and
43 : * limitations under the License.
44 : */
45 :
46 : /******************************************************************************
47 : *
48 : * Project: Meta Raster File Format Driver Implementation, Dataset
49 : * Purpose: Implementation of GDAL dataset
50 : *
51 : * Author: Lucian Plesea, Lucian.Plesea jpl.nasa.gov, lplesea esri.com
52 : *
53 : ******************************************************************************
54 : *
55 : * The MRF dataset and the band are closely tied together, they should be
56 : * considered a single class, or a class (dataset) with extensions (bands).
57 : *
58 : *
59 : ****************************************************************************/
60 :
61 : #include "marfa.h"
62 : #include "mrfdrivercore.h"
63 : #include "cpl_multiproc.h" /* for CPLSleep() */
64 : #include "gdal_priv.h"
65 : #include <assert.h>
66 :
67 : #include <algorithm>
68 : #include <vector>
69 : #if defined(ZSTD_SUPPORT)
70 : #include <zstd.h>
71 : #endif
72 : using std::string;
73 : using std::vector;
74 :
75 : NAMESPACE_MRF_START
76 :
77 : // Initialize as invalid
78 368 : MRFDataset::MRFDataset()
79 : : zslice(0), idxSize(0), clonedSource(FALSE), nocopy(FALSE),
80 : bypass_cache(
81 368 : CPLTestBool(CPLGetConfigOption("MRF_BYPASSCACHING", "FALSE"))),
82 : mp_safe(FALSE), hasVersions(FALSE), verCount(0),
83 : bCrystalized(TRUE), // Assume not in create mode
84 : spacing(0), no_errors(0), missing(0), poSrcDS(nullptr), level(-1),
85 : cds(nullptr), scale(0.0), pbuffer(nullptr), pbsize(0), tile(ILSize()),
86 : bdirty(0), bGeoTransformValid(TRUE), poColorTable(nullptr), Quality(0),
87 736 : pzscctx(nullptr), pzsdctx(nullptr), read_timer(), write_timer(0)
88 : {
89 368 : m_oSRS.SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER);
90 : // X0 Xx Xy Y0 Yx Yy
91 368 : double gt[6] = {0.0, 1.0, 0.0, 0.0, 0.0, 1.0};
92 :
93 368 : memcpy(GeoTransform, gt, sizeof(gt));
94 368 : ifp.FP = dfp.FP = nullptr;
95 368 : dfp.acc = GF_Read;
96 368 : ifp.acc = GF_Read;
97 368 : }
98 :
99 187 : bool MRFDataset::SetPBuffer(unsigned int sz)
100 : {
101 187 : if (sz == 0)
102 : {
103 0 : CPLFree(pbuffer);
104 0 : pbuffer = nullptr;
105 : }
106 187 : void *pbufferNew = VSIRealloc(pbuffer, sz);
107 187 : if (pbufferNew == nullptr)
108 : {
109 0 : CPLError(CE_Failure, CPLE_OutOfMemory, "Cannot allocate %u bytes", sz);
110 0 : return false;
111 : }
112 187 : pbuffer = pbufferNew;
113 187 : pbsize = sz;
114 187 : return true;
115 : }
116 :
117 256 : static GDALColorEntry GetXMLColorEntry(CPLXMLNode *p)
118 : {
119 : GDALColorEntry ce;
120 256 : ce.c1 = static_cast<short>(getXMLNum(p, "c1", 0));
121 256 : ce.c2 = static_cast<short>(getXMLNum(p, "c2", 0));
122 256 : ce.c3 = static_cast<short>(getXMLNum(p, "c3", 0));
123 256 : ce.c4 = static_cast<short>(getXMLNum(p, "c4", 255));
124 256 : return ce;
125 : }
126 :
127 : //
128 : // Called by dataset destructor or at GDAL termination, to avoid
129 : // closing datasets whose drivers have already been unloaded
130 : //
131 368 : int MRFDataset::CloseDependentDatasets()
132 : {
133 368 : int bHasDroppedRef = GDALPamDataset::CloseDependentDatasets();
134 :
135 368 : if (poSrcDS)
136 : {
137 3 : bHasDroppedRef = TRUE;
138 3 : GDALClose(GDALDataset::ToHandle(poSrcDS));
139 3 : poSrcDS = nullptr;
140 : }
141 :
142 368 : if (cds)
143 : {
144 2 : bHasDroppedRef = TRUE;
145 2 : GDALClose(GDALDataset::ToHandle(cds));
146 2 : cds = nullptr;
147 : }
148 :
149 368 : return bHasDroppedRef;
150 : }
151 :
152 736 : MRFDataset::~MRFDataset()
153 : { // Make sure everything gets written
154 368 : if (0 != write_timer.count())
155 41 : CPLDebug("MRF_Timing", "Compression took %fms",
156 41 : 1e-6 * write_timer.count());
157 :
158 368 : if (0 != read_timer.count())
159 141 : CPLDebug("MRF_Timing", "Decompression took %fms",
160 141 : 1e-6 * read_timer.count());
161 :
162 368 : if (eAccess != GA_ReadOnly && !bCrystalized)
163 51 : if (!MRFDataset::Crystalize())
164 : {
165 : // Can't return error code from a destructor, just emit the error
166 10 : CPLError(CE_Failure, CPLE_FileIO, "Error creating files");
167 : }
168 :
169 368 : MRFDataset::FlushCache(true);
170 368 : MRFDataset::CloseDependentDatasets();
171 :
172 368 : if (ifp.FP)
173 261 : VSIFCloseL(ifp.FP);
174 368 : if (dfp.FP)
175 264 : VSIFCloseL(dfp.FP);
176 :
177 368 : delete poColorTable;
178 :
179 : // CPLFree ignores being called with NULL
180 368 : CPLFree(pbuffer);
181 368 : pbsize = 0;
182 : #if defined(ZSTD_SUPPORT)
183 368 : ZSTD_freeCCtx(static_cast<ZSTD_CCtx *>(pzscctx));
184 368 : ZSTD_freeDCtx(static_cast<ZSTD_DCtx *>(pzsdctx));
185 : #endif
186 736 : }
187 :
188 : /*
189 : *\brief Format specific RasterIO, may be bypassed by BlockBasedRasterIO by
190 : *setting GDAL_FORCE_CACHING to Yes, in which case the band ReadBlock and
191 : *WriteBLock are called directly
192 : */
193 128 : CPLErr MRFDataset::IRasterIO(GDALRWFlag eRWFlag, int nXOff, int nYOff,
194 : int nXSize, int nYSize, void *pData, int nBufXSize,
195 : int nBufYSize, GDALDataType eBufType,
196 : int nBandCount, BANDMAP_TYPE panBandMap,
197 : GSpacing nPixelSpace, GSpacing nLineSpace,
198 : GSpacing nBandSpace,
199 : GDALRasterIOExtraArg *psExtraArgs)
200 : {
201 128 : CPLDebug("MRF_IO",
202 : "IRasterIO %s, %d, %d, %d, %d, bufsz %d,%d,%d strides P %d, L %d, "
203 : "B %d \n",
204 : eRWFlag == GF_Write ? "Write" : "Read", nXOff, nYOff, nXSize,
205 : nYSize, nBufXSize, nBufYSize, nBandCount,
206 : static_cast<int>(nPixelSpace), static_cast<int>(nLineSpace),
207 : static_cast<int>(nBandSpace));
208 :
209 128 : if (eRWFlag == GF_Write && !bCrystalized && !Crystalize())
210 : {
211 0 : CPLError(CE_Failure, CPLE_FileIO, "MRF: Error creating files");
212 0 : return CE_Failure;
213 : }
214 :
215 : //
216 : // Call the parent implementation, which splits it into bands and calls
217 : // their IRasterIO
218 : //
219 128 : return GDALPamDataset::IRasterIO(eRWFlag, nXOff, nYOff, nXSize, nYSize,
220 : pData, nBufXSize, nBufYSize, eBufType,
221 : nBandCount, panBandMap, nPixelSpace,
222 128 : nLineSpace, nBandSpace, psExtraArgs);
223 : }
224 :
225 : /**
226 : *\brief Build some overviews
227 : *
228 : * if nOverviews is 0, erase the overviews (reduce to base image only)
229 : */
230 :
231 31 : CPLErr MRFDataset::IBuildOverviews(const char *pszResampling, int nOverviews,
232 : const int *panOverviewList, int nBandsIn,
233 : const int *panBandList,
234 : GDALProgressFunc pfnProgress,
235 : void *pProgressData,
236 : CSLConstList papszOptions)
237 :
238 : {
239 31 : CPLErr eErr = CE_None;
240 31 : CPLDebug("MRF_OVERLAY", "IBuildOverviews %d, bands %d\n", nOverviews,
241 : nBandsIn);
242 :
243 31 : if (nBands != nBandsIn)
244 : {
245 0 : CPLError(CE_Failure, CPLE_NotSupported, "nBands = %d not supported",
246 : nBandsIn);
247 0 : return CE_Failure;
248 : }
249 :
250 : // If we don't have write access, then create external overviews
251 31 : if (GetAccess() != GA_Update)
252 : {
253 1 : CPLDebug("MRF", "File open read-only, creating overviews externally.");
254 1 : return GDALDataset::IBuildOverviews(
255 : pszResampling, nOverviews, panOverviewList, nBands, panBandList,
256 1 : pfnProgress, pProgressData, papszOptions);
257 : }
258 :
259 30 : if (nOverviews == 0)
260 : {
261 : // If there are none, nothing to do
262 0 : if (GetRasterBand(1)->GetOverviewCount() == 0)
263 0 : return CE_None;
264 :
265 0 : auto *b = static_cast<MRFRasterBand *>(GetRasterBand(1));
266 : // If the first band internal overviews don't exist, they are external
267 0 : if (b->overviews.empty())
268 0 : return GDALDataset::IBuildOverviews(
269 : pszResampling, nOverviews, panOverviewList, nBands, panBandList,
270 0 : pfnProgress, pProgressData, papszOptions);
271 :
272 : // We should clean overviews, but this is not allowed in an MRF
273 0 : CPLError(CE_Warning, CPLE_NotSupported,
274 : "MRF: Internal overviews cannot be removed, "
275 : "but they can be rebuilt");
276 0 : return CE_None;
277 : }
278 :
279 : // Array of source bands
280 : GDALRasterBand **papoBandList =
281 30 : static_cast<GDALRasterBand **>(CPLCalloc(sizeof(void *), nBands));
282 : // Array of destination bands
283 : GDALRasterBand **papoOverviewBandList =
284 30 : static_cast<GDALRasterBand **>(CPLCalloc(sizeof(void *), nBands));
285 : // Triple level pointer, that's what GDAL ROMB wants
286 : GDALRasterBand ***papapoOverviewBands =
287 30 : static_cast<GDALRasterBand ***>(CPLCalloc(sizeof(void *), nBands));
288 :
289 : int *panOverviewListNew =
290 30 : static_cast<int *>(CPLMalloc(sizeof(int) * nOverviews));
291 30 : memcpy(panOverviewListNew, panOverviewList, sizeof(int) * nOverviews);
292 :
293 : try
294 : { // Throw an error code, to make sure memory gets freed properly
295 : // Modify the metadata file if it doesn't already have the Rset model
296 : // set
297 30 : if (0.0 == scale)
298 : {
299 30 : CPLXMLNode *config = ReadConfig();
300 : try
301 : {
302 : const char *model =
303 30 : CPLGetXMLValue(config, "Rsets.model", "uniform");
304 30 : if (!EQUAL(model, "uniform"))
305 : {
306 0 : CPLError(CE_Failure, CPLE_AppDefined,
307 : "MRF:IBuildOverviews, Overviews not implemented "
308 : "for model %s",
309 : model);
310 0 : throw CE_Failure;
311 : }
312 :
313 : // The scale value is the same as first overview
314 30 : scale =
315 30 : strtod(CPLGetXMLValue(
316 : config, "Rsets.scale",
317 60 : CPLOPrintf("%d", panOverviewList[0]).c_str()),
318 : nullptr);
319 30 : if (scale == 0.0)
320 : {
321 0 : CPLError(CE_Failure, CPLE_IllegalArg,
322 : "Invalid Rsets.scale value");
323 0 : throw CE_Failure;
324 : }
325 :
326 30 : if (static_cast<int>(scale) != 2 &&
327 0 : (EQUALN("Avg", pszResampling, 3) ||
328 0 : EQUALN("Nnb", pszResampling, 3)))
329 : {
330 0 : CPLError(CE_Failure, CPLE_IllegalArg,
331 : "MRF internal resampling only works for a scale "
332 : "factor of two");
333 0 : throw CE_Failure;
334 : }
335 :
336 : // Initialize the empty overlays, all of them for a given scale
337 : // They could already exist, in which case they are not erased
338 30 : idxSize = AddOverviews(int(scale));
339 :
340 : // If we don't have overviews, don't try to generate them
341 30 : if (GetRasterBand(1)->GetOverviewCount() == 0)
342 0 : throw CE_None;
343 :
344 30 : if (!CheckFileSize(current.idxfname, idxSize, GA_Update))
345 : {
346 0 : CPLError(CE_Failure, CPLE_AppDefined,
347 : "MRF: Can't extend index file");
348 0 : throw CE_Failure;
349 : }
350 :
351 : // Set the uniform node, in case it was not set before, and
352 : // save the new configuration
353 30 : CPLSetXMLValue(config, "Rsets.#model", "uniform");
354 30 : CPLSetXMLValue(config, "Rsets.#scale", PrintDouble(scale));
355 :
356 30 : if (!WriteConfig(config))
357 : {
358 0 : CPLError(CE_Failure, CPLE_AppDefined,
359 : "MRF: Can't rewrite the metadata file");
360 0 : throw CE_Failure;
361 : }
362 30 : CPLDestroyXMLNode(config);
363 30 : config = nullptr;
364 : }
365 0 : catch (const CPLErr &)
366 : {
367 0 : CPLDestroyXMLNode(config);
368 0 : throw; // Rethrow
369 : }
370 :
371 : // To avoid issues with blacks overviews, generate all of them
372 : // if the user asked for a couple of overviews in the correct
373 : // sequence and starting with the lowest one
374 90 : if (!EQUAL(pszResampling, "NONE") &&
375 32 : nOverviews != GetRasterBand(1)->GetOverviewCount() &&
376 2 : CPLTestBool(
377 : CPLGetConfigOption("MRF_ALL_OVERVIEW_LEVELS", "YES")))
378 : {
379 2 : bool bIncreasingPowers =
380 2 : (panOverviewList[0] == static_cast<int>(scale));
381 3 : for (int i = 1; i < nOverviews; i++)
382 1 : bIncreasingPowers =
383 2 : bIncreasingPowers &&
384 1 : (panOverviewList[i] ==
385 1 : static_cast<int>(scale * panOverviewList[i - 1]));
386 :
387 2 : int ovrcount = GetRasterBand(1)->GetOverviewCount();
388 2 : if (bIncreasingPowers && nOverviews != ovrcount)
389 : {
390 2 : CPLDebug("MRF",
391 : "Generating %d levels instead of the %d requested",
392 : ovrcount, nOverviews);
393 2 : nOverviews = ovrcount;
394 4 : panOverviewListNew = reinterpret_cast<int *>(CPLRealloc(
395 2 : panOverviewListNew, sizeof(int) * nOverviews));
396 2 : panOverviewListNew[0] = static_cast<int>(scale);
397 5 : for (int i = 1; i < nOverviews; i++)
398 3 : panOverviewListNew[i] =
399 3 : static_cast<int>(scale * panOverviewListNew[i - 1]);
400 : }
401 : }
402 : }
403 :
404 30 : if (static_cast<int>(scale) != 2 && (EQUALN("Avg", pszResampling, 3) ||
405 0 : EQUALN("Nnb", pszResampling, 3)))
406 : {
407 0 : CPLError(
408 : CE_Failure, CPLE_IllegalArg,
409 : "MRF internal resampling only works for a scale factor of two");
410 0 : throw CE_Failure;
411 : }
412 :
413 63 : for (int i = 0; i < nOverviews; i++)
414 : {
415 : // Verify that scales are reasonable, val/scale has to be an integer
416 33 : if (!IsPower(panOverviewListNew[i], scale))
417 : {
418 0 : CPLError(CE_Warning, CPLE_AppDefined,
419 : "MRF:IBuildOverviews, overview factor %d is not a "
420 : "power of %f",
421 0 : panOverviewListNew[i], scale);
422 0 : continue;
423 : };
424 :
425 33 : int srclevel = int(logbase(panOverviewListNew[i], scale) - 0.5);
426 33 : MRFRasterBand *b = static_cast<MRFRasterBand *>(GetRasterBand(1));
427 :
428 : // Warn for requests for invalid levels
429 33 : if (srclevel >= b->GetOverviewCount())
430 : {
431 0 : CPLError(CE_Warning, CPLE_AppDefined,
432 : "MRF:IBuildOverviews, overview factor %d is not valid "
433 : "for this dataset",
434 0 : panOverviewListNew[i]);
435 0 : continue;
436 : }
437 :
438 : // Generate the overview using the previous level as the source
439 :
440 : // Use "avg" flag to trigger the internal average sampling
441 33 : if (EQUALN("Avg", pszResampling, 3) ||
442 19 : EQUALN("Nnb", pszResampling, 3))
443 : {
444 26 : int sampling = EQUALN("Avg", pszResampling, 3) ? SAMPLING_Avg
445 : : SAMPLING_Near;
446 : // Internal, using PatchOverview
447 26 : if (srclevel > 0)
448 : b = static_cast<MRFRasterBand *>(
449 3 : b->GetOverview(srclevel - 1));
450 :
451 : eErr =
452 26 : PatchOverview(0, 0, b->nBlocksPerRow, b->nBlocksPerColumn,
453 : srclevel, 0, sampling);
454 26 : if (eErr == CE_Failure)
455 26 : throw eErr;
456 : }
457 : else
458 : {
459 : //
460 : // Use the GDAL method, which is slightly different for bilinear
461 : // interpolation and also handles nearest mode
462 : //
463 : //
464 14 : for (int iBand = 0; iBand < nBands; iBand++)
465 : {
466 : // This is the base level
467 7 : papoBandList[iBand] = GetRasterBand(panBandList[iBand]);
468 : // Set up the destination
469 14 : papoOverviewBandList[iBand] =
470 7 : papoBandList[iBand]->GetOverview(srclevel);
471 :
472 : // Use the previous level as the source, the overviews are 0
473 : // based thus an extra -1
474 7 : if (srclevel > 0)
475 0 : papoBandList[iBand] =
476 0 : papoBandList[iBand]->GetOverview(srclevel - 1);
477 :
478 : // Hook it up, via triple pointer level
479 7 : papapoOverviewBands[iBand] = &(papoOverviewBandList[iBand]);
480 : }
481 :
482 : //
483 : // Ready, generate this overview
484 : // Note that this function has a bug in GDAL, the block stepping
485 : // is incorrect It can generate multiple overview in one call,
486 : // Could rewrite this loop so this function only gets called
487 : // once
488 : //
489 7 : GDALRegenerateOverviewsMultiBand(
490 : nBands, papoBandList, 1, papapoOverviewBands, pszResampling,
491 : pfnProgress, pProgressData, papszOptions);
492 : }
493 : }
494 : }
495 0 : catch (const CPLErr &e)
496 : {
497 0 : eErr = e;
498 : }
499 :
500 30 : CPLFree(panOverviewListNew);
501 30 : CPLFree(papapoOverviewBands);
502 30 : CPLFree(papoOverviewBandList);
503 30 : CPLFree(papoBandList);
504 30 : return eErr;
505 : }
506 :
507 : /*
508 : *\brief blank separated list to vector of doubles
509 : */
510 34 : static void list2vec(std::vector<double> &v, const char *pszList)
511 : {
512 34 : if ((pszList == nullptr) || (pszList[0] == 0))
513 0 : return;
514 34 : char **papszTokens = CSLTokenizeString2(
515 : pszList, " \t\n\r", CSLT_STRIPLEADSPACES | CSLT_STRIPENDSPACES);
516 34 : v.clear();
517 69 : for (int i = 0; i < CSLCount(papszTokens); i++)
518 35 : v.push_back(CPLStrtod(papszTokens[i], nullptr));
519 34 : CSLDestroy(papszTokens);
520 : }
521 :
522 20 : void MRFDataset::SetNoDataValue(const char *pszVal)
523 : {
524 20 : list2vec(vNoData, pszVal);
525 20 : }
526 :
527 7 : void MRFDataset::SetMinValue(const char *pszVal)
528 : {
529 7 : list2vec(vMin, pszVal);
530 7 : }
531 :
532 7 : void MRFDataset::SetMaxValue(const char *pszVal)
533 : {
534 7 : list2vec(vMax, pszVal);
535 7 : }
536 :
537 : /**
538 : *
539 : *\brief Read the XML config tree, from file
540 : * Caller is responsible for freeing the memory
541 : *
542 : * @return NULL on failure, or the document tree on success.
543 : *
544 : */
545 30 : CPLXMLNode *MRFDataset::ReadConfig() const
546 : {
547 30 : if (fname[0] == '<')
548 0 : return CPLParseXMLString(fname);
549 30 : return CPLParseXMLFile(fname);
550 : }
551 :
552 : /**
553 : *\brief Write the XML config tree
554 : * Caller is responsible for correctness of data
555 : * and for freeing the memory
556 : *
557 : * @param config The document tree to write
558 : * @return TRUE on success, FALSE otherwise
559 : */
560 201 : int MRFDataset::WriteConfig(CPLXMLNode *config)
561 : {
562 201 : if (fname[0] == '<')
563 0 : return FALSE;
564 201 : return CPLSerializeXMLTreeToFile(config, fname);
565 : }
566 :
567 : static void
568 5 : stringSplit(vector<string> &theStringVector, // Altered/returned value
569 : const string &theString, size_t start = 0,
570 : const char theDelimiter = ' ')
571 : {
572 : while (true)
573 : {
574 5 : size_t end = theString.find(theDelimiter, start);
575 5 : if (string::npos == end)
576 : {
577 5 : theStringVector.push_back(theString.substr(start));
578 5 : return;
579 : }
580 0 : theStringVector.push_back(theString.substr(start, end - start));
581 0 : start = end + 1;
582 0 : }
583 : }
584 :
585 : // Returns the number following the prefix if it exists in one of the vector
586 : // strings Otherwise it returns the default
587 15 : static int getnum(const vector<string> &theStringVector, const char prefix,
588 : int def)
589 : {
590 25 : for (unsigned int i = 0; i < theStringVector.size(); i++)
591 15 : if (theStringVector[i][0] == prefix)
592 5 : return atoi(theStringVector[i].c_str() + 1);
593 10 : return def;
594 : }
595 :
596 : /**
597 : *\brief Open a MRF file
598 : *
599 : */
600 202 : GDALDataset *MRFDataset::Open(GDALOpenInfo *poOpenInfo)
601 : {
602 202 : if (!MRFDriverIdentify(poOpenInfo))
603 0 : return nullptr;
604 :
605 202 : CPLXMLNode *config = nullptr;
606 202 : CPLErr ret = CE_None;
607 202 : const char *pszFileName = poOpenInfo->pszFilename;
608 :
609 202 : int level = -1; // All levels
610 202 : int version = 0; // Current
611 202 : int zslice = 0;
612 404 : string fn; // Used to parse and adjust the file name
613 404 : string insidefn; // inside tar file name
614 :
615 : // Different ways to open an MRF
616 202 : if (poOpenInfo->nHeaderBytes >= 10)
617 : {
618 197 : const char *pszHeader =
619 : reinterpret_cast<char *>(poOpenInfo->pabyHeader);
620 197 : fn.assign(pszHeader, poOpenInfo->nHeaderBytes);
621 197 : if (STARTS_WITH(pszHeader, "<MRF_META>")) // Regular file name
622 190 : config = CPLParseXMLFile(pszFileName);
623 7 : else if (poOpenInfo->eAccess == GA_ReadOnly && fn.size() > 600 &&
624 7 : (fn[262] == 0 || fn[262] == 32) &&
625 1 : STARTS_WITH(fn.c_str() + 257, "ustar") &&
626 15 : strlen(CPLGetPathSafe(fn.c_str()).c_str()) == 0 &&
627 1 : STARTS_WITH(fn.c_str() + 512, "<MRF_META>"))
628 : { // An MRF inside a tar
629 1 : insidefn = string("/vsitar/") + pszFileName + "/" + pszHeader;
630 1 : config = CPLParseXMLFile(insidefn.c_str());
631 : }
632 : #if defined(LERC)
633 : else
634 6 : config = LERC_Band::GetMRFConfig(poOpenInfo);
635 : #endif
636 : }
637 : else
638 : {
639 5 : if (EQUALN(pszFileName, "<MRF_META>", 10)) // Content as file name
640 0 : config = CPLParseXMLString(pszFileName);
641 : else
642 : { // Try Ornate file name
643 5 : fn = pszFileName;
644 5 : size_t pos = fn.find(":MRF:");
645 5 : if (string::npos != pos)
646 : { // Tokenize and pick known options
647 5 : vector<string> tokens;
648 5 : stringSplit(tokens, fn, pos + 5, ':');
649 5 : level = getnum(tokens, 'L', -1);
650 5 : version = getnum(tokens, 'V', 0);
651 5 : zslice = getnum(tokens, 'Z', 0);
652 5 : fn.resize(pos); // Cut the ornamentations
653 5 : pszFileName = fn.c_str();
654 5 : config = CPLParseXMLFile(pszFileName);
655 : }
656 : }
657 : }
658 :
659 202 : if (!config)
660 0 : return nullptr;
661 :
662 202 : MRFDataset *ds = new MRFDataset();
663 202 : ds->fname = pszFileName;
664 202 : if (!insidefn.empty())
665 : {
666 1 : ds->publicname = pszFileName;
667 1 : ds->fname = insidefn;
668 : }
669 202 : ds->eAccess = poOpenInfo->eAccess;
670 202 : ds->level = level;
671 202 : ds->zslice = zslice;
672 :
673 : // OpenOptions can override file name arguments
674 202 : ds->ProcessOpenOptions(poOpenInfo->papszOpenOptions);
675 :
676 202 : if (level == -1)
677 200 : ret = ds->Initialize(config);
678 : else
679 : {
680 : // Open the whole dataset, then pick one level
681 2 : ds->cds = new MRFDataset();
682 2 : ds->cds->fname = ds->fname;
683 2 : ds->cds->eAccess = ds->eAccess;
684 2 : ds->zslice = zslice;
685 2 : ret = ds->cds->Initialize(config);
686 2 : if (ret == CE_None)
687 2 : ret = ds->LevelInit(level);
688 : }
689 202 : CPLDestroyXMLNode(config);
690 :
691 202 : if (ret != CE_None)
692 : {
693 23 : delete ds;
694 23 : return nullptr;
695 : }
696 :
697 : // Open a single version
698 179 : if (version != 0)
699 2 : ret = ds->SetVersion(version);
700 :
701 179 : if (ret != CE_None)
702 : {
703 1 : delete ds;
704 1 : return nullptr;
705 : }
706 :
707 : // Tell PAM what our real file name is, to help it find the aux.xml
708 178 : ds->SetPhysicalFilename(ds->fname);
709 : // Don't mess with metadata after this, otherwise PAM will re-write the
710 : // aux.xml
711 178 : ds->TryLoadXML();
712 :
713 : /* -------------------------------------------------------------------- */
714 : /* Open external overviews. */
715 : /* -------------------------------------------------------------------- */
716 178 : ds->oOvManager.Initialize(ds, ds->fname);
717 :
718 178 : return ds;
719 : }
720 :
721 : // Adjust the band images with the right offset, then adjust the sizes
722 2 : CPLErr MRFDataset::SetVersion(int version)
723 : {
724 2 : if (!hasVersions || version > verCount)
725 : {
726 1 : CPLError(CE_Failure, CPLE_AppDefined,
727 : "GDAL MRF: Version number error!");
728 1 : return CE_Failure;
729 : }
730 : // Size of one version index
731 2 : for (int bcount = 1; bcount <= nBands; bcount++)
732 : {
733 : MRFRasterBand *srcband =
734 1 : reinterpret_cast<MRFRasterBand *>(GetRasterBand(bcount));
735 1 : srcband->img.idxoffset += idxSize * verCount;
736 1 : for (int l = 0; l < srcband->GetOverviewCount(); l++)
737 : {
738 : MRFRasterBand *band =
739 0 : reinterpret_cast<MRFRasterBand *>(srcband->GetOverview(l));
740 0 : if (band != nullptr)
741 0 : band->img.idxoffset += idxSize * verCount;
742 : }
743 : }
744 1 : hasVersions = 0;
745 1 : return CE_None;
746 : }
747 :
748 2 : CPLErr MRFDataset::LevelInit(const int l)
749 : {
750 : // Test that this level does exist
751 2 : if (l < 0 || l >= cds->GetRasterBand(1)->GetOverviewCount())
752 : {
753 1 : CPLError(CE_Failure, CPLE_AppDefined,
754 : "GDAL MRF: Overview not present!");
755 1 : return CE_Failure;
756 : }
757 :
758 : MRFRasterBand *srcband = reinterpret_cast<MRFRasterBand *>(
759 1 : cds->GetRasterBand(1)->GetOverview(l));
760 :
761 : // Copy the sizes from this level
762 1 : full = srcband->img;
763 1 : current = srcband->img;
764 1 : current.size.c = cds->current.size.c;
765 1 : scale = cds->scale;
766 1 : const auto poSRS = cds->GetSpatialRef();
767 1 : if (poSRS)
768 1 : m_oSRS = *poSRS;
769 :
770 1 : SetMetadataItem("INTERLEAVE", OrderName(current.order), "IMAGE_STRUCTURE");
771 1 : SetMetadataItem("COMPRESSION", CompName(current.comp), "IMAGE_STRUCTURE");
772 :
773 1 : bGeoTransformValid = (CE_None == cds->GetGeoTransform(GeoTransform));
774 4 : for (int i = 0; i < l + 1; i++)
775 : {
776 3 : GeoTransform[1] *= scale;
777 3 : GeoTransform[5] *= scale;
778 : }
779 :
780 1 : nRasterXSize = current.size.x;
781 1 : nRasterYSize = current.size.y;
782 1 : nBands = current.size.c;
783 :
784 : // Add the bands, copy constructor so they can be closed independently
785 2 : for (int i = 1; i <= nBands; i++)
786 1 : SetBand(i, new MRFLRasterBand(reinterpret_cast<MRFRasterBand *>(
787 1 : cds->GetRasterBand(i)->GetOverview(l))));
788 1 : return CE_None;
789 : }
790 :
791 : // Is the string positive or not
792 1268 : inline bool on(const char *pszValue)
793 : {
794 1268 : if (!pszValue || pszValue[0] == 0)
795 108 : return false;
796 2308 : return EQUAL(pszValue, "ON") || EQUAL(pszValue, "TRUE") ||
797 2308 : EQUAL(pszValue, "YES");
798 : }
799 :
800 : /**
801 : *\brief Initialize the image structure and the dataset from the XML Raster node
802 : *
803 : * @param image the structure to be initialized
804 : * @param ds the parent dataset, some things get inherited
805 : * @param defimage defimage
806 : *
807 : * The structure should be initialized with the default values as much as
808 : *possible
809 : *
810 : */
811 :
812 363 : static CPLErr Init_Raster(ILImage &image, MRFDataset *ds, CPLXMLNode *defimage)
813 : {
814 : CPLXMLNode *node; // temporary
815 363 : if (!defimage)
816 : {
817 0 : CPLError(CE_Failure, CPLE_AppDefined,
818 : "GDAL MRF: Can't find raster info");
819 0 : return CE_Failure;
820 : }
821 :
822 : // Size is mandatory
823 363 : node = CPLGetXMLNode(defimage, "Size");
824 :
825 363 : if (node)
826 : {
827 363 : image.size = ILSize(static_cast<int>(getXMLNum(node, "x", -1)),
828 363 : static_cast<int>(getXMLNum(node, "y", -1)),
829 363 : static_cast<int>(getXMLNum(node, "z", 1)),
830 363 : static_cast<int>(getXMLNum(node, "c", 1)), 0);
831 : }
832 :
833 : // Basic checks
834 363 : if (!node || image.size.x < 1 || image.size.y < 1 || image.size.z < 0 ||
835 726 : image.size.c < 0 || !GDALCheckBandCount(image.size.c, FALSE))
836 : {
837 0 : CPLError(CE_Failure, CPLE_AppDefined, "Raster size missing or invalid");
838 0 : return CE_Failure;
839 : }
840 :
841 : // Pagesize, defaults to 512,512,1,c
842 363 : image.pagesize = ILSize(std::min(512, image.size.x),
843 363 : std::min(512, image.size.y), 1, image.size.c);
844 :
845 363 : node = CPLGetXMLNode(defimage, "PageSize");
846 363 : if (node)
847 : {
848 363 : image.pagesize =
849 363 : ILSize(static_cast<int>(getXMLNum(node, "x", image.pagesize.x)),
850 363 : static_cast<int>(getXMLNum(node, "y", image.pagesize.y)),
851 : 1, // One slice at a time, forced
852 363 : static_cast<int>(getXMLNum(node, "c", image.pagesize.c)));
853 363 : if (image.pagesize.x < 1 || image.pagesize.y < 1 ||
854 363 : image.pagesize.c <= 0)
855 : {
856 0 : CPLError(CE_Failure, CPLE_IllegalArg, "Invalid PageSize");
857 0 : return CE_Failure;
858 : }
859 : }
860 :
861 : // Page Encoding, defaults to PNG
862 363 : const char *pszCompression = CPLGetXMLValue(defimage, "Compression", "PNG");
863 363 : image.comp = CompToken(pszCompression);
864 363 : if (image.comp == IL_ERR_COMP)
865 : {
866 0 : CPLError(CE_Failure, CPLE_IllegalArg,
867 : "GDAL MRF: Compression %s is unknown", pszCompression);
868 0 : return CE_Failure;
869 : }
870 :
871 : // Is there a palette?
872 : //
873 : // GDAL only supports RGB+A palette, the other modes don't work
874 : //
875 :
876 669 : if ((image.pagesize.c == 1) &&
877 306 : (nullptr != (node = CPLGetXMLNode(defimage, "Palette"))))
878 : {
879 1 : int entries = static_cast<int>(getXMLNum(node, "Size", 255));
880 1 : GDALPaletteInterp eInterp = GPI_RGB;
881 1 : if ((entries > 0) && (entries < 257))
882 : {
883 1 : GDALColorEntry ce_start = {0, 0, 0, 255}, ce_end = {0, 0, 0, 255};
884 :
885 : // Create it and initialize it to black opaque
886 1 : GDALColorTable *poColorTable = new GDALColorTable(eInterp);
887 1 : poColorTable->CreateColorRamp(0, &ce_start, entries - 1, &ce_end);
888 : // Read the values
889 1 : CPLXMLNode *p = CPLGetXMLNode(node, "Entry");
890 1 : if (p)
891 : {
892 : // Initialize the first entry
893 1 : ce_start = GetXMLColorEntry(p);
894 1 : int start_idx = static_cast<int>(getXMLNum(p, "idx", 0));
895 1 : if (start_idx < 0)
896 : {
897 0 : CPLError(CE_Failure, CPLE_IllegalArg,
898 : "GDAL MRF: Palette index %d not allowed",
899 : start_idx);
900 0 : delete poColorTable;
901 0 : return CE_Failure;
902 : }
903 1 : poColorTable->SetColorEntry(start_idx, &ce_start);
904 256 : while (nullptr != (p = SearchXMLSiblings(p, "Entry")))
905 : {
906 : // For every entry, create a ramp
907 255 : ce_end = GetXMLColorEntry(p);
908 : int end_idx =
909 255 : static_cast<int>(getXMLNum(p, "idx", start_idx + 1));
910 255 : if ((end_idx <= start_idx) || (start_idx >= entries))
911 : {
912 0 : CPLError(CE_Failure, CPLE_IllegalArg,
913 : "GDAL MRF: Index Error at index %d", end_idx);
914 0 : delete poColorTable;
915 0 : return CE_Failure;
916 : }
917 255 : poColorTable->CreateColorRamp(start_idx, &ce_start, end_idx,
918 : &ce_end);
919 255 : ce_start = ce_end;
920 255 : start_idx = end_idx;
921 : }
922 : }
923 :
924 1 : ds->SetColorTable(poColorTable);
925 : }
926 : else
927 : {
928 0 : CPLError(CE_Failure, CPLE_IllegalArg,
929 : "GDAL MRF: Palette definition error");
930 0 : return CE_Failure;
931 : }
932 : }
933 :
934 : // Order of increment
935 363 : if (image.pagesize.c != image.size.c && image.pagesize.c != 1)
936 : {
937 : // Fixes heap buffer overflow in
938 : // GDALMRFRasterBand::ReadInterleavedBlock() See
939 : // https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=2884
940 0 : CPLError(CE_Failure, CPLE_NotSupported,
941 : "GDAL MRF: image.pagesize.c = %d and image.size.c = %d",
942 : image.pagesize.c, image.size.c);
943 0 : return CE_Failure;
944 : }
945 :
946 363 : image.order = OrderToken(
947 : CPLGetXMLValue(defimage, "Order",
948 363 : (image.pagesize.c != image.size.c) ? "BAND" : "PIXEL"));
949 363 : if (image.order == IL_ERR_ORD)
950 : {
951 0 : CPLError(CE_Failure, CPLE_AppDefined, "GDAL MRF: Order %s is unknown",
952 : CPLGetXMLValue(defimage, "Order", nullptr));
953 0 : return CE_Failure;
954 : }
955 :
956 363 : const char *photo_val = CPLGetXMLValue(defimage, "Photometric", nullptr);
957 363 : if (photo_val)
958 5 : ds->SetPhotometricInterpretation(photo_val);
959 :
960 363 : image.quality = atoi(CPLGetXMLValue(defimage, "Quality", "85"));
961 363 : if (image.quality < 0 || image.quality > 99)
962 : {
963 0 : CPLError(CE_Warning, CPLE_AppDefined,
964 : "GDAL MRF: Quality setting error, using default of 85");
965 0 : image.quality = 85;
966 : }
967 :
968 : // Data Type, use GDAL Names
969 363 : image.dt = GDALGetDataTypeByName(
970 : CPLGetXMLValue(defimage, "DataType", GDALGetDataTypeName(image.dt)));
971 363 : if (image.dt == GDT_Unknown || GDALGetDataTypeSize(image.dt) == 0)
972 : {
973 0 : CPLError(CE_Failure, CPLE_AppDefined, "GDAL MRF: Unsupported type");
974 0 : return CE_Failure;
975 : }
976 :
977 : // Check the endianness if needed, assume host order
978 363 : if (is_Endianness_Dependent(image.dt, image.comp))
979 69 : image.nbo = on(CPLGetXMLValue(defimage, "NetByteOrder", "No"));
980 :
981 363 : CPLXMLNode *DataValues = CPLGetXMLNode(defimage, "DataValues");
982 363 : if (nullptr != DataValues)
983 : {
984 27 : const char *pszValue = CPLGetXMLValue(DataValues, "NoData", nullptr);
985 27 : if (pszValue)
986 20 : ds->SetNoDataValue(pszValue);
987 27 : pszValue = CPLGetXMLValue(DataValues, "min", nullptr);
988 27 : if (pszValue)
989 7 : ds->SetMinValue(pszValue);
990 27 : pszValue = CPLGetXMLValue(DataValues, "max", nullptr);
991 27 : if (pszValue)
992 7 : ds->SetMaxValue(pszValue);
993 : }
994 :
995 : // Calculate the page size in bytes
996 363 : const int nDTSize = GDALGetDataTypeSizeBytes(image.dt);
997 363 : if (nDTSize <= 0 || image.pagesize.z <= 0 ||
998 363 : image.pagesize.x > INT_MAX / image.pagesize.y ||
999 363 : image.pagesize.x * image.pagesize.y > INT_MAX / image.pagesize.z ||
1000 363 : image.pagesize.x * image.pagesize.y * image.pagesize.z >
1001 363 : INT_MAX / image.pagesize.c ||
1002 363 : image.pagesize.x * image.pagesize.y * image.pagesize.z *
1003 363 : image.pagesize.c >
1004 363 : INT_MAX / nDTSize)
1005 : {
1006 0 : CPLError(CE_Failure, CPLE_AppDefined, "MRF page size too big");
1007 0 : return CE_Failure;
1008 : }
1009 363 : image.pageSizeBytes = nDTSize * image.pagesize.x * image.pagesize.y *
1010 363 : image.pagesize.z * image.pagesize.c;
1011 :
1012 : // Calculate the page count, including the total for the level
1013 363 : image.pagecount = pcount(image.size, image.pagesize);
1014 363 : if (image.pagecount.l < 0)
1015 : {
1016 0 : return CE_Failure;
1017 : }
1018 :
1019 : // Data File Name and base offset
1020 : image.datfname =
1021 363 : getFname(defimage, "DataFile", ds->GetFname(), ILComp_Ext[image.comp]);
1022 363 : image.dataoffset = static_cast<int>(
1023 363 : getXMLNum(CPLGetXMLNode(defimage, "DataFile"), "offset", 0.0));
1024 :
1025 : // Index File Name and base offset
1026 363 : image.idxfname = getFname(defimage, "IndexFile", ds->GetFname(), ".idx");
1027 363 : image.idxoffset = static_cast<int>(
1028 363 : getXMLNum(CPLGetXMLNode(defimage, "IndexFile"), "offset", 0.0));
1029 :
1030 363 : return CE_None;
1031 : }
1032 :
1033 159 : char **MRFDataset::GetFileList()
1034 : {
1035 159 : char **papszFileList = nullptr;
1036 :
1037 159 : string usename = fname;
1038 159 : if (!publicname.empty())
1039 0 : usename = publicname;
1040 : // Add the header file name if it is real
1041 : VSIStatBufL sStat;
1042 159 : if (VSIStatExL(usename.c_str(), &sStat, VSI_STAT_EXISTS_FLAG) == 0)
1043 159 : papszFileList = CSLAddString(papszFileList, usename.c_str());
1044 :
1045 : // These two should be real
1046 : // We don't really want to add these files, since they will be erased when
1047 : // an mrf is overwritten This collides with the concept that the data file
1048 : // never shrinks. Same goes with the index, in case we just want to add
1049 : // things to it.
1050 : // papszFileList = CSLAddString( papszFileList, full.datfname);
1051 : // papszFileList = CSLAddString( papszFileList, full.idxfname);
1052 : // if (!source.empty())
1053 : // papszFileList = CSLAddString( papszFileList, source);
1054 :
1055 318 : return papszFileList;
1056 : }
1057 :
1058 : // Try to create all the folders in the path in sequence, ignore errors
1059 4 : static void mkdir_r(string const &fname)
1060 : {
1061 4 : size_t loc = fname.find_first_of("\\/");
1062 4 : if (loc == string::npos)
1063 0 : return;
1064 : while (true)
1065 : {
1066 5 : ++loc;
1067 5 : loc = fname.find_first_of("\\/", loc);
1068 5 : if (loc == string::npos)
1069 4 : break;
1070 1 : VSIMkdir(fname.substr(0, loc).c_str(), 0);
1071 : }
1072 : }
1073 :
1074 : // Returns the dataset index file or null
1075 7254 : VSILFILE *MRFDataset::IdxFP()
1076 : {
1077 7254 : if (ifp.FP != nullptr)
1078 6987 : return ifp.FP;
1079 :
1080 : // If missing is set, we already checked, there is no index
1081 267 : if (missing)
1082 0 : return nullptr;
1083 :
1084 : // If name starts with '(' it is not a real file name
1085 267 : if (current.idxfname[0] == '(')
1086 0 : return nullptr;
1087 :
1088 267 : const char *mode = "rb";
1089 267 : ifp.acc = GF_Read;
1090 :
1091 267 : if (eAccess == GA_Update || !source.empty())
1092 : {
1093 158 : mode = "r+b";
1094 158 : ifp.acc = GF_Write;
1095 : }
1096 :
1097 267 : ifp.FP = VSIFOpenL(current.idxfname, mode);
1098 :
1099 : // If file didn't open for reading and no_errors is set, just return null
1100 : // and make a note
1101 267 : if (ifp.FP == nullptr && eAccess == GA_ReadOnly && no_errors)
1102 : {
1103 0 : missing = 1;
1104 0 : return nullptr;
1105 : }
1106 :
1107 : // need to create the index file
1108 367 : if (ifp.FP == nullptr && !bCrystalized &&
1109 100 : (eAccess == GA_Update || !source.empty()))
1110 : {
1111 100 : mode = "w+b";
1112 100 : ifp.FP = VSIFOpenL(current.idxfname, mode);
1113 : }
1114 :
1115 267 : if (nullptr == ifp.FP && !source.empty())
1116 : {
1117 : // caching and cloning, try making the folder and attempt again
1118 4 : mkdir_r(current.idxfname);
1119 4 : ifp.FP = VSIFOpenL(current.idxfname, mode);
1120 : }
1121 :
1122 267 : GIntBig expected_size = idxSize;
1123 267 : if (clonedSource)
1124 2 : expected_size *= 2;
1125 :
1126 267 : if (nullptr != ifp.FP)
1127 : {
1128 406 : if (!bCrystalized &&
1129 149 : !CheckFileSize(current.idxfname, expected_size, GA_Update))
1130 : {
1131 0 : CPLError(CE_Failure, CPLE_FileIO,
1132 : "MRF: Can't extend the cache index file %s",
1133 : current.idxfname.c_str());
1134 0 : return nullptr;
1135 : }
1136 :
1137 257 : if (source.empty())
1138 253 : return ifp.FP;
1139 :
1140 : // Make sure the index is large enough before proceeding
1141 : // Timeout in .1 seconds, can't really guarantee the accuracy
1142 : // So this is about half second, should be sufficient
1143 4 : int timeout = 5;
1144 0 : do
1145 : {
1146 4 : if (CheckFileSize(current.idxfname, expected_size, GA_ReadOnly))
1147 4 : return ifp.FP;
1148 0 : CPLSleep(0.100); /* 100 ms */
1149 0 : } while (--timeout);
1150 :
1151 : // If we get here it is a time-out
1152 0 : CPLError(CE_Failure, CPLE_AppDefined,
1153 : "GDAL MRF: Timeout on fetching cloned index file %s\n",
1154 : current.idxfname.c_str());
1155 0 : return nullptr;
1156 : }
1157 :
1158 : // If single tile, and no index file, let the caller figure it out
1159 10 : if (IsSingleTile())
1160 6 : return nullptr;
1161 :
1162 : // Error if this is not a caching MRF
1163 4 : if (source.empty())
1164 : {
1165 0 : CPLError(CE_Failure, CPLE_AppDefined,
1166 : "GDAL MRF: Can't open index file %s\n",
1167 : current.idxfname.c_str());
1168 0 : return nullptr;
1169 : }
1170 :
1171 : // Caching/Cloning MRF and index could be read only
1172 : // Is this actually works, we should try again, maybe somebody else just
1173 : // created the file?
1174 4 : mode = "rb";
1175 4 : ifp.acc = GF_Read;
1176 4 : ifp.FP = VSIFOpenL(current.idxfname, mode);
1177 4 : if (nullptr != ifp.FP)
1178 0 : return ifp.FP;
1179 :
1180 : // Caching and index file absent, create it
1181 : // Due to a race, multiple processes might do this at the same time, but
1182 : // that is fine
1183 4 : ifp.FP = VSIFOpenL(current.idxfname, "wb");
1184 4 : if (nullptr == ifp.FP)
1185 : {
1186 0 : CPLError(CE_Failure, CPLE_AppDefined,
1187 : "Can't create the MRF cache index file %s",
1188 : current.idxfname.c_str());
1189 0 : return nullptr;
1190 : }
1191 4 : VSIFCloseL(ifp.FP);
1192 4 : ifp.FP = nullptr;
1193 :
1194 : // Make it large enough for caching and for cloning
1195 4 : if (!CheckFileSize(current.idxfname, expected_size, GA_Update))
1196 : {
1197 0 : CPLError(CE_Failure, CPLE_AppDefined,
1198 : "Can't extend the cache index file %s",
1199 : current.idxfname.c_str());
1200 0 : return nullptr;
1201 : }
1202 :
1203 : // Try opening it again in rw mode so we can read and write
1204 4 : mode = "r+b";
1205 4 : ifp.acc = GF_Write;
1206 4 : ifp.FP = VSIFOpenL(current.idxfname.c_str(), mode);
1207 :
1208 4 : if (nullptr == ifp.FP)
1209 : {
1210 0 : CPLError(CE_Failure, CPLE_AppDefined,
1211 : "GDAL MRF: Can't reopen cache index file %s\n",
1212 : full.idxfname.c_str());
1213 0 : return nullptr;
1214 : }
1215 4 : return ifp.FP;
1216 : }
1217 :
1218 : //
1219 : // Returns the dataset data file or null
1220 : // Data file is opened either in Read or Append mode, never in straight write
1221 : //
1222 7129 : VSILFILE *MRFDataset::DataFP()
1223 : {
1224 7129 : if (dfp.FP != nullptr)
1225 6865 : return dfp.FP;
1226 264 : const char *mode = "rb";
1227 264 : dfp.acc = GF_Read;
1228 :
1229 : // Open it for writing if updating or if caching
1230 264 : if (eAccess == GA_Update || !source.empty())
1231 : {
1232 157 : mode = "a+b";
1233 157 : dfp.acc = GF_Write;
1234 : }
1235 :
1236 264 : dfp.FP = VSIFOpenL(current.datfname, mode);
1237 264 : if (dfp.FP)
1238 264 : return dfp.FP;
1239 :
1240 : // It could be a caching MRF
1241 0 : if (source.empty())
1242 0 : goto io_error;
1243 :
1244 : // May be there but read only, remember that it was open that way
1245 0 : mode = "rb";
1246 0 : dfp.acc = GF_Read;
1247 0 : dfp.FP = VSIFOpenL(current.datfname, mode);
1248 0 : if (nullptr != dfp.FP)
1249 : {
1250 0 : CPLDebug("MRF_IO", "Opened %s RO mode %s\n", current.datfname.c_str(),
1251 : mode);
1252 0 : return dfp.FP;
1253 : }
1254 :
1255 0 : if (source.empty())
1256 0 : goto io_error;
1257 :
1258 : // caching, maybe the folder didn't exist
1259 0 : mkdir_r(current.datfname);
1260 0 : mode = "a+b";
1261 0 : dfp.acc = GF_Write;
1262 0 : dfp.FP = VSIFOpenL(current.datfname, mode);
1263 0 : if (dfp.FP)
1264 0 : return dfp.FP;
1265 :
1266 0 : io_error:
1267 0 : dfp.FP = nullptr;
1268 0 : CPLError(CE_Failure, CPLE_FileIO, "GDAL MRF: %s : %s", strerror(errno),
1269 : current.datfname.c_str());
1270 0 : return nullptr;
1271 : }
1272 :
1273 : // Builds an XML tree from the current MRF. If written to a file it becomes an
1274 : // MRF
1275 332 : CPLXMLNode *MRFDataset::BuildConfig()
1276 : {
1277 332 : CPLXMLNode *config = CPLCreateXMLNode(nullptr, CXT_Element, "MRF_META");
1278 :
1279 332 : if (!source.empty())
1280 : {
1281 : CPLXMLNode *psCachedSource =
1282 4 : CPLCreateXMLNode(config, CXT_Element, "CachedSource");
1283 : // Should wrap the string in CDATA, in case it is XML
1284 : CPLXMLNode *psSource =
1285 4 : CPLCreateXMLElementAndValue(psCachedSource, "Source", source);
1286 4 : if (clonedSource)
1287 0 : CPLSetXMLValue(psSource, "#clone", "true");
1288 : }
1289 :
1290 : // Use the full size
1291 332 : CPLXMLNode *raster = CPLCreateXMLNode(config, CXT_Element, "Raster");
1292 :
1293 : // Preserve the file names if not the default ones
1294 332 : if (full.datfname != getFname(GetFname(), ILComp_Ext[full.comp]))
1295 0 : CPLCreateXMLElementAndValue(raster, "DataFile", full.datfname.c_str());
1296 332 : if (full.idxfname != getFname(GetFname(), ".idx"))
1297 0 : CPLCreateXMLElementAndValue(raster, "IndexFile", full.idxfname.c_str());
1298 332 : if (spacing != 0)
1299 0 : XMLSetAttributeVal(raster, "Spacing", static_cast<double>(spacing),
1300 : "%.0f");
1301 :
1302 332 : XMLSetAttributeVal(raster, "Size", full.size, "%.0f");
1303 332 : XMLSetAttributeVal(raster, "PageSize", full.pagesize, "%.0f");
1304 :
1305 : #ifdef HAVE_PNG
1306 332 : if (full.comp != IL_PNG)
1307 : #endif
1308 : {
1309 192 : CPLCreateXMLElementAndValue(raster, "Compression", CompName(full.comp));
1310 : }
1311 :
1312 332 : if (full.dt != GDT_Byte)
1313 202 : CPLCreateXMLElementAndValue(raster, "DataType",
1314 : GDALGetDataTypeName(full.dt));
1315 :
1316 : // special photometric interpretation
1317 332 : if (!photometric.empty())
1318 4 : CPLCreateXMLElementAndValue(raster, "Photometric", photometric);
1319 :
1320 332 : if (!vNoData.empty() || !vMin.empty() || !vMax.empty())
1321 : {
1322 : CPLXMLNode *values =
1323 43 : CPLCreateXMLNode(raster, CXT_Element, "DataValues");
1324 43 : XMLSetAttributeVal(values, "NoData", vNoData);
1325 43 : XMLSetAttributeVal(values, "min", vMin);
1326 43 : XMLSetAttributeVal(values, "max", vMax);
1327 : }
1328 :
1329 : // palette, if we have one
1330 332 : if (poColorTable != nullptr)
1331 : {
1332 1 : const char *pfrmt = "%.0f";
1333 1 : CPLXMLNode *pal = CPLCreateXMLNode(raster, CXT_Element, "Palette");
1334 1 : int sz = poColorTable->GetColorEntryCount();
1335 1 : if (sz != 256)
1336 0 : XMLSetAttributeVal(pal, "Size", poColorTable->GetColorEntryCount());
1337 : // RGB or RGBA for now
1338 257 : for (int i = 0; i < sz; i++)
1339 : {
1340 256 : CPLXMLNode *entry = CPLCreateXMLNode(pal, CXT_Element, "Entry");
1341 256 : const GDALColorEntry *ent = poColorTable->GetColorEntry(i);
1342 : // No need to set the index, it is always from 0 no size-1
1343 256 : XMLSetAttributeVal(entry, "c1", ent->c1, pfrmt);
1344 256 : XMLSetAttributeVal(entry, "c2", ent->c2, pfrmt);
1345 256 : XMLSetAttributeVal(entry, "c3", ent->c3, pfrmt);
1346 256 : if (ent->c4 != 255)
1347 0 : XMLSetAttributeVal(entry, "c4", ent->c4, pfrmt);
1348 : }
1349 : }
1350 :
1351 332 : if (is_Endianness_Dependent(full.dt, full.comp)) // Need to set the order
1352 62 : CPLCreateXMLElementAndValue(raster, "NetByteOrder",
1353 62 : (full.nbo || NET_ORDER) ? "TRUE" : "FALSE");
1354 :
1355 332 : if (full.quality > 0 && full.quality != 85)
1356 12 : CPLCreateXMLElementAndValue(raster, "Quality",
1357 24 : CPLOPrintf("%d", full.quality));
1358 :
1359 : // Done with the raster node
1360 :
1361 332 : if (scale != 0.0)
1362 : {
1363 0 : CPLCreateXMLNode(config, CXT_Element, "Rsets");
1364 0 : CPLSetXMLValue(config, "Rsets.#model", "uniform");
1365 0 : CPLSetXMLValue(config, "Rsets.#scale", PrintDouble(scale));
1366 : }
1367 332 : CPLXMLNode *gtags = CPLCreateXMLNode(config, CXT_Element, "GeoTags");
1368 :
1369 : // Do we have an affine transform different from identity?
1370 : double gt[6];
1371 664 : if ((MRFDataset::GetGeoTransform(gt) == CE_None) &&
1372 332 : (gt[0] != 0 || gt[1] != 1 || gt[2] != 0 || gt[3] != 0 || gt[4] != 0 ||
1373 197 : gt[5] != 1))
1374 : {
1375 135 : double minx = gt[0];
1376 135 : double maxx = gt[1] * full.size.x + minx;
1377 135 : double maxy = gt[3];
1378 135 : double miny = gt[5] * full.size.y + maxy;
1379 135 : CPLXMLNode *bbox = CPLCreateXMLNode(gtags, CXT_Element, "BoundingBox");
1380 135 : XMLSetAttributeVal(bbox, "minx", minx);
1381 135 : XMLSetAttributeVal(bbox, "miny", miny);
1382 135 : XMLSetAttributeVal(bbox, "maxx", maxx);
1383 135 : XMLSetAttributeVal(bbox, "maxy", maxy);
1384 : }
1385 :
1386 332 : const char *pszProj = GetProjectionRef();
1387 332 : if (pszProj && (!EQUAL(pszProj, "")))
1388 136 : CPLCreateXMLElementAndValue(gtags, "Projection", pszProj);
1389 :
1390 332 : if (optlist.Count() != 0)
1391 : {
1392 48 : CPLString options;
1393 50 : for (int i = 0; i < optlist.size(); i++)
1394 : {
1395 26 : options += optlist[i];
1396 26 : options += ' ';
1397 : }
1398 24 : options.pop_back();
1399 24 : CPLCreateXMLElementAndValue(config, "Options", options);
1400 : }
1401 :
1402 332 : return config;
1403 : }
1404 :
1405 : /**
1406 : * \brief Populates the dataset variables from the XML definition
1407 : *
1408 : *
1409 : */
1410 363 : CPLErr MRFDataset::Initialize(CPLXMLNode *config)
1411 : {
1412 : // We only need a basic initialization here, usually gets overwritten by the
1413 : // image params
1414 363 : full.dt = GDT_Byte;
1415 363 : full.hasNoData = false;
1416 363 : full.NoDataValue = 0;
1417 363 : Quality = 85;
1418 :
1419 363 : CPLErr ret = Init_Raster(full, this, CPLGetXMLNode(config, "Raster"));
1420 363 : if (CE_None != ret)
1421 0 : return ret;
1422 :
1423 363 : hasVersions = on(CPLGetXMLValue(config, "Raster.versioned", "no"));
1424 363 : mp_safe = on(CPLGetXMLValue(config, "Raster.mp_safe", "no"));
1425 363 : spacing = atoi(CPLGetXMLValue(config, "Raster.Spacing", "0"));
1426 :
1427 : // The zslice defined in the file wins over the oo or the file argument
1428 363 : if (CPLGetXMLNode(config, "Raster.zslice"))
1429 0 : zslice = atoi(CPLGetXMLValue(config, "Raster.zslice", "0"));
1430 :
1431 363 : Quality = full.quality;
1432 :
1433 : // Bounding box
1434 363 : CPLXMLNode *bbox = CPLGetXMLNode(config, "GeoTags.BoundingBox");
1435 363 : if (nullptr != bbox)
1436 : {
1437 : double x0, x1, y0, y1;
1438 :
1439 159 : x0 = atof(CPLGetXMLValue(bbox, "minx", "0"));
1440 159 : x1 = atof(CPLGetXMLValue(bbox, "maxx", "1"));
1441 159 : y1 = atof(CPLGetXMLValue(bbox, "maxy", "1"));
1442 159 : y0 = atof(CPLGetXMLValue(bbox, "miny", "0"));
1443 :
1444 159 : GeoTransform[0] = x0;
1445 159 : GeoTransform[1] = (x1 - x0) / full.size.x;
1446 159 : GeoTransform[2] = 0;
1447 159 : GeoTransform[3] = y1;
1448 159 : GeoTransform[4] = 0;
1449 159 : GeoTransform[5] = (y0 - y1) / full.size.y;
1450 159 : bGeoTransformValid = TRUE;
1451 : }
1452 :
1453 : const char *pszRawProjFromXML =
1454 363 : CPLGetXMLValue(config, "GeoTags.Projection", "");
1455 363 : if (strlen(pszRawProjFromXML) != 0)
1456 160 : m_oSRS.SetFromUserInput(
1457 : pszRawProjFromXML,
1458 : OGRSpatialReference::SET_FROM_USER_INPUT_LIMITATIONS_get());
1459 :
1460 : // Copy the full size to current, data and index are not yet open
1461 363 : current = full;
1462 363 : if (current.size.z != 1)
1463 : {
1464 0 : SetMetadataItem("ZSIZE", CPLOPrintf("%d", current.size.z),
1465 0 : "IMAGE_STRUCTURE");
1466 0 : SetMetadataItem("ZSLICE", CPLOPrintf("%d", zslice), "IMAGE_STRUCTURE");
1467 : // Capture the zslice in pagesize.l
1468 0 : current.pagesize.l = zslice;
1469 : // Adjust offset for base image
1470 0 : if (full.size.z <= 0)
1471 : {
1472 0 : CPLError(CE_Failure, CPLE_AppDefined,
1473 : "GDAL MRF: Invalid Raster.z value");
1474 0 : return CE_Failure;
1475 : }
1476 0 : if (zslice >= full.size.z)
1477 : {
1478 0 : CPLError(CE_Failure, CPLE_AppDefined, "GDAL MRF: Invalid slice");
1479 0 : return CE_Failure;
1480 : }
1481 :
1482 0 : current.idxoffset +=
1483 0 : (current.pagecount.l / full.size.z) * zslice * sizeof(ILIdx);
1484 : }
1485 :
1486 : // Dataset metadata setup
1487 363 : SetMetadataItem("INTERLEAVE", OrderName(current.order), "IMAGE_STRUCTURE");
1488 363 : SetMetadataItem("COMPRESSION", CompName(current.comp), "IMAGE_STRUCTURE");
1489 :
1490 363 : if (is_Endianness_Dependent(current.dt, current.comp))
1491 69 : SetMetadataItem("NETBYTEORDER", current.nbo ? "TRUE" : "FALSE",
1492 69 : "IMAGE_STRUCTURE");
1493 :
1494 : // Open the files for the current image, either RW or RO
1495 363 : nRasterXSize = current.size.x;
1496 363 : nRasterYSize = current.size.y;
1497 363 : nBands = current.size.c;
1498 :
1499 363 : if (!nBands || !nRasterXSize || !nRasterYSize)
1500 : {
1501 0 : CPLError(CE_Failure, CPLE_AppDefined, "GDAL MRF: Image size missing");
1502 0 : return CE_Failure;
1503 : }
1504 :
1505 : // Pick up the source data image, if there is one
1506 363 : source = CPLGetXMLValue(config, "CachedSource.Source", "");
1507 : // Is it a clone?
1508 363 : clonedSource =
1509 363 : on(CPLGetXMLValue(config, "CachedSource.Source.clone", "no"));
1510 : // Pick up the options, if any
1511 : optlist.Assign(CSLTokenizeString2(
1512 : CPLGetXMLValue(config, "Options", nullptr), " \t\n\r",
1513 363 : CSLT_STRIPLEADSPACES | CSLT_STRIPENDSPACES));
1514 :
1515 : // Load all the options in the IMAGE_STRUCTURE metadata
1516 391 : for (int i = 0; i < optlist.Count(); i++)
1517 : {
1518 56 : CPLString s(optlist[i]);
1519 28 : size_t nSepPos = s.find_first_of(":=");
1520 28 : if (std::string::npos != nSepPos)
1521 : {
1522 28 : s.resize(nSepPos);
1523 28 : SetMetadataItem(s, optlist.FetchNameValue(s), "IMAGE_STRUCTURE");
1524 : }
1525 : }
1526 :
1527 : // We have the options, so we can call rasterband
1528 781 : for (int i = 1; i <= nBands; i++)
1529 : {
1530 : // The overviews are low resolution copies of the current one.
1531 471 : MRFRasterBand *band = newMRFRasterBand(this, current, i);
1532 471 : if (!band)
1533 53 : return CE_Failure;
1534 :
1535 418 : GDALColorInterp ci = GCI_Undefined;
1536 :
1537 : // Default color interpretation
1538 418 : switch (nBands)
1539 : {
1540 264 : case 1:
1541 : case 2:
1542 264 : ci = (i == 1) ? GCI_GrayIndex : GCI_AlphaBand;
1543 264 : break;
1544 144 : case 3:
1545 : case 4:
1546 144 : if (i < 3)
1547 94 : ci = (i == 1) ? GCI_RedBand : GCI_GreenBand;
1548 : else
1549 50 : ci = (i == 3) ? GCI_BlueBand : GCI_AlphaBand;
1550 : }
1551 :
1552 418 : if (GetColorTable())
1553 1 : ci = GCI_PaletteIndex;
1554 :
1555 : // Legacy, deprecated
1556 418 : if (optlist.FetchBoolean("MULTISPECTRAL", FALSE))
1557 0 : ci = GCI_Undefined;
1558 :
1559 : // New style
1560 418 : if (!photometric.empty())
1561 : {
1562 15 : if ("MULTISPECTRAL" == photometric)
1563 0 : ci = GCI_Undefined;
1564 : }
1565 :
1566 418 : band->SetColorInterpretation(ci);
1567 418 : SetBand(i, band);
1568 : }
1569 :
1570 310 : CPLXMLNode *rsets = CPLGetXMLNode(config, "Rsets");
1571 310 : if (nullptr != rsets && nullptr != rsets->psChild)
1572 : {
1573 : // We have rsets
1574 :
1575 : // Regular spaced overlays, until everything fits in a single tile
1576 32 : if (EQUAL("uniform", CPLGetXMLValue(rsets, "model", "uniform")))
1577 : {
1578 32 : scale = getXMLNum(rsets, "scale", 2.0);
1579 32 : if (scale <= 1)
1580 : {
1581 0 : CPLError(CE_Failure, CPLE_AppDefined,
1582 : "MRF: zoom factor less than unit not allowed");
1583 0 : return CE_Failure;
1584 : }
1585 : // Looks like there are overlays
1586 32 : AddOverviews(int(scale));
1587 : }
1588 : else
1589 : {
1590 0 : CPLError(CE_Failure, CPLE_AppDefined, "Unknown Rset definition");
1591 0 : return CE_Failure;
1592 : }
1593 : }
1594 :
1595 310 : idxSize = IdxSize(full, int(scale));
1596 310 : if (idxSize == 0)
1597 0 : return CE_Failure;
1598 :
1599 : // If not set by the bands, get a pageSizeBytes buffer
1600 310 : if (GetPBufferSize() == 0 && !SetPBuffer(current.pageSizeBytes))
1601 0 : return CE_Failure;
1602 :
1603 310 : if (hasVersions)
1604 : { // It has versions, but how many?
1605 5 : verCount = 0; // Assume it only has one
1606 : VSIStatBufL statb;
1607 : // If the file exists, compute the last version number
1608 5 : if (0 == VSIStatL(full.idxfname, &statb))
1609 5 : verCount = int(statb.st_size / idxSize - 1);
1610 : }
1611 :
1612 310 : return CE_None;
1613 : }
1614 :
1615 0 : static inline bool has_path(const CPLString &name)
1616 : {
1617 0 : return name.find_first_of("/\\") != string::npos;
1618 : }
1619 :
1620 : // Does name look like an absolute gdal file name?
1621 4 : static inline bool is_absolute(const CPLString &name)
1622 : {
1623 4 : return (name.find_first_of("/\\") == 0) // Starts with root
1624 3 : || (name.size() > 1 && name[1] == ':' &&
1625 0 : isalpha(static_cast<unsigned char>(
1626 0 : name[0]))) // Starts with drive letter
1627 7 : || (name[0] == '<'); // Maybe it is XML
1628 : }
1629 :
1630 : // Add the dirname of path to the beginning of name, if it is relative
1631 : // returns true if name was modified
1632 4 : static inline bool make_absolute(CPLString &name, const CPLString &path)
1633 : {
1634 4 : if (!is_absolute(path) && (path.find_first_of("/\\") != string::npos))
1635 : {
1636 3 : name = path.substr(0, path.find_last_of("/\\") + 1) + name;
1637 3 : return true;
1638 : }
1639 1 : return false;
1640 : }
1641 :
1642 : /**
1643 : *\brief Get the source dataset, open it if necessary
1644 : */
1645 5 : GDALDataset *MRFDataset::GetSrcDS()
1646 : {
1647 5 : if (poSrcDS)
1648 1 : return poSrcDS;
1649 4 : if (source.empty())
1650 0 : return nullptr;
1651 :
1652 : // Try open the source dataset as is
1653 4 : poSrcDS =
1654 4 : GDALDataset::FromHandle(GDALOpenShared(source.c_str(), GA_ReadOnly));
1655 :
1656 : // It the open fails, try again with the current dataset path prepended
1657 4 : if (!poSrcDS && make_absolute(source, fname))
1658 3 : poSrcDS = GDALDataset::FromHandle(
1659 : GDALOpenShared(source.c_str(), GA_ReadOnly));
1660 :
1661 4 : if (0 == source.find("<MRF_META>") && has_path(fname))
1662 : {
1663 : // MRF XML source, might need to patch the file names with the current
1664 : // one
1665 0 : MRFDataset *poMRFDS = dynamic_cast<MRFDataset *>(poSrcDS);
1666 0 : if (!poMRFDS)
1667 : {
1668 0 : delete poSrcDS;
1669 0 : poSrcDS = nullptr;
1670 0 : return nullptr;
1671 : }
1672 0 : make_absolute(poMRFDS->current.datfname, fname);
1673 0 : make_absolute(poMRFDS->current.idxfname, fname);
1674 : }
1675 4 : mp_safe = true; // Turn on MP safety
1676 4 : return poSrcDS;
1677 : }
1678 :
1679 : /**
1680 : *\brief Add or verify that all overlays exits
1681 : *
1682 : * @return size of the index file
1683 : */
1684 :
1685 62 : GIntBig MRFDataset::AddOverviews(int scaleIn)
1686 : {
1687 : // Fit the overlays
1688 62 : ILImage img = current;
1689 134 : while (1 != img.pagecount.x * img.pagecount.y)
1690 : {
1691 : // Adjust raster data for next level
1692 : // Adjust the offsets for indices left at this level
1693 72 : img.idxoffset += sizeof(ILIdx) * img.pagecount.l / img.size.z *
1694 72 : (img.size.z - zslice);
1695 :
1696 : // Next overview size
1697 72 : img.size.x = pcount(img.size.x, scaleIn);
1698 72 : img.size.y = pcount(img.size.y, scaleIn);
1699 72 : img.size.l++; // Increment the level
1700 72 : img.pagecount = pcount(img.size, img.pagesize);
1701 :
1702 : // And adjust the offset again, within next level
1703 72 : img.idxoffset += sizeof(ILIdx) * img.pagecount.l / img.size.z * zslice;
1704 72 : int l = static_cast<int>(img.size.l);
1705 : // Create and register the overviews for each band
1706 144 : for (int i = 1; i <= nBands; i++)
1707 : {
1708 : MRFRasterBand *b =
1709 72 : reinterpret_cast<MRFRasterBand *>(GetRasterBand(i));
1710 72 : if (!(b->GetOverview(l - 1)))
1711 72 : b->AddOverview(newMRFRasterBand(this, img, i, l));
1712 : }
1713 : }
1714 :
1715 : // Last adjustment, should be a single set of c and leftover z tiles
1716 62 : return img.idxoffset +
1717 124 : sizeof(ILIdx) * img.pagecount.l / img.size.z * (img.size.z - zslice);
1718 : }
1719 :
1720 : //
1721 : // set an entry if it doesn't already exist
1722 : //
1723 134 : static char **CSLAddIfMissing(char **papszList, const char *pszName,
1724 : const char *pszValue)
1725 : {
1726 134 : if (CSLFetchNameValue(papszList, pszName))
1727 9 : return papszList;
1728 125 : return CSLSetNameValue(papszList, pszName, pszValue);
1729 : }
1730 :
1731 : // CreateCopy implemented based on Create
1732 133 : GDALDataset *MRFDataset::CreateCopy(const char *pszFilename,
1733 : GDALDataset *poSrcDS, int /*bStrict*/,
1734 : char **papszOptions,
1735 : GDALProgressFunc pfnProgress,
1736 : void *pProgressData)
1737 : {
1738 266 : ILImage img;
1739 :
1740 133 : int x = poSrcDS->GetRasterXSize();
1741 133 : int y = poSrcDS->GetRasterYSize();
1742 133 : int nBands = poSrcDS->GetRasterCount();
1743 133 : if (nBands == 0)
1744 : {
1745 1 : CPLError(CE_Failure, CPLE_NotSupported, "nBands == 0 not supported");
1746 1 : return nullptr;
1747 : }
1748 132 : GDALRasterBand *poSrcBand1 = poSrcDS->GetRasterBand(1);
1749 :
1750 132 : GDALDataType dt = poSrcBand1->GetRasterDataType();
1751 : // Have our own options, to modify as we want
1752 132 : char **options = CSLDuplicate(papszOptions);
1753 :
1754 : const char *pszValue =
1755 132 : poSrcDS->GetMetadataItem("INTERLEAVE", "IMAGE_STRUCTURE");
1756 : options =
1757 132 : CSLAddIfMissing(options, "INTERLEAVE", pszValue ? pszValue : "PIXEL");
1758 : int xb, yb;
1759 132 : poSrcBand1->GetBlockSize(&xb, &yb);
1760 :
1761 : // Keep input block size if it exists and not explicitly set
1762 133 : if (CSLFetchNameValue(options, "BLOCKSIZE") == nullptr && xb != x &&
1763 1 : yb != y)
1764 : {
1765 1 : options = CSLAddIfMissing(options, "BLOCKXSIZE",
1766 2 : PrintDouble(xb, "%d").c_str());
1767 1 : options = CSLAddIfMissing(options, "BLOCKYSIZE",
1768 2 : PrintDouble(yb, "%d").c_str());
1769 : }
1770 :
1771 132 : MRFDataset *poDS = nullptr;
1772 : try
1773 : {
1774 : poDS = reinterpret_cast<MRFDataset *>(
1775 132 : Create(pszFilename, x, y, nBands, dt, options));
1776 :
1777 132 : if (poDS == nullptr || poDS->bCrystalized)
1778 12 : throw CPLOPrintf("MRF: Can't create %s", pszFilename);
1779 :
1780 120 : img = poDS->current; // Deal with the current one here
1781 :
1782 : // Copy data values from source
1783 274 : for (int i = 0; i < poDS->nBands; i++)
1784 : {
1785 : int bHas;
1786 : double dfData;
1787 154 : GDALRasterBand *srcBand = poSrcDS->GetRasterBand(i + 1);
1788 154 : GDALRasterBand *mBand = poDS->GetRasterBand(i + 1);
1789 154 : dfData = srcBand->GetNoDataValue(&bHas);
1790 154 : if (bHas)
1791 : {
1792 16 : poDS->vNoData.push_back(dfData);
1793 16 : mBand->SetNoDataValue(dfData);
1794 : }
1795 154 : dfData = srcBand->GetMinimum(&bHas);
1796 154 : if (bHas)
1797 17 : poDS->vMin.push_back(dfData);
1798 154 : dfData = srcBand->GetMaximum(&bHas);
1799 154 : if (bHas)
1800 17 : poDS->vMax.push_back(dfData);
1801 :
1802 : // Copy the band metadata, PAM will handle it
1803 154 : char **meta = srcBand->GetMetadata("IMAGE_STRUCTURE");
1804 154 : if (CSLCount(meta))
1805 7 : mBand->SetMetadata(meta, "IMAGE_STRUCTURE");
1806 :
1807 154 : meta = srcBand->GetMetadata();
1808 154 : if (CSLCount(meta))
1809 18 : mBand->SetMetadata(meta);
1810 : }
1811 :
1812 : // Geotags
1813 : double gt[6];
1814 120 : if (CE_None == poSrcDS->GetGeoTransform(gt))
1815 117 : poDS->SetGeoTransform(gt);
1816 :
1817 120 : const auto poSRS = poSrcDS->GetSpatialRef();
1818 120 : if (poSRS)
1819 116 : poDS->m_oSRS = *poSRS;
1820 :
1821 : // Color palette if we only have one band
1822 223 : if (1 == nBands &&
1823 103 : GCI_PaletteIndex == poSrcBand1->GetColorInterpretation())
1824 1 : poDS->SetColorTable(poSrcBand1->GetColorTable()->Clone());
1825 :
1826 : // Finally write the XML in the right file name
1827 120 : if (!poDS->Crystalize())
1828 10 : throw CPLString("MRF: Error creating files");
1829 : }
1830 22 : catch (const CPLString &e)
1831 : {
1832 22 : if (nullptr != poDS)
1833 10 : delete poDS;
1834 22 : CPLError(CE_Failure, CPLE_ObjectNull, "%s", e.c_str());
1835 22 : poDS = nullptr;
1836 : }
1837 :
1838 132 : CSLDestroy(options);
1839 132 : if (nullptr == poDS)
1840 22 : return nullptr;
1841 :
1842 110 : char **papszFileList = poDS->GetFileList();
1843 110 : poDS->oOvManager.Initialize(poDS, poDS->GetPhysicalFilename(),
1844 : papszFileList);
1845 110 : CSLDestroy(papszFileList);
1846 :
1847 110 : CPLErr err = CE_None;
1848 : // Have PAM copy all, but skip the mask
1849 110 : int nCloneFlags = GCIF_PAM_DEFAULT & ~GCIF_MASK;
1850 :
1851 : // If copy is disabled, we're done, we just created an empty MRF
1852 110 : if (!on(CSLFetchNameValue(papszOptions, "NOCOPY")))
1853 : {
1854 : // Use the GDAL copy call
1855 : // Need to flag the dataset as compressed (COMPRESSED=TRUE) to force
1856 : // block writes This might not be what we want, if the input and out
1857 : // order is truly separate
1858 108 : nCloneFlags |= GCIF_MASK; // We do copy the data, so copy the mask too
1859 : // if necessary
1860 108 : char **papszCWROptions = nullptr;
1861 : papszCWROptions =
1862 108 : CSLAddNameValue(papszCWROptions, "COMPRESSED", "TRUE");
1863 :
1864 : #ifdef HAVE_JPEG
1865 : // Use the Zen version of the CopyWholeRaster if input has a dataset
1866 : // mask and JPEGs are generated
1867 110 : if (GMF_PER_DATASET == poSrcDS->GetRasterBand(1)->GetMaskFlags() &&
1868 2 : (poDS->current.comp == IL_JPEG
1869 : #ifdef HAVE_PNG
1870 0 : || poDS->current.comp == IL_JPNG
1871 : #endif
1872 : ))
1873 : {
1874 2 : err = poDS->ZenCopy(poSrcDS, pfnProgress, pProgressData);
1875 2 : nCloneFlags ^= GCIF_MASK; // Turn the external mask off
1876 : }
1877 : else
1878 : #endif
1879 : {
1880 106 : err = GDALDatasetCopyWholeRaster(
1881 : (GDALDatasetH)poSrcDS, (GDALDatasetH)poDS, papszCWROptions,
1882 : pfnProgress, pProgressData);
1883 : }
1884 :
1885 108 : CSLDestroy(papszCWROptions);
1886 : }
1887 :
1888 110 : if (CE_None == err)
1889 110 : err = poDS->CloneInfo(poSrcDS, nCloneFlags);
1890 :
1891 110 : if (CE_Failure == err)
1892 : {
1893 0 : delete poDS;
1894 0 : return nullptr;
1895 : }
1896 :
1897 110 : return poDS;
1898 : }
1899 :
1900 : // Prepares the data so it is suitable for Zen JPEG encoding, based on input
1901 : // mask If bFBO is set, only the values of the first band are set non-zero when
1902 : // needed
1903 : template <typename T>
1904 2 : static void ZenFilter(T *buffer, GByte *mask, int nPixels, int nBands,
1905 : bool bFBO)
1906 : {
1907 524290 : for (int i = 0; i < nPixels; i++)
1908 : {
1909 524288 : if (mask[i] == 0)
1910 : { // enforce zero values
1911 516096 : for (int b = 0; b < nBands; b++)
1912 387072 : buffer[nBands * i + b] = 0;
1913 : }
1914 : else
1915 : { // enforce non-zero
1916 395264 : if (bFBO)
1917 : { // First band only
1918 197632 : bool f = true;
1919 790528 : for (int b = 0; b < nBands; b++)
1920 : {
1921 592896 : if (0 == buffer[nBands * i + b])
1922 : {
1923 0 : f = false;
1924 0 : break;
1925 : }
1926 : }
1927 197632 : if (f)
1928 197632 : buffer[nBands * i] = 1;
1929 : }
1930 : else
1931 : { // Every band
1932 790528 : for (int b = 0; b < nBands; b++)
1933 592896 : if (0 == buffer[nBands * i + b])
1934 0 : buffer[nBands * i + b] = 1;
1935 : }
1936 : }
1937 : }
1938 2 : }
1939 :
1940 : // Custom CopyWholeRaster for Zen JPEG, called when the input has a PER_DATASET
1941 : // mask Works like GDALDatasetCopyWholeRaster, but it does filter the input data
1942 : // based on the mask
1943 : //
1944 2 : CPLErr MRFDataset::ZenCopy(GDALDataset *poSrc, GDALProgressFunc pfnProgress,
1945 : void *pProgressData)
1946 : {
1947 2 : VALIDATE_POINTER1(poSrc, "MRF:ZenCopy", CE_Failure);
1948 :
1949 2 : if (!pfnProgress)
1950 0 : pfnProgress = GDALDummyProgress;
1951 :
1952 : /* -------------------------------------------------------------------- */
1953 : /* Confirm the datasets match in size and band counts. */
1954 : /* -------------------------------------------------------------------- */
1955 2 : const int nXSize = GetRasterXSize();
1956 2 : const int nYSize = GetRasterYSize();
1957 2 : const int nBandCount = GetRasterCount();
1958 :
1959 2 : if (poSrc->GetRasterXSize() != nXSize ||
1960 4 : poSrc->GetRasterYSize() != nYSize ||
1961 2 : poSrc->GetRasterCount() != nBandCount)
1962 : {
1963 0 : CPLError(CE_Failure, CPLE_AppDefined,
1964 : "Input and output dataset sizes or band counts do not\n"
1965 : "match in GDALDatasetCopyWholeRaster()");
1966 0 : return CE_Failure;
1967 : }
1968 :
1969 : /* -------------------------------------------------------------------- */
1970 : /* Get our prototype band, and assume the others are similarly */
1971 : /* configured. Also get the per_dataset mask */
1972 : /* -------------------------------------------------------------------- */
1973 2 : GDALRasterBand *poSrcPrototypeBand = poSrc->GetRasterBand(1);
1974 2 : GDALRasterBand *poDstPrototypeBand = GetRasterBand(1);
1975 2 : GDALRasterBand *poSrcMask = poSrcPrototypeBand->GetMaskBand();
1976 :
1977 2 : const int nPageXSize = current.pagesize.x;
1978 2 : const int nPageYSize = current.pagesize.y;
1979 2 : const double nTotalBlocks =
1980 2 : static_cast<double>(DIV_ROUND_UP(nYSize, nPageYSize)) *
1981 2 : static_cast<double>(DIV_ROUND_UP(nXSize, nPageXSize));
1982 2 : const GDALDataType eDT = poDstPrototypeBand->GetRasterDataType();
1983 :
1984 : // All the bands are done per block
1985 : // this flag tells us to apply the Zen filter to the first band only
1986 2 : const bool bFirstBandOnly = (current.order == IL_Interleaved);
1987 :
1988 2 : if (!pfnProgress(0.0, nullptr, pProgressData))
1989 : {
1990 0 : CPLError(CE_Failure, CPLE_UserInterrupt,
1991 : "User terminated CreateCopy()");
1992 0 : return CE_Failure;
1993 : }
1994 :
1995 2 : const int nPixelCount = nPageXSize * nPageYSize;
1996 2 : const int dts = GDALGetDataTypeSizeBytes(eDT);
1997 2 : void *buffer = VSI_MALLOC3_VERBOSE(nPixelCount, nBandCount, dts);
1998 2 : GByte *buffer_mask = nullptr;
1999 2 : if (buffer)
2000 : buffer_mask =
2001 2 : reinterpret_cast<GByte *>(VSI_MALLOC_VERBOSE(nPixelCount));
2002 :
2003 2 : if (!buffer || !buffer_mask)
2004 : {
2005 : // Just in case buffers did get allocated
2006 0 : CPLFree(buffer);
2007 0 : CPLFree(buffer_mask);
2008 0 : CPLError(CE_Failure, CPLE_OutOfMemory, "Can't allocate copy buffer");
2009 0 : return CE_Failure;
2010 : }
2011 :
2012 2 : int nBlocksDone = 0;
2013 2 : CPLErr eErr = CE_None;
2014 : // Advise the source that a complete read will be done
2015 2 : poSrc->AdviseRead(0, 0, nXSize, nYSize, nXSize, nYSize, eDT, nBandCount,
2016 2 : nullptr, nullptr);
2017 :
2018 : // For every block, break on error
2019 4 : for (int row = 0; row < nYSize && eErr == CE_None; row += nPageYSize)
2020 : {
2021 2 : int nRows = std::min(nPageYSize, nYSize - row);
2022 4 : for (int col = 0; col < nXSize && eErr == CE_None; col += nPageXSize)
2023 : {
2024 2 : int nCols = std::min(nPageXSize, nXSize - col);
2025 :
2026 : // Report
2027 2 : if (eErr == CE_None && !pfnProgress(nBlocksDone++ / nTotalBlocks,
2028 : nullptr, pProgressData))
2029 : {
2030 0 : eErr = CE_Failure;
2031 0 : CPLError(CE_Failure, CPLE_UserInterrupt,
2032 : "User terminated CreateCopy()");
2033 0 : break;
2034 : }
2035 :
2036 : // Get the data mask as byte
2037 2 : eErr = poSrcMask->RasterIO(GF_Read, col, row, nCols, nRows,
2038 : buffer_mask, nCols, nRows, GDT_Byte, 0,
2039 : 0, nullptr);
2040 :
2041 2 : if (eErr != CE_None)
2042 0 : break;
2043 :
2044 : // If there is no data at all, skip this block
2045 2 : if (MatchCount(buffer_mask, nPixelCount, static_cast<GByte>(0)) ==
2046 : nPixelCount)
2047 0 : continue;
2048 :
2049 : // get the data in the buffer, interleaved
2050 4 : eErr = poSrc->RasterIO(
2051 : GF_Read, col, row, nCols, nRows, buffer, nCols, nRows, eDT,
2052 2 : nBandCount, nullptr, static_cast<GSpacing>(nBands) * dts,
2053 2 : static_cast<GSpacing>(nBands) * dts * nCols, dts, nullptr);
2054 :
2055 2 : if (eErr != CE_None)
2056 0 : break;
2057 :
2058 : // This is JPEG, only 8 and 12(16) bits unsigned integer types are
2059 : // valid
2060 2 : switch (eDT)
2061 : {
2062 2 : case GDT_Byte:
2063 2 : ZenFilter(reinterpret_cast<GByte *>(buffer), buffer_mask,
2064 : nPixelCount, nBandCount, bFirstBandOnly);
2065 2 : break;
2066 0 : case GDT_UInt16:
2067 0 : ZenFilter(reinterpret_cast<GUInt16 *>(buffer), buffer_mask,
2068 : nPixelCount, nBandCount, bFirstBandOnly);
2069 0 : break;
2070 0 : default:
2071 0 : CPLError(CE_Failure, CPLE_AppDefined,
2072 : "Unsupported data type for Zen filter");
2073 0 : eErr = CE_Failure;
2074 0 : break;
2075 : }
2076 :
2077 : // Write
2078 2 : if (eErr == CE_None)
2079 2 : eErr = RasterIO(
2080 : GF_Write, col, row, nCols, nRows, buffer, nCols, nRows, eDT,
2081 2 : nBandCount, nullptr, static_cast<GSpacing>(nBands) * dts,
2082 2 : static_cast<GSpacing>(nBands) * dts * nCols, dts, nullptr);
2083 :
2084 : } // Columns
2085 2 : if (eErr != CE_None)
2086 0 : break;
2087 :
2088 : } // Rows
2089 :
2090 : // Cleanup
2091 2 : CPLFree(buffer);
2092 2 : CPLFree(buffer_mask);
2093 :
2094 : // Final report
2095 2 : if (eErr == CE_None && !pfnProgress(1.0, nullptr, pProgressData))
2096 : {
2097 0 : eErr = CE_Failure;
2098 0 : CPLError(CE_Failure, CPLE_UserInterrupt,
2099 : "User terminated CreateCopy()");
2100 : }
2101 :
2102 2 : return eErr;
2103 : }
2104 :
2105 : // Apply open options to the current dataset
2106 : // Called before the configuration is read
2107 202 : void MRFDataset::ProcessOpenOptions(char **papszOptions)
2108 : {
2109 404 : CPLStringList opt(papszOptions, FALSE);
2110 202 : no_errors = opt.FetchBoolean("NOERRORS", FALSE);
2111 202 : const char *val = opt.FetchNameValue("ZSLICE");
2112 202 : if (val)
2113 0 : zslice = atoi(val);
2114 202 : }
2115 :
2116 : // Apply create options to the current dataset, only valid during creation
2117 161 : void MRFDataset::ProcessCreateOptions(char **papszOptions)
2118 : {
2119 161 : assert(!bCrystalized);
2120 322 : CPLStringList opt(papszOptions, FALSE);
2121 161 : ILImage &img(full);
2122 :
2123 161 : const char *val = opt.FetchNameValue("COMPRESS");
2124 161 : if (val && IL_ERR_COMP == (img.comp = CompToken(val)))
2125 0 : throw CPLString("GDAL MRF: Error setting compression");
2126 :
2127 161 : val = opt.FetchNameValue("INTERLEAVE");
2128 161 : if (val && IL_ERR_ORD == (img.order = OrderToken(val)))
2129 0 : throw CPLString("GDAL MRF: Error setting interleave");
2130 :
2131 161 : val = opt.FetchNameValue("QUALITY");
2132 161 : if (val)
2133 6 : img.quality = atoi(val);
2134 :
2135 161 : val = opt.FetchNameValue("ZSIZE");
2136 161 : if (val)
2137 0 : img.size.z = atoi(val);
2138 :
2139 161 : val = opt.FetchNameValue("BLOCKXSIZE");
2140 161 : if (val)
2141 1 : img.pagesize.x = atoi(val);
2142 :
2143 161 : val = opt.FetchNameValue("BLOCKYSIZE");
2144 161 : if (val)
2145 1 : img.pagesize.y = atoi(val);
2146 :
2147 161 : val = opt.FetchNameValue("BLOCKSIZE");
2148 161 : if (val)
2149 30 : img.pagesize.x = img.pagesize.y = atoi(val);
2150 :
2151 161 : img.nbo = opt.FetchBoolean("NETBYTEORDER", FALSE) != FALSE;
2152 :
2153 161 : val = opt.FetchNameValue("CACHEDSOURCE");
2154 161 : if (val)
2155 : {
2156 2 : source = val;
2157 2 : nocopy = opt.FetchBoolean("NOCOPY", FALSE);
2158 : }
2159 :
2160 161 : val = opt.FetchNameValue("UNIFORM_SCALE");
2161 161 : if (val)
2162 0 : scale = atoi(val);
2163 :
2164 161 : val = opt.FetchNameValue("PHOTOMETRIC");
2165 161 : if (val)
2166 2 : photometric = val;
2167 :
2168 161 : val = opt.FetchNameValue("DATANAME");
2169 161 : if (val)
2170 0 : img.datfname = val;
2171 :
2172 161 : val = opt.FetchNameValue("INDEXNAME");
2173 161 : if (val)
2174 0 : img.idxfname = val;
2175 :
2176 161 : val = opt.FetchNameValue("SPACING");
2177 161 : if (val)
2178 0 : spacing = atoi(val);
2179 :
2180 : optlist.Assign(
2181 : CSLTokenizeString2(opt.FetchNameValue("OPTIONS"), " \t\n\r",
2182 161 : CSLT_STRIPLEADSPACES | CSLT_STRIPENDSPACES));
2183 :
2184 : // General Fixups
2185 161 : if (img.order == IL_Interleaved)
2186 65 : img.pagesize.c = img.size.c;
2187 :
2188 : // Compression dependent fixups
2189 161 : }
2190 :
2191 : /**
2192 : *\brief Create an MRF dataset, some settings can be changed later
2193 : * papszOptions might be anything that an MRF might take
2194 : * Still missing are the georeference ...
2195 : *
2196 : */
2197 :
2198 165 : GDALDataset *MRFDataset::Create(const char *pszName, int nXSize, int nYSize,
2199 : int nBandsIn, GDALDataType eType,
2200 : char **papszOptions)
2201 : {
2202 165 : if (nBandsIn == 0)
2203 : {
2204 1 : CPLError(CE_Failure, CPLE_NotSupported, "No bands defined");
2205 1 : return nullptr;
2206 : }
2207 :
2208 164 : MRFDataset *poDS = new MRFDataset();
2209 164 : CPLErr err = CE_None;
2210 164 : poDS->fname = pszName;
2211 164 : poDS->nBands = nBandsIn;
2212 :
2213 : // Don't know what to do with these in this call
2214 : // int level = -1;
2215 : // int version = 0;
2216 :
2217 164 : size_t pos = poDS->fname.find(":MRF:");
2218 164 : if (string::npos != pos)
2219 : { // Tokenize and pick known options
2220 0 : vector<string> tokens;
2221 0 : stringSplit(tokens, poDS->fname, pos + 5, ':');
2222 : // level = getnum(tokens, 'L', -1);
2223 : // version = getnum(tokens, 'V', 0);
2224 0 : poDS->zslice = getnum(tokens, 'Z', 0);
2225 0 : poDS->fname.resize(pos); // Cut the ornamentations
2226 : }
2227 :
2228 : // Try creating the mrf file early, to avoid failing on Crystalize later
2229 164 : if (!STARTS_WITH(poDS->fname.c_str(), "<MRF_META>"))
2230 : {
2231 : // Try opening it first, even though we still clobber it later
2232 164 : VSILFILE *mainfile = VSIFOpenL(poDS->fname.c_str(), "r+b");
2233 164 : if (!mainfile)
2234 : { // Then try creating it
2235 115 : mainfile = VSIFOpenL(poDS->fname.c_str(), "w+b");
2236 115 : if (!mainfile)
2237 : {
2238 3 : CPLError(CE_Failure, CPLE_OpenFailed,
2239 : "MRF: Can't open %s for writing", poDS->fname.c_str());
2240 3 : delete poDS;
2241 3 : return nullptr;
2242 : }
2243 : }
2244 161 : VSIFCloseL(mainfile);
2245 : }
2246 :
2247 : // Use the full, set some initial parameters
2248 161 : ILImage &img = poDS->full;
2249 161 : img.size = ILSize(nXSize, nYSize, 1, nBandsIn);
2250 : #ifdef HAVE_PNG
2251 161 : img.comp = IL_PNG;
2252 : #else
2253 : img.comp = IL_NONE;
2254 : #endif
2255 161 : img.order = (nBandsIn < 5) ? IL_Interleaved : IL_Separate;
2256 161 : img.pagesize = ILSize(512, 512, 1, 1);
2257 161 : img.quality = 85;
2258 161 : img.dt = eType;
2259 161 : img.dataoffset = 0;
2260 161 : img.idxoffset = 0;
2261 161 : img.hasNoData = false;
2262 161 : img.nbo = false;
2263 :
2264 : // Set the guard that tells us it needs saving before IO can take place
2265 161 : poDS->bCrystalized = FALSE;
2266 :
2267 : // Process the options, anything that an MRF might take
2268 :
2269 : try
2270 : {
2271 : // Adjust the dataset and the full image
2272 161 : poDS->ProcessCreateOptions(papszOptions);
2273 :
2274 : // Set default file names
2275 161 : if (img.datfname.empty())
2276 161 : img.datfname = getFname(poDS->GetFname(), ILComp_Ext[img.comp]);
2277 161 : if (img.idxfname.empty())
2278 161 : img.idxfname = getFname(poDS->GetFname(), ".idx");
2279 :
2280 161 : poDS->eAccess = GA_Update;
2281 : }
2282 :
2283 0 : catch (const CPLString &e)
2284 : {
2285 0 : CPLError(CE_Failure, CPLE_OpenFailed, "%s", e.c_str());
2286 0 : delete poDS;
2287 0 : return nullptr;
2288 : }
2289 :
2290 161 : poDS->current = poDS->full;
2291 161 : poDS->SetDescription(poDS->GetFname());
2292 :
2293 : // Build a MRF XML and initialize from it, this creates the bands
2294 161 : CPLXMLNode *config = poDS->BuildConfig();
2295 161 : err = poDS->Initialize(config);
2296 161 : CPLDestroyXMLNode(config);
2297 :
2298 161 : if (CPLE_None != err)
2299 : {
2300 31 : delete poDS;
2301 31 : return nullptr;
2302 : }
2303 :
2304 : // If not set by the band, get a pageSizeBytes buffer
2305 130 : if (poDS->GetPBufferSize() == 0 &&
2306 0 : !poDS->SetPBuffer(poDS->current.pageSizeBytes))
2307 : {
2308 0 : delete poDS;
2309 0 : return nullptr;
2310 : }
2311 :
2312 : // Tell PAM what our real file name is, to help it find the aux.xml
2313 130 : poDS->SetPhysicalFilename(poDS->GetFname());
2314 130 : return poDS;
2315 : }
2316 :
2317 171 : int MRFDataset::Crystalize()
2318 : {
2319 171 : if (bCrystalized || eAccess != GA_Update)
2320 : {
2321 0 : bCrystalized = TRUE;
2322 0 : return TRUE;
2323 : }
2324 :
2325 : // No need to write to disk if there is no filename. This is a
2326 : // memory only dataset.
2327 342 : if (strlen(GetDescription()) == 0 ||
2328 171 : EQUALN(GetDescription(), "<MRF_META>", 10))
2329 : {
2330 0 : bCrystalized = TRUE;
2331 0 : return TRUE;
2332 : }
2333 :
2334 171 : CPLXMLNode *config = BuildConfig();
2335 171 : if (!WriteConfig(config))
2336 20 : return FALSE;
2337 151 : CPLDestroyXMLNode(config);
2338 151 : if (!nocopy && (!IdxFP() || !DataFP()))
2339 0 : return FALSE;
2340 151 : bCrystalized = TRUE;
2341 151 : return TRUE;
2342 : }
2343 :
2344 : // Copy the first index at the end of the file and bump the version count
2345 1 : CPLErr MRFDataset::AddVersion()
2346 : {
2347 1 : VSILFILE *l_ifp = IdxFP();
2348 1 : void *tbuff = CPLMalloc(static_cast<size_t>(idxSize));
2349 1 : VSIFSeekL(l_ifp, 0, SEEK_SET);
2350 1 : VSIFReadL(tbuff, 1, static_cast<size_t>(idxSize), l_ifp);
2351 1 : verCount++; // The one we write
2352 1 : VSIFSeekL(l_ifp, idxSize * verCount,
2353 : SEEK_SET); // At the end, this can mess things up royally
2354 1 : VSIFWriteL(tbuff, 1, static_cast<size_t>(idxSize), l_ifp);
2355 1 : CPLFree(tbuff);
2356 1 : return CE_None;
2357 : }
2358 :
2359 : //
2360 : // Write a tile at the end of the data file
2361 : // If buff and size are zero, it is equivalent to erasing the tile
2362 : // If only size is zero, it is a special empty tile,
2363 : // when used for caching, offset should be 1
2364 : //
2365 : // To make it multi-processor safe, open the file in append mode
2366 : // and verify after write
2367 : //
2368 5084 : CPLErr MRFDataset::WriteTile(void *buff, GUIntBig infooffset, GUIntBig size)
2369 : {
2370 5084 : CPLErr ret = CE_None;
2371 5084 : ILIdx tinfo = {0, 0};
2372 :
2373 5084 : VSILFILE *l_dfp = DataFP();
2374 5084 : VSILFILE *l_ifp = IdxFP();
2375 :
2376 : // Verify buffer
2377 10168 : std::vector<GByte> tbuff;
2378 :
2379 5084 : if (l_ifp == nullptr || l_dfp == nullptr)
2380 0 : return CE_Failure;
2381 :
2382 : // Flag that versioned access requires a write even if empty
2383 5084 : int new_tile = false;
2384 : // If it has versions, might need to start a new one
2385 5084 : if (hasVersions)
2386 : {
2387 1 : int new_version = false; // Assume no need to build new version
2388 :
2389 : // Read the current tile info
2390 1 : VSIFSeekL(l_ifp, infooffset, SEEK_SET);
2391 1 : VSIFReadL(&tinfo, 1, sizeof(ILIdx), l_ifp);
2392 :
2393 1 : if (verCount == 0)
2394 1 : new_version = true; // No previous yet, might create a new version
2395 : else
2396 : { // We need at least two versions before we can test for changes
2397 0 : ILIdx prevtinfo = {0, 0};
2398 :
2399 : // Read the previous one
2400 0 : VSIFSeekL(l_ifp, infooffset + verCount * idxSize, SEEK_SET);
2401 0 : VSIFReadL(&prevtinfo, 1, sizeof(ILIdx), l_ifp);
2402 :
2403 : // current and previous tiles are different, might create version
2404 0 : if (tinfo.size != prevtinfo.size ||
2405 0 : tinfo.offset != prevtinfo.offset)
2406 0 : new_version = true;
2407 : }
2408 :
2409 : // tinfo contains the current info or 0,0
2410 1 : if (tinfo.size == GIntBig(net64(size)))
2411 : { // Might be identical
2412 0 : if (size != 0)
2413 : {
2414 : // Use the temporary buffer
2415 0 : tbuff.resize(static_cast<size_t>(size));
2416 0 : VSIFSeekL(l_dfp, infooffset, SEEK_SET);
2417 0 : VSIFReadL(tbuff.data(), 1, tbuff.size(), l_dfp);
2418 : // Need to write it if not the same
2419 0 : new_tile = !std::equal(tbuff.begin(), tbuff.end(),
2420 : static_cast<GByte *>(buff));
2421 0 : tbuff.clear();
2422 : }
2423 : else
2424 : {
2425 : // Writing a null tile on top of a null tile, does it count?
2426 0 : if (tinfo.offset != GIntBig(net64(GUIntBig(buff))))
2427 0 : new_tile = true;
2428 : }
2429 : }
2430 : else
2431 : {
2432 1 : new_tile = true; // Need to write it because it is different
2433 1 : if (verCount == 0 && tinfo.size == 0)
2434 0 : new_version = false; // Don't create a version if current is
2435 : // empty and there is no previous
2436 : }
2437 :
2438 1 : if (!new_tile)
2439 0 : return CE_None; // No reason to write
2440 :
2441 : // Do we need to start a new version before writing the tile?
2442 1 : if (new_version)
2443 1 : AddVersion();
2444 : }
2445 :
2446 5084 : bool same = true;
2447 5084 : if (size)
2448 0 : do
2449 : {
2450 : // start of critical MP section
2451 5083 : VSIFSeekL(l_dfp, 0, SEEK_END);
2452 5083 : GUIntBig offset = VSIFTellL(l_dfp) + spacing;
2453 :
2454 : // Spacing should be 0 in MP safe mode, this doesn't have much of
2455 : // effect Use the existing data, spacing content is not guaranteed
2456 5083 : for (GUIntBig pending = spacing; pending != 0;
2457 0 : pending -= std::min(pending, size))
2458 0 : VSIFWriteL(buff, 1,
2459 0 : static_cast<size_t>(std::min(pending, size)),
2460 : l_dfp); // Usually only once
2461 :
2462 5083 : if (static_cast<size_t>(size) !=
2463 5083 : VSIFWriteL(buff, 1, static_cast<size_t>(size), l_dfp))
2464 0 : ret = CE_Failure;
2465 : // End of critical section
2466 :
2467 5083 : tinfo.offset = net64(offset);
2468 : //
2469 : // For MP ops, check that we can read the same content, otherwise
2470 : // try again This makes the caching MRF MP safe on file systems that
2471 : // implement append mode fully, without using explicit locks
2472 : //
2473 5083 : if (CE_None == ret && mp_safe)
2474 : { // readback and check
2475 3 : if (tbuff.size() < size)
2476 3 : tbuff.resize(static_cast<size_t>(size));
2477 3 : VSIFSeekL(l_dfp, offset, SEEK_SET);
2478 3 : VSIFReadL(tbuff.data(), 1, tbuff.size(), l_dfp);
2479 3 : same = std::equal(tbuff.begin(), tbuff.end(),
2480 : static_cast<GByte *>(buff));
2481 : }
2482 5083 : } while (CE_None == ret && mp_safe && !same);
2483 :
2484 5084 : if (CE_None != ret)
2485 : {
2486 0 : CPLError(CE_Failure, CPLE_AppDefined, "MRF: Tile write failed");
2487 0 : return ret;
2488 : }
2489 :
2490 : // Convert index to net format, offset is set already
2491 5084 : tinfo.size = net64(size);
2492 : // Do nothing if the tile is empty and the file record is also empty
2493 5084 : if (!new_tile && 0 == size && nullptr == buff)
2494 : {
2495 0 : VSIFSeekL(l_ifp, infooffset, SEEK_SET);
2496 0 : VSIFReadL(&tinfo, 1, sizeof(ILIdx), l_ifp);
2497 0 : if (0 == tinfo.offset && 0 == tinfo.size)
2498 0 : return ret;
2499 : }
2500 :
2501 : // Special case, any non-zero offset will do
2502 5084 : if (nullptr != buff && 0 == size)
2503 0 : tinfo.offset = ~GUIntBig(0);
2504 :
2505 5084 : VSIFSeekL(l_ifp, infooffset, SEEK_SET);
2506 5084 : if (sizeof(tinfo) != VSIFWriteL(&tinfo, 1, sizeof(tinfo), l_ifp))
2507 : {
2508 0 : CPLError(CE_Failure, CPLE_AppDefined, "MRF: Index write failed");
2509 0 : ret = CE_Failure;
2510 : }
2511 :
2512 5084 : return ret;
2513 : }
2514 :
2515 126 : CPLErr MRFDataset::SetGeoTransform(double *gt)
2516 : {
2517 126 : if (GetAccess() != GA_Update || bCrystalized)
2518 : {
2519 0 : CPLError(CE_Failure, CPLE_NotSupported,
2520 : "SetGeoTransform only works during Create call");
2521 0 : return CE_Failure;
2522 : }
2523 126 : memcpy(GeoTransform, gt, 6 * sizeof(double));
2524 126 : bGeoTransformValid = TRUE;
2525 126 : return CE_None;
2526 : }
2527 :
2528 16 : bool MRFDataset::IsSingleTile()
2529 : {
2530 16 : if (current.pagecount.l != 1 || !source.empty() || nullptr == DataFP())
2531 4 : return FALSE;
2532 12 : return 0 == reinterpret_cast<MRFRasterBand *>(GetRasterBand(1))
2533 12 : ->GetOverviewCount();
2534 : }
2535 :
2536 : /*
2537 : * Returns 0,1,0,0,0,1 even if it was not set
2538 : */
2539 451 : CPLErr MRFDataset::GetGeoTransform(double *gt)
2540 : {
2541 451 : memcpy(gt, GeoTransform, 6 * sizeof(double));
2542 451 : if (GetMetadata("RPC") || GetGCPCount())
2543 0 : bGeoTransformValid = FALSE;
2544 451 : if (!bGeoTransformValid)
2545 0 : return CE_Failure;
2546 451 : return CE_None;
2547 : }
2548 :
2549 : /**
2550 : *\brief Read a tile index
2551 : *
2552 : * It handles the non-existent index case, for no compression
2553 : * The bias is non-zero only when the cloned index is read
2554 : */
2555 :
2556 2019 : CPLErr MRFDataset::ReadTileIdx(ILIdx &tinfo, const ILSize &pos,
2557 : const ILImage &img, const GIntBig bias)
2558 : {
2559 2019 : VSILFILE *l_ifp = IdxFP();
2560 :
2561 : // Initialize the tinfo structure, in case the files are missing
2562 2019 : if (missing)
2563 0 : return CE_None;
2564 :
2565 2019 : GIntBig offset = bias + IdxOffset(pos, img);
2566 2019 : if (l_ifp == nullptr && img.comp == IL_NONE)
2567 : {
2568 0 : tinfo.size = current.pageSizeBytes;
2569 0 : tinfo.offset = offset * tinfo.size;
2570 0 : return CE_None;
2571 : }
2572 :
2573 2019 : if (l_ifp == nullptr && IsSingleTile())
2574 : {
2575 6 : tinfo.offset = 0;
2576 6 : VSILFILE *l_dfp = DataFP(); // IsSingleTile() checks that fp is valid
2577 6 : VSIFSeekL(l_dfp, 0, SEEK_END);
2578 6 : tinfo.size = VSIFTellL(l_dfp);
2579 :
2580 : // It should be less than the pagebuffer
2581 6 : tinfo.size = std::min(tinfo.size, static_cast<GIntBig>(pbsize));
2582 6 : return CE_None;
2583 : }
2584 :
2585 2013 : if (l_ifp == nullptr)
2586 : {
2587 0 : CPLError(CE_Failure, CPLE_FileIO, "Can't open index file");
2588 0 : return CE_Failure;
2589 : }
2590 :
2591 2013 : VSIFSeekL(l_ifp, offset, SEEK_SET);
2592 2013 : if (1 != VSIFReadL(&tinfo, sizeof(ILIdx), 1, l_ifp))
2593 0 : return CE_Failure;
2594 : // Convert them to native form
2595 2013 : tinfo.offset = net64(tinfo.offset);
2596 2013 : tinfo.size = net64(tinfo.size);
2597 :
2598 2013 : if (0 == bias || 0 != tinfo.size || 0 != tinfo.offset)
2599 2012 : return CE_None;
2600 :
2601 : // zero size and zero offset in sourced index means that this portion is
2602 : // un-initialized
2603 :
2604 : // Should be cloned and the offset within the cloned index
2605 1 : offset -= bias;
2606 1 : assert(offset < bias);
2607 1 : assert(clonedSource);
2608 :
2609 : // Read this block from the remote index, prepare it and store it in the
2610 : // right place The block size in bytes, should be a multiple of 16, to have
2611 : // full index entries
2612 1 : const int CPYSZ = 32768;
2613 : // Adjust offset to the start of the block
2614 1 : offset = (offset / CPYSZ) * CPYSZ;
2615 1 : GIntBig size = std::min(size_t(CPYSZ), size_t(bias - offset));
2616 1 : size /= sizeof(ILIdx); // In records
2617 2 : vector<ILIdx> buf(static_cast<size_t>(size));
2618 1 : ILIdx *buffer = &buf[0]; // Buffer to copy the source to the clone index
2619 :
2620 : // Fetch the data from the cloned index
2621 1 : MRFDataset *pSrc = static_cast<MRFDataset *>(GetSrcDS());
2622 1 : if (nullptr == pSrc)
2623 : {
2624 0 : CPLError(CE_Failure, CPLE_FileIO, "Can't open cloned source index");
2625 0 : return CE_Failure; // Source reported the error
2626 : }
2627 :
2628 1 : VSILFILE *srcidx = pSrc->IdxFP();
2629 1 : if (nullptr == srcidx)
2630 : {
2631 0 : CPLError(CE_Failure, CPLE_FileIO, "Can't open cloned source index");
2632 0 : return CE_Failure; // Source reported the error
2633 : }
2634 :
2635 1 : VSIFSeekL(srcidx, offset, SEEK_SET);
2636 1 : size = VSIFReadL(buffer, sizeof(ILIdx), static_cast<size_t>(size), srcidx);
2637 1 : if (size != GIntBig(buf.size()))
2638 : {
2639 0 : CPLError(CE_Failure, CPLE_FileIO, "Can't read cloned source index");
2640 0 : return CE_Failure; // Source reported the error
2641 : }
2642 :
2643 : // Mark the empty records as checked, by making the offset non-zero
2644 2 : for (vector<ILIdx>::iterator it = buf.begin(); it != buf.end(); ++it)
2645 : {
2646 1 : if (it->offset == 0 && it->size == 0)
2647 0 : it->offset = net64(1);
2648 : }
2649 :
2650 : // Write it in the right place in the local index file
2651 1 : VSIFSeekL(l_ifp, bias + offset, SEEK_SET);
2652 1 : size = VSIFWriteL(&buf[0], sizeof(ILIdx), static_cast<size_t>(size), l_ifp);
2653 1 : if (size != GIntBig(buf.size()))
2654 : {
2655 0 : CPLError(CE_Failure, CPLE_FileIO, "Can't write to cloning MRF index");
2656 0 : return CE_Failure; // Source reported the error
2657 : }
2658 :
2659 : // Cloned index updated, restart this function, it will work now
2660 1 : return ReadTileIdx(tinfo, pos, img, bias);
2661 : }
2662 :
2663 : NAMESPACE_MRF_END
|
