Line data Source code
1 : /*
2 : * Copyright (c) 2002-2012, California Institute of Technology.
3 : * All rights reserved. Based on Government Sponsored Research under contracts
4 : * NAS7-1407 and/or NAS7-03001.
5 : *
6 : * Redistribution and use in source and binary forms, with or without
7 : * modification, are permitted provided that the following conditions are met:
8 : * 1. Redistributions of source code must retain the above copyright notice,
9 : * this list of conditions and the following disclaimer.
10 : * 2. Redistributions in binary form must reproduce the above copyright
11 : * notice, this list of conditions and the following disclaimer in the
12 : * documentation and/or other materials provided with the distribution.
13 : * 3. Neither the name of the California Institute of Technology (Caltech),
14 : * its operating division the Jet Propulsion Laboratory (JPL), the National
15 : * Aeronautics and Space Administration (NASA), nor the names of its
16 : * contributors may be used to endorse or promote products derived from this
17 : * software without specific prior written permission.
18 : *
19 : * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20 : * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 : * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 : * ARE DISCLAIMED. IN NO EVENT SHALL THE CALIFORNIA INSTITUTE OF TECHNOLOGY BE
23 : * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 : * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 : * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 : * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 : * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 : * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 : * POSSIBILITY OF SUCH DAMAGE.
30 : *
31 : * Portions copyright 2014-2021 Esri
32 : *
33 : * Licensed under the Apache License, Version 2.0 (the "License");
34 : * you may not use this file except in compliance with the License.
35 : * You may obtain a copy of the License at
36 : *
37 : * http://www.apache.org/licenses/LICENSE-2.0
38 : *
39 : * Unless required by applicable law or agreed to in writing, software
40 : * distributed under the License is distributed on an "AS IS" BASIS,
41 : * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
42 : * See the License for the specific language governing permissions and
43 : * limitations under the License.
44 : */
45 :
46 : /******************************************************************************
47 : *
48 : * Project: Meta Raster File Format Driver Implementation, Dataset
49 : * Purpose: Implementation of GDAL dataset
50 : *
51 : * Author: Lucian Plesea, Lucian.Plesea jpl.nasa.gov, lplesea esri.com
52 : *
53 : ******************************************************************************
54 : *
55 : * The MRF dataset and the band are closely tied together, they should be
56 : * considered a single class, or a class (dataset) with extensions (bands).
57 : *
58 : *
59 : ****************************************************************************/
60 :
61 : #include "marfa.h"
62 : #include "mrfdrivercore.h"
63 : #include "cpl_multiproc.h" /* for CPLSleep() */
64 : #include "gdal_priv.h"
65 : #include <assert.h>
66 :
67 : #include <algorithm>
68 : #include <vector>
69 : #if defined(ZSTD_SUPPORT)
70 : #include <zstd.h>
71 : #endif
72 : using std::string;
73 : using std::vector;
74 :
75 : NAMESPACE_MRF_START
76 :
77 : // Initialize as invalid
78 355 : MRFDataset::MRFDataset()
79 : : zslice(0), idxSize(0), clonedSource(FALSE), nocopy(FALSE),
80 : bypass_cache(
81 355 : CPLTestBool(CPLGetConfigOption("MRF_BYPASSCACHING", "FALSE"))),
82 : mp_safe(FALSE), hasVersions(FALSE), verCount(0),
83 : bCrystalized(TRUE), // Assume not in create mode
84 : spacing(0), no_errors(0), missing(0), poSrcDS(nullptr), level(-1),
85 : cds(nullptr), scale(0.0), pbuffer(nullptr), pbsize(0), tile(ILSize()),
86 : bdirty(0), bGeoTransformValid(TRUE), poColorTable(nullptr), Quality(0),
87 710 : pzscctx(nullptr), pzsdctx(nullptr), read_timer(), write_timer(0)
88 : {
89 355 : m_oSRS.SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER);
90 : // X0 Xx Xy Y0 Yx Yy
91 355 : double gt[6] = {0.0, 1.0, 0.0, 0.0, 0.0, 1.0};
92 :
93 355 : memcpy(GeoTransform, gt, sizeof(gt));
94 355 : ifp.FP = dfp.FP = nullptr;
95 355 : dfp.acc = GF_Read;
96 355 : ifp.acc = GF_Read;
97 355 : }
98 :
99 177 : bool MRFDataset::SetPBuffer(unsigned int sz)
100 : {
101 177 : if (sz == 0)
102 : {
103 0 : CPLFree(pbuffer);
104 0 : pbuffer = nullptr;
105 : }
106 177 : void *pbufferNew = VSIRealloc(pbuffer, sz);
107 177 : if (pbufferNew == nullptr)
108 : {
109 0 : CPLError(CE_Failure, CPLE_OutOfMemory, "Cannot allocate %u bytes", sz);
110 0 : return false;
111 : }
112 177 : pbuffer = pbufferNew;
113 177 : pbsize = sz;
114 177 : return true;
115 : }
116 :
117 256 : static GDALColorEntry GetXMLColorEntry(CPLXMLNode *p)
118 : {
119 : GDALColorEntry ce;
120 256 : ce.c1 = static_cast<short>(getXMLNum(p, "c1", 0));
121 256 : ce.c2 = static_cast<short>(getXMLNum(p, "c2", 0));
122 256 : ce.c3 = static_cast<short>(getXMLNum(p, "c3", 0));
123 256 : ce.c4 = static_cast<short>(getXMLNum(p, "c4", 255));
124 256 : return ce;
125 : }
126 :
127 : //
128 : // Called by dataset destructor or at GDAL termination, to avoid
129 : // closing datasets whose drivers have already been unloaded
130 : //
131 355 : int MRFDataset::CloseDependentDatasets()
132 : {
133 355 : int bHasDroppedRef = GDALPamDataset::CloseDependentDatasets();
134 :
135 355 : if (poSrcDS)
136 : {
137 3 : bHasDroppedRef = TRUE;
138 3 : GDALClose(GDALDataset::ToHandle(poSrcDS));
139 3 : poSrcDS = nullptr;
140 : }
141 :
142 355 : if (cds)
143 : {
144 2 : bHasDroppedRef = TRUE;
145 2 : GDALClose(GDALDataset::ToHandle(cds));
146 2 : cds = nullptr;
147 : }
148 :
149 355 : return bHasDroppedRef;
150 : }
151 :
152 710 : MRFDataset::~MRFDataset()
153 : { // Make sure everything gets written
154 355 : if (0 != write_timer.count())
155 40 : CPLDebug("MRF_Timing", "Compression took %fms",
156 40 : 1e-6 * write_timer.count());
157 :
158 355 : if (0 != read_timer.count())
159 136 : CPLDebug("MRF_Timing", "Decompression took %fms",
160 136 : 1e-6 * read_timer.count());
161 :
162 355 : if (eAccess != GA_ReadOnly && !bCrystalized)
163 51 : if (!MRFDataset::Crystalize())
164 : {
165 : // Can't return error code from a destructor, just emit the error
166 10 : CPLError(CE_Failure, CPLE_FileIO, "Error creating files");
167 : }
168 :
169 355 : MRFDataset::FlushCache(true);
170 355 : MRFDataset::CloseDependentDatasets();
171 :
172 355 : if (ifp.FP)
173 251 : VSIFCloseL(ifp.FP);
174 355 : if (dfp.FP)
175 254 : VSIFCloseL(dfp.FP);
176 :
177 355 : delete poColorTable;
178 :
179 : // CPLFree ignores being called with NULL
180 355 : CPLFree(pbuffer);
181 355 : pbsize = 0;
182 : #if defined(ZSTD_SUPPORT)
183 355 : ZSTD_freeCCtx(static_cast<ZSTD_CCtx *>(pzscctx));
184 355 : ZSTD_freeDCtx(static_cast<ZSTD_DCtx *>(pzsdctx));
185 : #endif
186 710 : }
187 :
188 : /*
189 : *\brief Format specific RasterIO, may be bypassed by BlockBasedRasterIO by
190 : *setting GDAL_FORCE_CACHING to Yes, in which case the band ReadBlock and
191 : *WriteBLock are called directly
192 : */
193 123 : CPLErr MRFDataset::IRasterIO(GDALRWFlag eRWFlag, int nXOff, int nYOff,
194 : int nXSize, int nYSize, void *pData, int nBufXSize,
195 : int nBufYSize, GDALDataType eBufType,
196 : int nBandCount, int *panBandMap,
197 : GSpacing nPixelSpace, GSpacing nLineSpace,
198 : GSpacing nBandSpace,
199 : GDALRasterIOExtraArg *psExtraArgs)
200 : {
201 123 : CPLDebug("MRF_IO",
202 : "IRasterIO %s, %d, %d, %d, %d, bufsz %d,%d,%d strides P %d, L %d, "
203 : "B %d \n",
204 : eRWFlag == GF_Write ? "Write" : "Read", nXOff, nYOff, nXSize,
205 : nYSize, nBufXSize, nBufYSize, nBandCount,
206 : static_cast<int>(nPixelSpace), static_cast<int>(nLineSpace),
207 : static_cast<int>(nBandSpace));
208 :
209 123 : if (eRWFlag == GF_Write && !bCrystalized && !Crystalize())
210 : {
211 0 : CPLError(CE_Failure, CPLE_FileIO, "MRF: Error creating files");
212 0 : return CE_Failure;
213 : }
214 :
215 : //
216 : // Call the parent implementation, which splits it into bands and calls
217 : // their IRasterIO
218 : //
219 123 : return GDALPamDataset::IRasterIO(eRWFlag, nXOff, nYOff, nXSize, nYSize,
220 : pData, nBufXSize, nBufYSize, eBufType,
221 : nBandCount, panBandMap, nPixelSpace,
222 123 : nLineSpace, nBandSpace, psExtraArgs);
223 : }
224 :
225 : /**
226 : *\brief Build some overviews
227 : *
228 : * if nOverviews is 0, erase the overviews (reduce to base image only)
229 : */
230 :
231 31 : CPLErr MRFDataset::IBuildOverviews(const char *pszResampling, int nOverviews,
232 : const int *panOverviewList, int nBandsIn,
233 : const int *panBandList,
234 : GDALProgressFunc pfnProgress,
235 : void *pProgressData,
236 : CSLConstList papszOptions)
237 :
238 : {
239 31 : CPLErr eErr = CE_None;
240 31 : CPLDebug("MRF_OVERLAY", "IBuildOverviews %d, bands %d\n", nOverviews,
241 : nBandsIn);
242 :
243 31 : if (nBands != nBandsIn)
244 : {
245 0 : CPLError(CE_Failure, CPLE_NotSupported, "nBands = %d not supported",
246 : nBandsIn);
247 0 : return CE_Failure;
248 : }
249 :
250 : // If we don't have write access, then create external overviews
251 31 : if (GetAccess() != GA_Update)
252 : {
253 1 : CPLDebug("MRF", "File open read-only, creating overviews externally.");
254 1 : return GDALDataset::IBuildOverviews(
255 : pszResampling, nOverviews, panOverviewList, nBands, panBandList,
256 1 : pfnProgress, pProgressData, papszOptions);
257 : }
258 :
259 30 : if (nOverviews == 0)
260 : {
261 : // If there are none, nothing to do
262 0 : if (GetRasterBand(1)->GetOverviewCount() == 0)
263 0 : return CE_None;
264 :
265 0 : auto *b = static_cast<MRFRasterBand *>(GetRasterBand(1));
266 : // If the first band internal overviews don't exist, they are external
267 0 : if (b->overviews.empty())
268 0 : return GDALDataset::IBuildOverviews(
269 : pszResampling, nOverviews, panOverviewList, nBands, panBandList,
270 0 : pfnProgress, pProgressData, papszOptions);
271 :
272 : // We should clean overviews, but this is not allowed in an MRF
273 0 : CPLError(CE_Warning, CPLE_NotSupported,
274 : "MRF: Internal overviews cannot be removed, "
275 : "but they can be rebuilt");
276 0 : return CE_None;
277 : }
278 :
279 : // Array of source bands
280 : GDALRasterBand **papoBandList =
281 30 : static_cast<GDALRasterBand **>(CPLCalloc(sizeof(void *), nBands));
282 : // Array of destination bands
283 : GDALRasterBand **papoOverviewBandList =
284 30 : static_cast<GDALRasterBand **>(CPLCalloc(sizeof(void *), nBands));
285 : // Triple level pointer, that's what GDAL ROMB wants
286 : GDALRasterBand ***papapoOverviewBands =
287 30 : static_cast<GDALRasterBand ***>(CPLCalloc(sizeof(void *), nBands));
288 :
289 : int *panOverviewListNew =
290 30 : static_cast<int *>(CPLMalloc(sizeof(int) * nOverviews));
291 30 : memcpy(panOverviewListNew, panOverviewList, sizeof(int) * nOverviews);
292 :
293 : try
294 : { // Throw an error code, to make sure memory gets freed properly
295 : // Modify the metadata file if it doesn't already have the Rset model
296 : // set
297 30 : if (0.0 == scale)
298 : {
299 30 : CPLXMLNode *config = ReadConfig();
300 : try
301 : {
302 : const char *model =
303 30 : CPLGetXMLValue(config, "Rsets.model", "uniform");
304 30 : if (!EQUAL(model, "uniform"))
305 : {
306 0 : CPLError(CE_Failure, CPLE_AppDefined,
307 : "MRF:IBuildOverviews, Overviews not implemented "
308 : "for model %s",
309 : model);
310 0 : throw CE_Failure;
311 : }
312 :
313 : // The scale value is the same as first overview
314 30 : scale =
315 30 : strtod(CPLGetXMLValue(
316 : config, "Rsets.scale",
317 60 : CPLOPrintf("%d", panOverviewList[0]).c_str()),
318 : nullptr);
319 :
320 30 : if (static_cast<int>(scale) != 2 &&
321 0 : (EQUALN("Avg", pszResampling, 3) ||
322 0 : EQUALN("Nnb", pszResampling, 3)))
323 : {
324 0 : CPLError(CE_Failure, CPLE_IllegalArg,
325 : "MRF internal resampling only works for a scale "
326 : "factor of two");
327 0 : throw CE_Failure;
328 : }
329 :
330 : // Initialize the empty overlays, all of them for a given scale
331 : // They could already exist, in which case they are not erased
332 30 : idxSize = AddOverviews(int(scale));
333 :
334 : // If we don't have overviews, don't try to generate them
335 30 : if (GetRasterBand(1)->GetOverviewCount() == 0)
336 0 : throw CE_None;
337 :
338 30 : if (!CheckFileSize(current.idxfname, idxSize, GA_Update))
339 : {
340 0 : CPLError(CE_Failure, CPLE_AppDefined,
341 : "MRF: Can't extend index file");
342 0 : throw CE_Failure;
343 : }
344 :
345 : // Set the uniform node, in case it was not set before, and
346 : // save the new configuration
347 30 : CPLSetXMLValue(config, "Rsets.#model", "uniform");
348 30 : CPLSetXMLValue(config, "Rsets.#scale", PrintDouble(scale));
349 :
350 30 : if (!WriteConfig(config))
351 : {
352 0 : CPLError(CE_Failure, CPLE_AppDefined,
353 : "MRF: Can't rewrite the metadata file");
354 0 : throw CE_Failure;
355 : }
356 30 : CPLDestroyXMLNode(config);
357 30 : config = nullptr;
358 : }
359 0 : catch (const CPLErr &)
360 : {
361 0 : CPLDestroyXMLNode(config);
362 0 : throw; // Rethrow
363 : }
364 :
365 : // To avoid issues with blacks overviews, generate all of them
366 : // if the user asked for a couple of overviews in the correct
367 : // sequence and starting with the lowest one
368 90 : if (!EQUAL(pszResampling, "NONE") &&
369 32 : nOverviews != GetRasterBand(1)->GetOverviewCount() &&
370 2 : CPLTestBool(
371 : CPLGetConfigOption("MRF_ALL_OVERVIEW_LEVELS", "YES")))
372 : {
373 2 : bool bIncreasingPowers =
374 2 : (panOverviewList[0] == static_cast<int>(scale));
375 3 : for (int i = 1; i < nOverviews; i++)
376 1 : bIncreasingPowers =
377 2 : bIncreasingPowers &&
378 1 : (panOverviewList[i] ==
379 1 : static_cast<int>(scale * panOverviewList[i - 1]));
380 :
381 2 : int ovrcount = GetRasterBand(1)->GetOverviewCount();
382 2 : if (bIncreasingPowers && nOverviews != ovrcount)
383 : {
384 2 : CPLDebug("MRF",
385 : "Generating %d levels instead of the %d requested",
386 : ovrcount, nOverviews);
387 2 : nOverviews = ovrcount;
388 4 : panOverviewListNew = reinterpret_cast<int *>(CPLRealloc(
389 2 : panOverviewListNew, sizeof(int) * nOverviews));
390 2 : panOverviewListNew[0] = static_cast<int>(scale);
391 5 : for (int i = 1; i < nOverviews; i++)
392 3 : panOverviewListNew[i] =
393 3 : static_cast<int>(scale * panOverviewListNew[i - 1]);
394 : }
395 : }
396 : }
397 :
398 30 : if (static_cast<int>(scale) != 2 && (EQUALN("Avg", pszResampling, 3) ||
399 0 : EQUALN("Nnb", pszResampling, 3)))
400 : {
401 0 : CPLError(
402 : CE_Failure, CPLE_IllegalArg,
403 : "MRF internal resampling only works for a scale factor of two");
404 0 : throw CE_Failure;
405 : }
406 :
407 63 : for (int i = 0; i < nOverviews; i++)
408 : {
409 : // Verify that scales are reasonable, val/scale has to be an integer
410 33 : if (!IsPower(panOverviewListNew[i], scale))
411 : {
412 0 : CPLError(CE_Warning, CPLE_AppDefined,
413 : "MRF:IBuildOverviews, overview factor %d is not a "
414 : "power of %f",
415 0 : panOverviewListNew[i], scale);
416 0 : continue;
417 : };
418 :
419 33 : int srclevel = int(logbase(panOverviewListNew[i], scale) - 0.5);
420 33 : MRFRasterBand *b = static_cast<MRFRasterBand *>(GetRasterBand(1));
421 :
422 : // Warn for requests for invalid levels
423 33 : if (srclevel >= b->GetOverviewCount())
424 : {
425 0 : CPLError(CE_Warning, CPLE_AppDefined,
426 : "MRF:IBuildOverviews, overview factor %d is not valid "
427 : "for this dataset",
428 0 : panOverviewListNew[i]);
429 0 : continue;
430 : }
431 :
432 : // Generate the overview using the previous level as the source
433 :
434 : // Use "avg" flag to trigger the internal average sampling
435 33 : if (EQUALN("Avg", pszResampling, 3) ||
436 19 : EQUALN("Nnb", pszResampling, 3))
437 : {
438 26 : int sampling = EQUALN("Avg", pszResampling, 3) ? SAMPLING_Avg
439 : : SAMPLING_Near;
440 : // Internal, using PatchOverview
441 26 : if (srclevel > 0)
442 : b = static_cast<MRFRasterBand *>(
443 3 : b->GetOverview(srclevel - 1));
444 :
445 : eErr =
446 26 : PatchOverview(0, 0, b->nBlocksPerRow, b->nBlocksPerColumn,
447 : srclevel, 0, sampling);
448 26 : if (eErr == CE_Failure)
449 26 : throw eErr;
450 : }
451 : else
452 : {
453 : //
454 : // Use the GDAL method, which is slightly different for bilinear
455 : // interpolation and also handles nearest mode
456 : //
457 : //
458 14 : for (int iBand = 0; iBand < nBands; iBand++)
459 : {
460 : // This is the base level
461 7 : papoBandList[iBand] = GetRasterBand(panBandList[iBand]);
462 : // Set up the destination
463 14 : papoOverviewBandList[iBand] =
464 7 : papoBandList[iBand]->GetOverview(srclevel);
465 :
466 : // Use the previous level as the source, the overviews are 0
467 : // based thus an extra -1
468 7 : if (srclevel > 0)
469 0 : papoBandList[iBand] =
470 0 : papoBandList[iBand]->GetOverview(srclevel - 1);
471 :
472 : // Hook it up, via triple pointer level
473 7 : papapoOverviewBands[iBand] = &(papoOverviewBandList[iBand]);
474 : }
475 :
476 : //
477 : // Ready, generate this overview
478 : // Note that this function has a bug in GDAL, the block stepping
479 : // is incorrect It can generate multiple overview in one call,
480 : // Could rewrite this loop so this function only gets called
481 : // once
482 : //
483 7 : GDALRegenerateOverviewsMultiBand(
484 : nBands, papoBandList, 1, papapoOverviewBands, pszResampling,
485 : pfnProgress, pProgressData, papszOptions);
486 : }
487 : }
488 : }
489 0 : catch (const CPLErr &e)
490 : {
491 0 : eErr = e;
492 : }
493 :
494 30 : CPLFree(panOverviewListNew);
495 30 : CPLFree(papapoOverviewBands);
496 30 : CPLFree(papoOverviewBandList);
497 30 : CPLFree(papoBandList);
498 30 : return eErr;
499 : }
500 :
501 : /*
502 : *\brief blank separated list to vector of doubles
503 : */
504 20 : static void list2vec(std::vector<double> &v, const char *pszList)
505 : {
506 20 : if ((pszList == nullptr) || (pszList[0] == 0))
507 0 : return;
508 20 : char **papszTokens = CSLTokenizeString2(
509 : pszList, " \t\n\r", CSLT_STRIPLEADSPACES | CSLT_STRIPENDSPACES);
510 20 : v.clear();
511 41 : for (int i = 0; i < CSLCount(papszTokens); i++)
512 21 : v.push_back(CPLStrtod(papszTokens[i], nullptr));
513 20 : CSLDestroy(papszTokens);
514 : }
515 :
516 20 : void MRFDataset::SetNoDataValue(const char *pszVal)
517 : {
518 20 : list2vec(vNoData, pszVal);
519 20 : }
520 :
521 0 : void MRFDataset::SetMinValue(const char *pszVal)
522 : {
523 0 : list2vec(vMin, pszVal);
524 0 : }
525 :
526 0 : void MRFDataset::SetMaxValue(const char *pszVal)
527 : {
528 0 : list2vec(vMax, pszVal);
529 0 : }
530 :
531 : /**
532 : *
533 : *\brief Read the XML config tree, from file
534 : * Caller is responsible for freeing the memory
535 : *
536 : * @return NULL on failure, or the document tree on success.
537 : *
538 : */
539 30 : CPLXMLNode *MRFDataset::ReadConfig() const
540 : {
541 30 : if (fname[0] == '<')
542 0 : return CPLParseXMLString(fname);
543 30 : return CPLParseXMLFile(fname);
544 : }
545 :
546 : /**
547 : *\brief Write the XML config tree
548 : * Caller is responsible for correctness of data
549 : * and for freeing the memory
550 : *
551 : * @param config The document tree to write
552 : * @return TRUE on success, FALSE otherwise
553 : */
554 196 : int MRFDataset::WriteConfig(CPLXMLNode *config)
555 : {
556 196 : if (fname[0] == '<')
557 0 : return FALSE;
558 196 : return CPLSerializeXMLTreeToFile(config, fname);
559 : }
560 :
561 : static void
562 5 : stringSplit(vector<string> &theStringVector, // Altered/returned value
563 : const string &theString, size_t start = 0,
564 : const char theDelimiter = ' ')
565 : {
566 : while (true)
567 : {
568 5 : size_t end = theString.find(theDelimiter, start);
569 5 : if (string::npos == end)
570 : {
571 5 : theStringVector.push_back(theString.substr(start));
572 5 : return;
573 : }
574 0 : theStringVector.push_back(theString.substr(start, end - start));
575 0 : start = end + 1;
576 0 : }
577 : }
578 :
579 : // Returns the number following the prefix if it exists in one of the vector
580 : // strings Otherwise it returns the default
581 15 : static int getnum(const vector<string> &theStringVector, const char prefix,
582 : int def)
583 : {
584 25 : for (unsigned int i = 0; i < theStringVector.size(); i++)
585 15 : if (theStringVector[i][0] == prefix)
586 5 : return atoi(theStringVector[i].c_str() + 1);
587 10 : return def;
588 : }
589 :
590 : /**
591 : *\brief Open a MRF file
592 : *
593 : */
594 194 : GDALDataset *MRFDataset::Open(GDALOpenInfo *poOpenInfo)
595 : {
596 194 : if (!MRFDriverIdentify(poOpenInfo))
597 0 : return nullptr;
598 :
599 194 : CPLXMLNode *config = nullptr;
600 194 : CPLErr ret = CE_None;
601 194 : const char *pszFileName = poOpenInfo->pszFilename;
602 :
603 194 : int level = -1; // All levels
604 194 : int version = 0; // Current
605 194 : int zslice = 0;
606 388 : string fn; // Used to parse and adjust the file name
607 :
608 : // Different ways to open an MRF
609 194 : if (poOpenInfo->nHeaderBytes >= 10)
610 : {
611 189 : const char *pszHeader =
612 : reinterpret_cast<char *>(poOpenInfo->pabyHeader);
613 189 : if (STARTS_WITH(pszHeader, "<MRF_META>")) // Regular file name
614 183 : config = CPLParseXMLFile(pszFileName);
615 : #if defined(LERC)
616 : else
617 6 : config = LERC_Band::GetMRFConfig(poOpenInfo);
618 : #endif
619 : }
620 : else
621 : {
622 5 : if (EQUALN(pszFileName, "<MRF_META>", 10)) // Content as file name
623 0 : config = CPLParseXMLString(pszFileName);
624 : else
625 : { // Try Ornate file name
626 5 : fn = pszFileName;
627 5 : size_t pos = fn.find(":MRF:");
628 5 : if (string::npos != pos)
629 : { // Tokenize and pick known options
630 5 : vector<string> tokens;
631 5 : stringSplit(tokens, fn, pos + 5, ':');
632 5 : level = getnum(tokens, 'L', -1);
633 5 : version = getnum(tokens, 'V', 0);
634 5 : zslice = getnum(tokens, 'Z', 0);
635 5 : fn.resize(pos); // Cut the ornamentations
636 5 : pszFileName = fn.c_str();
637 5 : config = CPLParseXMLFile(pszFileName);
638 : }
639 : }
640 : }
641 :
642 194 : if (!config)
643 0 : return nullptr;
644 :
645 194 : MRFDataset *ds = new MRFDataset();
646 194 : ds->fname = pszFileName;
647 194 : ds->eAccess = poOpenInfo->eAccess;
648 194 : ds->level = level;
649 194 : ds->zslice = zslice;
650 :
651 : // OpenOptions can override file name arguments
652 194 : ds->ProcessOpenOptions(poOpenInfo->papszOpenOptions);
653 :
654 194 : if (level == -1)
655 192 : ret = ds->Initialize(config);
656 : else
657 : {
658 : // Open the whole dataset, then pick one level
659 2 : ds->cds = new MRFDataset();
660 2 : ds->cds->fname = pszFileName;
661 2 : ds->cds->eAccess = ds->eAccess;
662 2 : ds->zslice = zslice;
663 2 : ret = ds->cds->Initialize(config);
664 2 : if (ret == CE_None)
665 2 : ret = ds->LevelInit(level);
666 : }
667 194 : CPLDestroyXMLNode(config);
668 :
669 194 : if (ret != CE_None)
670 : {
671 23 : delete ds;
672 23 : return nullptr;
673 : }
674 :
675 : // Open a single version
676 171 : if (version != 0)
677 2 : ret = ds->SetVersion(version);
678 :
679 171 : if (ret != CE_None)
680 : {
681 1 : delete ds;
682 1 : return nullptr;
683 : }
684 :
685 : // Tell PAM what our real file name is, to help it find the aux.xml
686 170 : ds->SetPhysicalFilename(pszFileName);
687 : // Don't mess with metadata after this, otherwise PAM will re-write the
688 : // aux.xml
689 170 : ds->TryLoadXML();
690 :
691 : /* -------------------------------------------------------------------- */
692 : /* Open external overviews. */
693 : /* -------------------------------------------------------------------- */
694 170 : ds->oOvManager.Initialize(ds, pszFileName);
695 :
696 170 : return ds;
697 : }
698 :
699 : // Adjust the band images with the right offset, then adjust the sizes
700 2 : CPLErr MRFDataset::SetVersion(int version)
701 : {
702 2 : if (!hasVersions || version > verCount)
703 : {
704 1 : CPLError(CE_Failure, CPLE_AppDefined,
705 : "GDAL MRF: Version number error!");
706 1 : return CE_Failure;
707 : }
708 : // Size of one version index
709 2 : for (int bcount = 1; bcount <= nBands; bcount++)
710 : {
711 : MRFRasterBand *srcband =
712 1 : reinterpret_cast<MRFRasterBand *>(GetRasterBand(bcount));
713 1 : srcband->img.idxoffset += idxSize * verCount;
714 1 : for (int l = 0; l < srcband->GetOverviewCount(); l++)
715 : {
716 : MRFRasterBand *band =
717 0 : reinterpret_cast<MRFRasterBand *>(srcband->GetOverview(l));
718 0 : if (band != nullptr)
719 0 : band->img.idxoffset += idxSize * verCount;
720 : }
721 : }
722 1 : hasVersions = 0;
723 1 : return CE_None;
724 : }
725 :
726 2 : CPLErr MRFDataset::LevelInit(const int l)
727 : {
728 : // Test that this level does exist
729 2 : if (l < 0 || l >= cds->GetRasterBand(1)->GetOverviewCount())
730 : {
731 1 : CPLError(CE_Failure, CPLE_AppDefined,
732 : "GDAL MRF: Overview not present!");
733 1 : return CE_Failure;
734 : }
735 :
736 : MRFRasterBand *srcband = reinterpret_cast<MRFRasterBand *>(
737 1 : cds->GetRasterBand(1)->GetOverview(l));
738 :
739 : // Copy the sizes from this level
740 1 : full = srcband->img;
741 1 : current = srcband->img;
742 1 : current.size.c = cds->current.size.c;
743 1 : scale = cds->scale;
744 1 : const auto poSRS = cds->GetSpatialRef();
745 1 : if (poSRS)
746 1 : m_oSRS = *poSRS;
747 :
748 1 : SetMetadataItem("INTERLEAVE", OrderName(current.order), "IMAGE_STRUCTURE");
749 1 : SetMetadataItem("COMPRESSION", CompName(current.comp), "IMAGE_STRUCTURE");
750 :
751 1 : bGeoTransformValid = (CE_None == cds->GetGeoTransform(GeoTransform));
752 4 : for (int i = 0; i < l + 1; i++)
753 : {
754 3 : GeoTransform[1] *= scale;
755 3 : GeoTransform[5] *= scale;
756 : }
757 :
758 1 : nRasterXSize = current.size.x;
759 1 : nRasterYSize = current.size.y;
760 1 : nBands = current.size.c;
761 :
762 : // Add the bands, copy constructor so they can be closed independently
763 2 : for (int i = 1; i <= nBands; i++)
764 1 : SetBand(i, new MRFLRasterBand(reinterpret_cast<MRFRasterBand *>(
765 1 : cds->GetRasterBand(i)->GetOverview(l))));
766 1 : return CE_None;
767 : }
768 :
769 : // Is the string positive or not
770 1221 : inline bool on(const char *pszValue)
771 : {
772 1221 : if (!pszValue || pszValue[0] == 0)
773 103 : return false;
774 2224 : return EQUAL(pszValue, "ON") || EQUAL(pszValue, "TRUE") ||
775 2224 : EQUAL(pszValue, "YES");
776 : }
777 :
778 : /**
779 : *\brief Initialize the image structure and the dataset from the XML Raster node
780 : *
781 : * @param image the structure to be initialized
782 : * @param ds the parent dataset, some things get inherited
783 : * @param defimage defimage
784 : *
785 : * The structure should be initialized with the default values as much as
786 : *possible
787 : *
788 : */
789 :
790 350 : static CPLErr Init_Raster(ILImage &image, MRFDataset *ds, CPLXMLNode *defimage)
791 : {
792 : CPLXMLNode *node; // temporary
793 350 : if (!defimage)
794 : {
795 0 : CPLError(CE_Failure, CPLE_AppDefined,
796 : "GDAL MRF: Can't find raster info");
797 0 : return CE_Failure;
798 : }
799 :
800 : // Size is mandatory
801 350 : node = CPLGetXMLNode(defimage, "Size");
802 :
803 350 : if (node)
804 : {
805 350 : image.size = ILSize(static_cast<int>(getXMLNum(node, "x", -1)),
806 350 : static_cast<int>(getXMLNum(node, "y", -1)),
807 350 : static_cast<int>(getXMLNum(node, "z", 1)),
808 350 : static_cast<int>(getXMLNum(node, "c", 1)), 0);
809 : }
810 :
811 : // Basic checks
812 350 : if (!node || image.size.x < 1 || image.size.y < 1 || image.size.z < 0 ||
813 700 : image.size.c < 0 || !GDALCheckBandCount(image.size.c, FALSE))
814 : {
815 0 : CPLError(CE_Failure, CPLE_AppDefined, "Raster size missing or invalid");
816 0 : return CE_Failure;
817 : }
818 :
819 : // Pagesize, defaults to 512,512,1,c
820 350 : image.pagesize = ILSize(std::min(512, image.size.x),
821 350 : std::min(512, image.size.y), 1, image.size.c);
822 :
823 350 : node = CPLGetXMLNode(defimage, "PageSize");
824 350 : if (node)
825 : {
826 350 : image.pagesize =
827 350 : ILSize(static_cast<int>(getXMLNum(node, "x", image.pagesize.x)),
828 350 : static_cast<int>(getXMLNum(node, "y", image.pagesize.y)),
829 : 1, // One slice at a time, forced
830 350 : static_cast<int>(getXMLNum(node, "c", image.pagesize.c)));
831 350 : if (image.pagesize.x < 1 || image.pagesize.y < 1 ||
832 350 : image.pagesize.c <= 0)
833 : {
834 0 : CPLError(CE_Failure, CPLE_IllegalArg, "Invalid PageSize");
835 0 : return CE_Failure;
836 : }
837 : }
838 :
839 : // Page Encoding, defaults to PNG
840 350 : const char *pszCompression = CPLGetXMLValue(defimage, "Compression", "PNG");
841 350 : image.comp = CompToken(pszCompression);
842 350 : if (image.comp == IL_ERR_COMP)
843 : {
844 0 : CPLError(CE_Failure, CPLE_IllegalArg,
845 : "GDAL MRF: Compression %s is unknown", pszCompression);
846 0 : return CE_Failure;
847 : }
848 :
849 : // Is there a palette?
850 : //
851 : // GDAL only supports RGB+A palette, the other modes don't work
852 : //
853 :
854 643 : if ((image.pagesize.c == 1) &&
855 293 : (nullptr != (node = CPLGetXMLNode(defimage, "Palette"))))
856 : {
857 1 : int entries = static_cast<int>(getXMLNum(node, "Size", 255));
858 1 : GDALPaletteInterp eInterp = GPI_RGB;
859 1 : if ((entries > 0) && (entries < 257))
860 : {
861 1 : GDALColorEntry ce_start = {0, 0, 0, 255}, ce_end = {0, 0, 0, 255};
862 :
863 : // Create it and initialize it to black opaque
864 1 : GDALColorTable *poColorTable = new GDALColorTable(eInterp);
865 1 : poColorTable->CreateColorRamp(0, &ce_start, entries - 1, &ce_end);
866 : // Read the values
867 1 : CPLXMLNode *p = CPLGetXMLNode(node, "Entry");
868 1 : if (p)
869 : {
870 : // Initialize the first entry
871 1 : ce_start = GetXMLColorEntry(p);
872 1 : int start_idx = static_cast<int>(getXMLNum(p, "idx", 0));
873 1 : if (start_idx < 0)
874 : {
875 0 : CPLError(CE_Failure, CPLE_IllegalArg,
876 : "GDAL MRF: Palette index %d not allowed",
877 : start_idx);
878 0 : delete poColorTable;
879 0 : return CE_Failure;
880 : }
881 1 : poColorTable->SetColorEntry(start_idx, &ce_start);
882 256 : while (nullptr != (p = SearchXMLSiblings(p, "Entry")))
883 : {
884 : // For every entry, create a ramp
885 255 : ce_end = GetXMLColorEntry(p);
886 : int end_idx =
887 255 : static_cast<int>(getXMLNum(p, "idx", start_idx + 1));
888 255 : if ((end_idx <= start_idx) || (start_idx >= entries))
889 : {
890 0 : CPLError(CE_Failure, CPLE_IllegalArg,
891 : "GDAL MRF: Index Error at index %d", end_idx);
892 0 : delete poColorTable;
893 0 : return CE_Failure;
894 : }
895 255 : poColorTable->CreateColorRamp(start_idx, &ce_start, end_idx,
896 : &ce_end);
897 255 : ce_start = ce_end;
898 255 : start_idx = end_idx;
899 : }
900 : }
901 :
902 1 : ds->SetColorTable(poColorTable);
903 : }
904 : else
905 : {
906 0 : CPLError(CE_Failure, CPLE_IllegalArg,
907 : "GDAL MRF: Palette definition error");
908 0 : return CE_Failure;
909 : }
910 : }
911 :
912 : // Order of increment
913 350 : if (image.pagesize.c != image.size.c && image.pagesize.c != 1)
914 : {
915 : // Fixes heap buffer overflow in
916 : // GDALMRFRasterBand::ReadInterleavedBlock() See
917 : // https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=2884
918 0 : CPLError(CE_Failure, CPLE_NotSupported,
919 : "GDAL MRF: image.pagesize.c = %d and image.size.c = %d",
920 : image.pagesize.c, image.size.c);
921 0 : return CE_Failure;
922 : }
923 :
924 350 : image.order = OrderToken(
925 : CPLGetXMLValue(defimage, "Order",
926 350 : (image.pagesize.c != image.size.c) ? "BAND" : "PIXEL"));
927 350 : if (image.order == IL_ERR_ORD)
928 : {
929 0 : CPLError(CE_Failure, CPLE_AppDefined, "GDAL MRF: Order %s is unknown",
930 : CPLGetXMLValue(defimage, "Order", nullptr));
931 0 : return CE_Failure;
932 : }
933 :
934 350 : const char *photo_val = CPLGetXMLValue(defimage, "Photometric", nullptr);
935 350 : if (photo_val)
936 5 : ds->SetPhotometricInterpretation(photo_val);
937 :
938 350 : image.quality = atoi(CPLGetXMLValue(defimage, "Quality", "85"));
939 350 : if (image.quality < 0 || image.quality > 99)
940 : {
941 0 : CPLError(CE_Warning, CPLE_AppDefined,
942 : "GDAL MRF: Quality setting error, using default of 85");
943 0 : image.quality = 85;
944 : }
945 :
946 : // Data Type, use GDAL Names
947 350 : image.dt = GDALGetDataTypeByName(
948 : CPLGetXMLValue(defimage, "DataType", GDALGetDataTypeName(image.dt)));
949 350 : if (image.dt == GDT_Unknown || GDALGetDataTypeSize(image.dt) == 0)
950 : {
951 0 : CPLError(CE_Failure, CPLE_AppDefined, "GDAL MRF: Unsupported type");
952 0 : return CE_Failure;
953 : }
954 :
955 : // Check the endianness if needed, assume host order
956 350 : if (is_Endianness_Dependent(image.dt, image.comp))
957 66 : image.nbo = on(CPLGetXMLValue(defimage, "NetByteOrder", "No"));
958 :
959 350 : CPLXMLNode *DataValues = CPLGetXMLNode(defimage, "DataValues");
960 350 : if (nullptr != DataValues)
961 : {
962 20 : const char *pszValue = CPLGetXMLValue(DataValues, "NoData", nullptr);
963 20 : if (pszValue)
964 20 : ds->SetNoDataValue(pszValue);
965 20 : pszValue = CPLGetXMLValue(DataValues, "min", nullptr);
966 20 : if (pszValue)
967 0 : ds->SetMinValue(pszValue);
968 20 : pszValue = CPLGetXMLValue(DataValues, "max", nullptr);
969 20 : if (pszValue)
970 0 : ds->SetMaxValue(pszValue);
971 : }
972 :
973 : // Calculate the page size in bytes
974 350 : const int nDTSize = GDALGetDataTypeSizeBytes(image.dt);
975 350 : if (nDTSize <= 0 || image.pagesize.z <= 0 ||
976 350 : image.pagesize.x > INT_MAX / image.pagesize.y ||
977 350 : image.pagesize.x * image.pagesize.y > INT_MAX / image.pagesize.z ||
978 350 : image.pagesize.x * image.pagesize.y * image.pagesize.z >
979 350 : INT_MAX / image.pagesize.c ||
980 350 : image.pagesize.x * image.pagesize.y * image.pagesize.z *
981 350 : image.pagesize.c >
982 350 : INT_MAX / nDTSize)
983 : {
984 0 : CPLError(CE_Failure, CPLE_AppDefined, "MRF page size too big");
985 0 : return CE_Failure;
986 : }
987 350 : image.pageSizeBytes = nDTSize * image.pagesize.x * image.pagesize.y *
988 350 : image.pagesize.z * image.pagesize.c;
989 :
990 : // Calculate the page count, including the total for the level
991 350 : image.pagecount = pcount(image.size, image.pagesize);
992 350 : if (image.pagecount.l < 0)
993 : {
994 0 : return CE_Failure;
995 : }
996 :
997 : // Data File Name and base offset
998 : image.datfname =
999 350 : getFname(defimage, "DataFile", ds->GetFname(), ILComp_Ext[image.comp]);
1000 350 : image.dataoffset = static_cast<int>(
1001 350 : getXMLNum(CPLGetXMLNode(defimage, "DataFile"), "offset", 0.0));
1002 :
1003 : // Index File Name and base offset
1004 350 : image.idxfname = getFname(defimage, "IndexFile", ds->GetFname(), ".idx");
1005 350 : image.idxoffset = static_cast<int>(
1006 350 : getXMLNum(CPLGetXMLNode(defimage, "IndexFile"), "offset", 0.0));
1007 :
1008 350 : return CE_None;
1009 : }
1010 :
1011 151 : char **MRFDataset::GetFileList()
1012 : {
1013 151 : char **papszFileList = nullptr;
1014 :
1015 : // Add the header file name if it is real
1016 : VSIStatBufL sStat;
1017 151 : if (VSIStatExL(fname, &sStat, VSI_STAT_EXISTS_FLAG) == 0)
1018 151 : papszFileList = CSLAddString(papszFileList, fname);
1019 :
1020 : // These two should be real
1021 : // We don't really want to add these files, since they will be erased when
1022 : // an mrf is overwritten This collides with the concept that the data file
1023 : // never shrinks. Same goes with the index, in case we just want to add
1024 : // things to it.
1025 : // papszFileList = CSLAddString( papszFileList, full.datfname);
1026 : // papszFileList = CSLAddString( papszFileList, full.idxfname);
1027 : // if (!source.empty())
1028 : // papszFileList = CSLAddString( papszFileList, source);
1029 :
1030 151 : return papszFileList;
1031 : }
1032 :
1033 : // Try to create all the folders in the path in sequence, ignore errors
1034 4 : static void mkdir_r(string const &fname)
1035 : {
1036 4 : size_t loc = fname.find_first_of("\\/");
1037 4 : if (loc == string::npos)
1038 0 : return;
1039 : while (true)
1040 : {
1041 5 : ++loc;
1042 5 : loc = fname.find_first_of("\\/", loc);
1043 5 : if (loc == string::npos)
1044 4 : break;
1045 1 : VSIMkdir(fname.substr(0, loc).c_str(), 0);
1046 : }
1047 : }
1048 :
1049 : // Returns the dataset index file or null
1050 7239 : VSILFILE *MRFDataset::IdxFP()
1051 : {
1052 7239 : if (ifp.FP != nullptr)
1053 6982 : return ifp.FP;
1054 :
1055 : // If missing is set, we already checked, there is no index
1056 257 : if (missing)
1057 0 : return nullptr;
1058 :
1059 : // If name starts with '(' it is not a real file name
1060 257 : if (current.idxfname[0] == '(')
1061 0 : return nullptr;
1062 :
1063 257 : const char *mode = "rb";
1064 257 : ifp.acc = GF_Read;
1065 :
1066 257 : if (eAccess == GA_Update || !source.empty())
1067 : {
1068 153 : mode = "r+b";
1069 153 : ifp.acc = GF_Write;
1070 : }
1071 :
1072 257 : ifp.FP = VSIFOpenL(current.idxfname, mode);
1073 :
1074 : // If file didn't open for reading and no_errors is set, just return null
1075 : // and make a note
1076 257 : if (ifp.FP == nullptr && eAccess == GA_ReadOnly && no_errors)
1077 : {
1078 0 : missing = 1;
1079 0 : return nullptr;
1080 : }
1081 :
1082 : // need to create the index file
1083 355 : if (ifp.FP == nullptr && !bCrystalized &&
1084 98 : (eAccess == GA_Update || !source.empty()))
1085 : {
1086 98 : mode = "w+b";
1087 98 : ifp.FP = VSIFOpenL(current.idxfname, mode);
1088 : }
1089 :
1090 257 : if (nullptr == ifp.FP && !source.empty())
1091 : {
1092 : // caching and cloning, try making the folder and attempt again
1093 4 : mkdir_r(current.idxfname);
1094 4 : ifp.FP = VSIFOpenL(current.idxfname, mode);
1095 : }
1096 :
1097 257 : GIntBig expected_size = idxSize;
1098 257 : if (clonedSource)
1099 2 : expected_size *= 2;
1100 :
1101 257 : if (nullptr != ifp.FP)
1102 : {
1103 391 : if (!bCrystalized &&
1104 144 : !CheckFileSize(current.idxfname, expected_size, GA_Update))
1105 : {
1106 0 : CPLError(CE_Failure, CPLE_FileIO,
1107 : "MRF: Can't extend the cache index file %s",
1108 : current.idxfname.c_str());
1109 0 : return nullptr;
1110 : }
1111 :
1112 247 : if (source.empty())
1113 243 : return ifp.FP;
1114 :
1115 : // Make sure the index is large enough before proceeding
1116 : // Timeout in .1 seconds, can't really guarantee the accuracy
1117 : // So this is about half second, should be sufficient
1118 4 : int timeout = 5;
1119 0 : do
1120 : {
1121 4 : if (CheckFileSize(current.idxfname, expected_size, GA_ReadOnly))
1122 4 : return ifp.FP;
1123 0 : CPLSleep(0.100); /* 100 ms */
1124 0 : } while (--timeout);
1125 :
1126 : // If we get here it is a time-out
1127 0 : CPLError(CE_Failure, CPLE_AppDefined,
1128 : "GDAL MRF: Timeout on fetching cloned index file %s\n",
1129 : current.idxfname.c_str());
1130 0 : return nullptr;
1131 : }
1132 :
1133 : // If single tile, and no index file, let the caller figure it out
1134 10 : if (IsSingleTile())
1135 6 : return nullptr;
1136 :
1137 : // Error if this is not a caching MRF
1138 4 : if (source.empty())
1139 : {
1140 0 : CPLError(CE_Failure, CPLE_AppDefined,
1141 : "GDAL MRF: Can't open index file %s\n",
1142 : current.idxfname.c_str());
1143 0 : return nullptr;
1144 : }
1145 :
1146 : // Caching/Cloning MRF and index could be read only
1147 : // Is this actually works, we should try again, maybe somebody else just
1148 : // created the file?
1149 4 : mode = "rb";
1150 4 : ifp.acc = GF_Read;
1151 4 : ifp.FP = VSIFOpenL(current.idxfname, mode);
1152 4 : if (nullptr != ifp.FP)
1153 0 : return ifp.FP;
1154 :
1155 : // Caching and index file absent, create it
1156 : // Due to a race, multiple processes might do this at the same time, but
1157 : // that is fine
1158 4 : ifp.FP = VSIFOpenL(current.idxfname, "wb");
1159 4 : if (nullptr == ifp.FP)
1160 : {
1161 0 : CPLError(CE_Failure, CPLE_AppDefined,
1162 : "Can't create the MRF cache index file %s",
1163 : current.idxfname.c_str());
1164 0 : return nullptr;
1165 : }
1166 4 : VSIFCloseL(ifp.FP);
1167 4 : ifp.FP = nullptr;
1168 :
1169 : // Make it large enough for caching and for cloning
1170 4 : if (!CheckFileSize(current.idxfname, expected_size, GA_Update))
1171 : {
1172 0 : CPLError(CE_Failure, CPLE_AppDefined,
1173 : "Can't extend the cache index file %s",
1174 : current.idxfname.c_str());
1175 0 : return nullptr;
1176 : }
1177 :
1178 : // Try opening it again in rw mode so we can read and write
1179 4 : mode = "r+b";
1180 4 : ifp.acc = GF_Write;
1181 4 : ifp.FP = VSIFOpenL(current.idxfname.c_str(), mode);
1182 :
1183 4 : if (nullptr == ifp.FP)
1184 : {
1185 0 : CPLError(CE_Failure, CPLE_AppDefined,
1186 : "GDAL MRF: Can't reopen cache index file %s\n",
1187 : full.idxfname.c_str());
1188 0 : return nullptr;
1189 : }
1190 4 : return ifp.FP;
1191 : }
1192 :
1193 : //
1194 : // Returns the dataset data file or null
1195 : // Data file is opened either in Read or Append mode, never in straight write
1196 : //
1197 7114 : VSILFILE *MRFDataset::DataFP()
1198 : {
1199 7114 : if (dfp.FP != nullptr)
1200 6860 : return dfp.FP;
1201 254 : const char *mode = "rb";
1202 254 : dfp.acc = GF_Read;
1203 :
1204 : // Open it for writing if updating or if caching
1205 254 : if (eAccess == GA_Update || !source.empty())
1206 : {
1207 152 : mode = "a+b";
1208 152 : dfp.acc = GF_Write;
1209 : }
1210 :
1211 254 : dfp.FP = VSIFOpenL(current.datfname, mode);
1212 254 : if (dfp.FP)
1213 254 : return dfp.FP;
1214 :
1215 : // It could be a caching MRF
1216 0 : if (source.empty())
1217 0 : goto io_error;
1218 :
1219 : // May be there but read only, remember that it was open that way
1220 0 : mode = "rb";
1221 0 : dfp.acc = GF_Read;
1222 0 : dfp.FP = VSIFOpenL(current.datfname, mode);
1223 0 : if (nullptr != dfp.FP)
1224 : {
1225 0 : CPLDebug("MRF_IO", "Opened %s RO mode %s\n", current.datfname.c_str(),
1226 : mode);
1227 0 : return dfp.FP;
1228 : }
1229 :
1230 0 : if (source.empty())
1231 0 : goto io_error;
1232 :
1233 : // caching, maybe the folder didn't exist
1234 0 : mkdir_r(current.datfname);
1235 0 : mode = "a+b";
1236 0 : dfp.acc = GF_Write;
1237 0 : dfp.FP = VSIFOpenL(current.datfname, mode);
1238 0 : if (dfp.FP)
1239 0 : return dfp.FP;
1240 :
1241 0 : io_error:
1242 0 : dfp.FP = nullptr;
1243 0 : CPLError(CE_Failure, CPLE_FileIO, "GDAL MRF: %s : %s", strerror(errno),
1244 : current.datfname.c_str());
1245 0 : return nullptr;
1246 : }
1247 :
1248 : // Builds an XML tree from the current MRF. If written to a file it becomes an
1249 : // MRF
1250 322 : CPLXMLNode *MRFDataset::BuildConfig()
1251 : {
1252 322 : CPLXMLNode *config = CPLCreateXMLNode(nullptr, CXT_Element, "MRF_META");
1253 :
1254 322 : if (!source.empty())
1255 : {
1256 : CPLXMLNode *psCachedSource =
1257 4 : CPLCreateXMLNode(config, CXT_Element, "CachedSource");
1258 : // Should wrap the string in CDATA, in case it is XML
1259 : CPLXMLNode *psSource =
1260 4 : CPLCreateXMLElementAndValue(psCachedSource, "Source", source);
1261 4 : if (clonedSource)
1262 0 : CPLSetXMLValue(psSource, "#clone", "true");
1263 : }
1264 :
1265 : // Use the full size
1266 322 : CPLXMLNode *raster = CPLCreateXMLNode(config, CXT_Element, "Raster");
1267 :
1268 : // Preserve the file names if not the default ones
1269 322 : if (full.datfname != getFname(GetFname(), ILComp_Ext[full.comp]))
1270 0 : CPLCreateXMLElementAndValue(raster, "DataFile", full.datfname.c_str());
1271 322 : if (full.idxfname != getFname(GetFname(), ".idx"))
1272 0 : CPLCreateXMLElementAndValue(raster, "IndexFile", full.idxfname.c_str());
1273 322 : if (spacing != 0)
1274 0 : XMLSetAttributeVal(raster, "Spacing", static_cast<double>(spacing),
1275 : "%.0f");
1276 :
1277 322 : XMLSetAttributeVal(raster, "Size", full.size, "%.0f");
1278 322 : XMLSetAttributeVal(raster, "PageSize", full.pagesize, "%.0f");
1279 :
1280 : #ifdef HAVE_PNG
1281 322 : if (full.comp != IL_PNG)
1282 : #endif
1283 : {
1284 182 : CPLCreateXMLElementAndValue(raster, "Compression", CompName(full.comp));
1285 : }
1286 :
1287 322 : if (full.dt != GDT_Byte)
1288 194 : CPLCreateXMLElementAndValue(raster, "DataType",
1289 : GDALGetDataTypeName(full.dt));
1290 :
1291 : // special photometric interpretation
1292 322 : if (!photometric.empty())
1293 4 : CPLCreateXMLElementAndValue(raster, "Photometric", photometric);
1294 :
1295 322 : if (!vNoData.empty() || !vMin.empty() || !vMax.empty())
1296 : {
1297 : CPLXMLNode *values =
1298 39 : CPLCreateXMLNode(raster, CXT_Element, "DataValues");
1299 39 : XMLSetAttributeVal(values, "NoData", vNoData);
1300 39 : XMLSetAttributeVal(values, "min", vMin);
1301 39 : XMLSetAttributeVal(values, "max", vMax);
1302 : }
1303 :
1304 : // palette, if we have one
1305 322 : if (poColorTable != nullptr)
1306 : {
1307 1 : const char *pfrmt = "%.0f";
1308 1 : CPLXMLNode *pal = CPLCreateXMLNode(raster, CXT_Element, "Palette");
1309 1 : int sz = poColorTable->GetColorEntryCount();
1310 1 : if (sz != 256)
1311 0 : XMLSetAttributeVal(pal, "Size", poColorTable->GetColorEntryCount());
1312 : // RGB or RGBA for now
1313 257 : for (int i = 0; i < sz; i++)
1314 : {
1315 256 : CPLXMLNode *entry = CPLCreateXMLNode(pal, CXT_Element, "Entry");
1316 256 : const GDALColorEntry *ent = poColorTable->GetColorEntry(i);
1317 : // No need to set the index, it is always from 0 no size-1
1318 256 : XMLSetAttributeVal(entry, "c1", ent->c1, pfrmt);
1319 256 : XMLSetAttributeVal(entry, "c2", ent->c2, pfrmt);
1320 256 : XMLSetAttributeVal(entry, "c3", ent->c3, pfrmt);
1321 256 : if (ent->c4 != 255)
1322 0 : XMLSetAttributeVal(entry, "c4", ent->c4, pfrmt);
1323 : }
1324 : }
1325 :
1326 322 : if (is_Endianness_Dependent(full.dt, full.comp)) // Need to set the order
1327 60 : CPLCreateXMLElementAndValue(raster, "NetByteOrder",
1328 60 : (full.nbo || NET_ORDER) ? "TRUE" : "FALSE");
1329 :
1330 322 : if (full.quality > 0 && full.quality != 85)
1331 12 : CPLCreateXMLElementAndValue(raster, "Quality",
1332 24 : CPLOPrintf("%d", full.quality));
1333 :
1334 : // Done with the raster node
1335 :
1336 322 : if (scale != 0.0)
1337 : {
1338 0 : CPLCreateXMLNode(config, CXT_Element, "Rsets");
1339 0 : CPLSetXMLValue(config, "Rsets.#model", "uniform");
1340 0 : CPLSetXMLValue(config, "Rsets.#scale", PrintDouble(scale));
1341 : }
1342 322 : CPLXMLNode *gtags = CPLCreateXMLNode(config, CXT_Element, "GeoTags");
1343 :
1344 : // Do we have an affine transform different from identity?
1345 : double gt[6];
1346 644 : if ((MRFDataset::GetGeoTransform(gt) == CE_None) &&
1347 322 : (gt[0] != 0 || gt[1] != 1 || gt[2] != 0 || gt[3] != 0 || gt[4] != 0 ||
1348 192 : gt[5] != 1))
1349 : {
1350 130 : double minx = gt[0];
1351 130 : double maxx = gt[1] * full.size.x + minx;
1352 130 : double maxy = gt[3];
1353 130 : double miny = gt[5] * full.size.y + maxy;
1354 130 : CPLXMLNode *bbox = CPLCreateXMLNode(gtags, CXT_Element, "BoundingBox");
1355 130 : XMLSetAttributeVal(bbox, "minx", minx);
1356 130 : XMLSetAttributeVal(bbox, "miny", miny);
1357 130 : XMLSetAttributeVal(bbox, "maxx", maxx);
1358 130 : XMLSetAttributeVal(bbox, "maxy", maxy);
1359 : }
1360 :
1361 322 : const char *pszProj = GetProjectionRef();
1362 322 : if (pszProj && (!EQUAL(pszProj, "")))
1363 131 : CPLCreateXMLElementAndValue(gtags, "Projection", pszProj);
1364 :
1365 322 : if (optlist.Count() != 0)
1366 : {
1367 48 : CPLString options;
1368 50 : for (int i = 0; i < optlist.size(); i++)
1369 : {
1370 26 : options += optlist[i];
1371 26 : options += ' ';
1372 : }
1373 24 : options.resize(options.size() - 1);
1374 24 : CPLCreateXMLElementAndValue(config, "Options", options);
1375 : }
1376 :
1377 322 : return config;
1378 : }
1379 :
1380 : /**
1381 : * \brief Populates the dataset variables from the XML definition
1382 : *
1383 : *
1384 : */
1385 350 : CPLErr MRFDataset::Initialize(CPLXMLNode *config)
1386 : {
1387 : // We only need a basic initialization here, usually gets overwritten by the
1388 : // image params
1389 350 : full.dt = GDT_Byte;
1390 350 : full.hasNoData = false;
1391 350 : full.NoDataValue = 0;
1392 350 : Quality = 85;
1393 :
1394 350 : CPLErr ret = Init_Raster(full, this, CPLGetXMLNode(config, "Raster"));
1395 350 : if (CE_None != ret)
1396 0 : return ret;
1397 :
1398 350 : hasVersions = on(CPLGetXMLValue(config, "Raster.versioned", "no"));
1399 350 : mp_safe = on(CPLGetXMLValue(config, "Raster.mp_safe", "no"));
1400 350 : spacing = atoi(CPLGetXMLValue(config, "Raster.Spacing", "0"));
1401 :
1402 : // The zslice defined in the file wins over the oo or the file argument
1403 350 : if (CPLGetXMLNode(config, "Raster.zslice"))
1404 0 : zslice = atoi(CPLGetXMLValue(config, "Raster.zslice", "0"));
1405 :
1406 350 : Quality = full.quality;
1407 :
1408 : // Bounding box
1409 350 : CPLXMLNode *bbox = CPLGetXMLNode(config, "GeoTags.BoundingBox");
1410 350 : if (nullptr != bbox)
1411 : {
1412 : double x0, x1, y0, y1;
1413 :
1414 151 : x0 = atof(CPLGetXMLValue(bbox, "minx", "0"));
1415 151 : x1 = atof(CPLGetXMLValue(bbox, "maxx", "1"));
1416 151 : y1 = atof(CPLGetXMLValue(bbox, "maxy", "1"));
1417 151 : y0 = atof(CPLGetXMLValue(bbox, "miny", "0"));
1418 :
1419 151 : GeoTransform[0] = x0;
1420 151 : GeoTransform[1] = (x1 - x0) / full.size.x;
1421 151 : GeoTransform[2] = 0;
1422 151 : GeoTransform[3] = y1;
1423 151 : GeoTransform[4] = 0;
1424 151 : GeoTransform[5] = (y0 - y1) / full.size.y;
1425 151 : bGeoTransformValid = TRUE;
1426 : }
1427 :
1428 : const char *pszRawProjFromXML =
1429 350 : CPLGetXMLValue(config, "GeoTags.Projection", "");
1430 350 : if (strlen(pszRawProjFromXML) != 0)
1431 152 : m_oSRS.SetFromUserInput(
1432 : pszRawProjFromXML,
1433 : OGRSpatialReference::SET_FROM_USER_INPUT_LIMITATIONS_get());
1434 :
1435 : // Copy the full size to current, data and index are not yet open
1436 350 : current = full;
1437 350 : if (current.size.z != 1)
1438 : {
1439 0 : SetMetadataItem("ZSIZE", CPLOPrintf("%d", current.size.z),
1440 0 : "IMAGE_STRUCTURE");
1441 0 : SetMetadataItem("ZSLICE", CPLOPrintf("%d", zslice), "IMAGE_STRUCTURE");
1442 : // Capture the zslice in pagesize.l
1443 0 : current.pagesize.l = zslice;
1444 : // Adjust offset for base image
1445 0 : if (full.size.z <= 0)
1446 : {
1447 0 : CPLError(CE_Failure, CPLE_AppDefined,
1448 : "GDAL MRF: Invalid Raster.z value");
1449 0 : return CE_Failure;
1450 : }
1451 0 : if (zslice >= full.size.z)
1452 : {
1453 0 : CPLError(CE_Failure, CPLE_AppDefined, "GDAL MRF: Invalid slice");
1454 0 : return CE_Failure;
1455 : }
1456 :
1457 0 : current.idxoffset +=
1458 0 : (current.pagecount.l / full.size.z) * zslice * sizeof(ILIdx);
1459 : }
1460 :
1461 : // Dataset metadata setup
1462 350 : SetMetadataItem("INTERLEAVE", OrderName(current.order), "IMAGE_STRUCTURE");
1463 350 : SetMetadataItem("COMPRESSION", CompName(current.comp), "IMAGE_STRUCTURE");
1464 :
1465 350 : if (is_Endianness_Dependent(current.dt, current.comp))
1466 66 : SetMetadataItem("NETBYTEORDER", current.nbo ? "TRUE" : "FALSE",
1467 66 : "IMAGE_STRUCTURE");
1468 :
1469 : // Open the files for the current image, either RW or RO
1470 350 : nRasterXSize = current.size.x;
1471 350 : nRasterYSize = current.size.y;
1472 350 : nBands = current.size.c;
1473 :
1474 350 : if (!nBands || !nRasterXSize || !nRasterYSize)
1475 : {
1476 0 : CPLError(CE_Failure, CPLE_AppDefined, "GDAL MRF: Image size missing");
1477 0 : return CE_Failure;
1478 : }
1479 :
1480 : // Pick up the source data image, if there is one
1481 350 : source = CPLGetXMLValue(config, "CachedSource.Source", "");
1482 : // Is it a clone?
1483 350 : clonedSource =
1484 350 : on(CPLGetXMLValue(config, "CachedSource.Source.clone", "no"));
1485 : // Pick up the options, if any
1486 : optlist.Assign(CSLTokenizeString2(
1487 : CPLGetXMLValue(config, "Options", nullptr), " \t\n\r",
1488 350 : CSLT_STRIPLEADSPACES | CSLT_STRIPENDSPACES));
1489 :
1490 : // Load all the options in the IMAGE_STRUCTURE metadata
1491 378 : for (int i = 0; i < optlist.Count(); i++)
1492 : {
1493 56 : CPLString s(optlist[i]);
1494 28 : size_t nSepPos = s.find_first_of(":=");
1495 28 : if (std::string::npos != nSepPos)
1496 : {
1497 28 : s.resize(nSepPos);
1498 28 : SetMetadataItem(s, optlist.FetchNameValue(s), "IMAGE_STRUCTURE");
1499 : }
1500 : }
1501 :
1502 : // We have the options, so we can call rasterband
1503 755 : for (int i = 1; i <= nBands; i++)
1504 : {
1505 : // The overviews are low resolution copies of the current one.
1506 458 : MRFRasterBand *band = newMRFRasterBand(this, current, i);
1507 458 : if (!band)
1508 53 : return CE_Failure;
1509 :
1510 405 : GDALColorInterp ci = GCI_Undefined;
1511 :
1512 : // Default color interpretation
1513 405 : switch (nBands)
1514 : {
1515 251 : case 1:
1516 : case 2:
1517 251 : ci = (i == 1) ? GCI_GrayIndex : GCI_AlphaBand;
1518 251 : break;
1519 144 : case 3:
1520 : case 4:
1521 144 : if (i < 3)
1522 94 : ci = (i == 1) ? GCI_RedBand : GCI_GreenBand;
1523 : else
1524 50 : ci = (i == 3) ? GCI_BlueBand : GCI_AlphaBand;
1525 : }
1526 :
1527 405 : if (GetColorTable())
1528 1 : ci = GCI_PaletteIndex;
1529 :
1530 : // Legacy, deprecated
1531 405 : if (optlist.FetchBoolean("MULTISPECTRAL", FALSE))
1532 0 : ci = GCI_Undefined;
1533 :
1534 : // New style
1535 405 : if (!photometric.empty())
1536 : {
1537 15 : if ("MULTISPECTRAL" == photometric)
1538 0 : ci = GCI_Undefined;
1539 : }
1540 :
1541 405 : band->SetColorInterpretation(ci);
1542 405 : SetBand(i, band);
1543 : }
1544 :
1545 297 : CPLXMLNode *rsets = CPLGetXMLNode(config, "Rsets");
1546 297 : if (nullptr != rsets && nullptr != rsets->psChild)
1547 : {
1548 : // We have rsets
1549 :
1550 : // Regular spaced overlays, until everything fits in a single tile
1551 32 : if (EQUAL("uniform", CPLGetXMLValue(rsets, "model", "uniform")))
1552 : {
1553 32 : scale = getXMLNum(rsets, "scale", 2.0);
1554 32 : if (scale <= 1)
1555 : {
1556 0 : CPLError(CE_Failure, CPLE_AppDefined,
1557 : "MRF: zoom factor less than unit not allowed");
1558 0 : return CE_Failure;
1559 : }
1560 : // Looks like there are overlays
1561 32 : AddOverviews(int(scale));
1562 : }
1563 : else
1564 : {
1565 0 : CPLError(CE_Failure, CPLE_AppDefined, "Unknown Rset definition");
1566 0 : return CE_Failure;
1567 : }
1568 : }
1569 :
1570 297 : idxSize = IdxSize(full, int(scale));
1571 297 : if (idxSize == 0)
1572 0 : return CE_Failure;
1573 :
1574 : // If not set by the bands, get a pageSizeBytes buffer
1575 297 : if (GetPBufferSize() == 0 && !SetPBuffer(current.pageSizeBytes))
1576 0 : return CE_Failure;
1577 :
1578 297 : if (hasVersions)
1579 : { // It has versions, but how many?
1580 5 : verCount = 0; // Assume it only has one
1581 : VSIStatBufL statb;
1582 : // If the file exists, compute the last version number
1583 5 : if (0 == VSIStatL(full.idxfname, &statb))
1584 5 : verCount = int(statb.st_size / idxSize - 1);
1585 : }
1586 :
1587 297 : return CE_None;
1588 : }
1589 :
1590 0 : static inline bool has_path(const CPLString &name)
1591 : {
1592 0 : return name.find_first_of("/\\") != string::npos;
1593 : }
1594 :
1595 : // Does name look like an absolute gdal file name?
1596 4 : static inline bool is_absolute(const CPLString &name)
1597 : {
1598 4 : return (name.find_first_of("/\\") == 0) // Starts with root
1599 3 : || (name.size() > 1 && name[1] == ':' &&
1600 0 : isalpha(static_cast<unsigned char>(
1601 0 : name[0]))) // Starts with drive letter
1602 7 : || (name[0] == '<'); // Maybe it is XML
1603 : }
1604 :
1605 : // Add the dirname of path to the beginning of name, if it is relative
1606 : // returns true if name was modified
1607 4 : static inline bool make_absolute(CPLString &name, const CPLString &path)
1608 : {
1609 4 : if (!is_absolute(path) && (path.find_first_of("/\\") != string::npos))
1610 : {
1611 3 : name = path.substr(0, path.find_last_of("/\\") + 1) + name;
1612 3 : return true;
1613 : }
1614 1 : return false;
1615 : }
1616 :
1617 : /**
1618 : *\brief Get the source dataset, open it if necessary
1619 : */
1620 5 : GDALDataset *MRFDataset::GetSrcDS()
1621 : {
1622 5 : if (poSrcDS)
1623 1 : return poSrcDS;
1624 4 : if (source.empty())
1625 0 : return nullptr;
1626 :
1627 : // Try open the source dataset as is
1628 4 : poSrcDS =
1629 4 : GDALDataset::FromHandle(GDALOpenShared(source.c_str(), GA_ReadOnly));
1630 :
1631 : // It the open fails, try again with the current dataset path prepended
1632 4 : if (!poSrcDS && make_absolute(source, fname))
1633 3 : poSrcDS = GDALDataset::FromHandle(
1634 : GDALOpenShared(source.c_str(), GA_ReadOnly));
1635 :
1636 4 : if (0 == source.find("<MRF_META>") && has_path(fname))
1637 : {
1638 : // MRF XML source, might need to patch the file names with the current
1639 : // one
1640 0 : MRFDataset *poMRFDS = dynamic_cast<MRFDataset *>(poSrcDS);
1641 0 : if (!poMRFDS)
1642 : {
1643 0 : delete poSrcDS;
1644 0 : poSrcDS = nullptr;
1645 0 : return nullptr;
1646 : }
1647 0 : make_absolute(poMRFDS->current.datfname, fname);
1648 0 : make_absolute(poMRFDS->current.idxfname, fname);
1649 : }
1650 4 : mp_safe = true; // Turn on MP safety
1651 4 : return poSrcDS;
1652 : }
1653 :
1654 : /**
1655 : *\brief Add or verify that all overlays exits
1656 : *
1657 : * @return size of the index file
1658 : */
1659 :
1660 62 : GIntBig MRFDataset::AddOverviews(int scaleIn)
1661 : {
1662 : // Fit the overlays
1663 62 : ILImage img = current;
1664 134 : while (1 != img.pagecount.x * img.pagecount.y)
1665 : {
1666 : // Adjust raster data for next level
1667 : // Adjust the offsets for indices left at this level
1668 72 : img.idxoffset += sizeof(ILIdx) * img.pagecount.l / img.size.z *
1669 72 : (img.size.z - zslice);
1670 :
1671 : // Next overview size
1672 72 : img.size.x = pcount(img.size.x, scaleIn);
1673 72 : img.size.y = pcount(img.size.y, scaleIn);
1674 72 : img.size.l++; // Increment the level
1675 72 : img.pagecount = pcount(img.size, img.pagesize);
1676 :
1677 : // And adjust the offset again, within next level
1678 72 : img.idxoffset += sizeof(ILIdx) * img.pagecount.l / img.size.z * zslice;
1679 :
1680 : // Create and register the overviews for each band
1681 144 : for (int i = 1; i <= nBands; i++)
1682 : {
1683 : MRFRasterBand *b =
1684 72 : reinterpret_cast<MRFRasterBand *>(GetRasterBand(i));
1685 72 : if (!(b->GetOverview(static_cast<int>(img.size.l) - 1)))
1686 72 : b->AddOverview(newMRFRasterBand(this, img, i,
1687 72 : static_cast<int>(img.size.l)));
1688 : }
1689 : }
1690 :
1691 : // Last adjustment, should be a single set of c and leftover z tiles
1692 62 : return img.idxoffset +
1693 124 : sizeof(ILIdx) * img.pagecount.l / img.size.z * (img.size.z - zslice);
1694 : }
1695 :
1696 : //
1697 : // set an entry if it doesn't already exist
1698 : //
1699 129 : static char **CSLAddIfMissing(char **papszList, const char *pszName,
1700 : const char *pszValue)
1701 : {
1702 129 : if (CSLFetchNameValue(papszList, pszName))
1703 9 : return papszList;
1704 120 : return CSLSetNameValue(papszList, pszName, pszValue);
1705 : }
1706 :
1707 : // CreateCopy implemented based on Create
1708 128 : GDALDataset *MRFDataset::CreateCopy(const char *pszFilename,
1709 : GDALDataset *poSrcDS, int /*bStrict*/,
1710 : char **papszOptions,
1711 : GDALProgressFunc pfnProgress,
1712 : void *pProgressData)
1713 : {
1714 256 : ILImage img;
1715 :
1716 128 : int x = poSrcDS->GetRasterXSize();
1717 128 : int y = poSrcDS->GetRasterYSize();
1718 128 : int nBands = poSrcDS->GetRasterCount();
1719 128 : if (nBands == 0)
1720 : {
1721 1 : CPLError(CE_Failure, CPLE_NotSupported, "nBands == 0 not supported");
1722 1 : return nullptr;
1723 : }
1724 127 : GDALRasterBand *poSrcBand1 = poSrcDS->GetRasterBand(1);
1725 :
1726 127 : GDALDataType dt = poSrcBand1->GetRasterDataType();
1727 : // Have our own options, to modify as we want
1728 127 : char **options = CSLDuplicate(papszOptions);
1729 :
1730 : const char *pszValue =
1731 127 : poSrcDS->GetMetadataItem("INTERLEAVE", "IMAGE_STRUCTURE");
1732 : options =
1733 127 : CSLAddIfMissing(options, "INTERLEAVE", pszValue ? pszValue : "PIXEL");
1734 : int xb, yb;
1735 127 : poSrcBand1->GetBlockSize(&xb, &yb);
1736 :
1737 : // Keep input block size if it exists and not explicitly set
1738 128 : if (CSLFetchNameValue(options, "BLOCKSIZE") == nullptr && xb != x &&
1739 1 : yb != y)
1740 : {
1741 1 : options = CSLAddIfMissing(options, "BLOCKXSIZE",
1742 2 : PrintDouble(xb, "%d").c_str());
1743 1 : options = CSLAddIfMissing(options, "BLOCKYSIZE",
1744 2 : PrintDouble(yb, "%d").c_str());
1745 : }
1746 :
1747 127 : MRFDataset *poDS = nullptr;
1748 : try
1749 : {
1750 : poDS = reinterpret_cast<MRFDataset *>(
1751 127 : Create(pszFilename, x, y, nBands, dt, options));
1752 :
1753 127 : if (poDS == nullptr || poDS->bCrystalized)
1754 12 : throw CPLOPrintf("MRF: Can't create %s", pszFilename);
1755 :
1756 115 : img = poDS->current; // Deal with the current one here
1757 :
1758 : // Copy data values from source
1759 264 : for (int i = 0; i < poDS->nBands; i++)
1760 : {
1761 : int bHas;
1762 : double dfData;
1763 149 : GDALRasterBand *srcBand = poSrcDS->GetRasterBand(i + 1);
1764 149 : GDALRasterBand *mBand = poDS->GetRasterBand(i + 1);
1765 149 : dfData = srcBand->GetNoDataValue(&bHas);
1766 149 : if (bHas)
1767 : {
1768 16 : poDS->vNoData.push_back(dfData);
1769 16 : mBand->SetNoDataValue(dfData);
1770 : }
1771 149 : dfData = srcBand->GetMinimum(&bHas);
1772 149 : if (bHas)
1773 13 : poDS->vMin.push_back(dfData);
1774 149 : dfData = srcBand->GetMaximum(&bHas);
1775 149 : if (bHas)
1776 13 : poDS->vMax.push_back(dfData);
1777 :
1778 : // Copy the band metadata, PAM will handle it
1779 149 : char **meta = srcBand->GetMetadata("IMAGE_STRUCTURE");
1780 149 : if (CSLCount(meta))
1781 7 : mBand->SetMetadata(meta, "IMAGE_STRUCTURE");
1782 :
1783 149 : meta = srcBand->GetMetadata();
1784 149 : if (CSLCount(meta))
1785 14 : mBand->SetMetadata(meta);
1786 : }
1787 :
1788 : // Geotags
1789 : double gt[6];
1790 115 : if (CE_None == poSrcDS->GetGeoTransform(gt))
1791 112 : poDS->SetGeoTransform(gt);
1792 :
1793 115 : const auto poSRS = poSrcDS->GetSpatialRef();
1794 115 : if (poSRS)
1795 111 : poDS->m_oSRS = *poSRS;
1796 :
1797 : // Color palette if we only have one band
1798 213 : if (1 == nBands &&
1799 98 : GCI_PaletteIndex == poSrcBand1->GetColorInterpretation())
1800 1 : poDS->SetColorTable(poSrcBand1->GetColorTable()->Clone());
1801 :
1802 : // Finally write the XML in the right file name
1803 115 : if (!poDS->Crystalize())
1804 10 : throw CPLString("MRF: Error creating files");
1805 : }
1806 22 : catch (const CPLString &e)
1807 : {
1808 22 : if (nullptr != poDS)
1809 10 : delete poDS;
1810 22 : CPLError(CE_Failure, CPLE_ObjectNull, "%s", e.c_str());
1811 22 : poDS = nullptr;
1812 : }
1813 :
1814 127 : CSLDestroy(options);
1815 127 : if (nullptr == poDS)
1816 22 : return nullptr;
1817 :
1818 105 : char **papszFileList = poDS->GetFileList();
1819 105 : poDS->oOvManager.Initialize(poDS, poDS->GetPhysicalFilename(),
1820 : papszFileList);
1821 105 : CSLDestroy(papszFileList);
1822 :
1823 105 : CPLErr err = CE_None;
1824 : // Have PAM copy all, but skip the mask
1825 105 : int nCloneFlags = GCIF_PAM_DEFAULT & ~GCIF_MASK;
1826 :
1827 : // If copy is disabled, we're done, we just created an empty MRF
1828 105 : if (!on(CSLFetchNameValue(papszOptions, "NOCOPY")))
1829 : {
1830 : // Use the GDAL copy call
1831 : // Need to flag the dataset as compressed (COMPRESSED=TRUE) to force
1832 : // block writes This might not be what we want, if the input and out
1833 : // order is truly separate
1834 103 : nCloneFlags |= GCIF_MASK; // We do copy the data, so copy the mask too
1835 : // if necessary
1836 103 : char **papszCWROptions = nullptr;
1837 : papszCWROptions =
1838 103 : CSLAddNameValue(papszCWROptions, "COMPRESSED", "TRUE");
1839 :
1840 : #ifdef HAVE_JPEG
1841 : // Use the Zen version of the CopyWholeRaster if input has a dataset
1842 : // mask and JPEGs are generated
1843 105 : if (GMF_PER_DATASET == poSrcDS->GetRasterBand(1)->GetMaskFlags() &&
1844 2 : (poDS->current.comp == IL_JPEG
1845 : #ifdef HAVE_PNG
1846 0 : || poDS->current.comp == IL_JPNG
1847 : #endif
1848 : ))
1849 : {
1850 2 : err = poDS->ZenCopy(poSrcDS, pfnProgress, pProgressData);
1851 2 : nCloneFlags ^= GCIF_MASK; // Turn the external mask off
1852 : }
1853 : else
1854 : #endif
1855 : {
1856 101 : err = GDALDatasetCopyWholeRaster(
1857 : (GDALDatasetH)poSrcDS, (GDALDatasetH)poDS, papszCWROptions,
1858 : pfnProgress, pProgressData);
1859 : }
1860 :
1861 103 : CSLDestroy(papszCWROptions);
1862 : }
1863 :
1864 105 : if (CE_None == err)
1865 105 : err = poDS->CloneInfo(poSrcDS, nCloneFlags);
1866 :
1867 105 : if (CE_Failure == err)
1868 : {
1869 0 : delete poDS;
1870 0 : return nullptr;
1871 : }
1872 :
1873 105 : return poDS;
1874 : }
1875 :
1876 : // Prepares the data so it is suitable for Zen JPEG encoding, based on input
1877 : // mask If bFBO is set, only the values of the first band are set non-zero when
1878 : // needed
1879 : template <typename T>
1880 2 : static void ZenFilter(T *buffer, GByte *mask, int nPixels, int nBands,
1881 : bool bFBO)
1882 : {
1883 524290 : for (int i = 0; i < nPixels; i++)
1884 : {
1885 524288 : if (mask[i] == 0)
1886 : { // enforce zero values
1887 516096 : for (int b = 0; b < nBands; b++)
1888 387072 : buffer[nBands * i + b] = 0;
1889 : }
1890 : else
1891 : { // enforce non-zero
1892 395264 : if (bFBO)
1893 : { // First band only
1894 197632 : bool f = true;
1895 790528 : for (int b = 0; b < nBands; b++)
1896 : {
1897 592896 : if (0 == buffer[nBands * i + b])
1898 : {
1899 0 : f = false;
1900 0 : break;
1901 : }
1902 : }
1903 197632 : if (f)
1904 197632 : buffer[nBands * i] = 1;
1905 : }
1906 : else
1907 : { // Every band
1908 790528 : for (int b = 0; b < nBands; b++)
1909 592896 : if (0 == buffer[nBands * i + b])
1910 0 : buffer[nBands * i + b] = 1;
1911 : }
1912 : }
1913 : }
1914 2 : }
1915 :
1916 : // Custom CopyWholeRaster for Zen JPEG, called when the input has a PER_DATASET
1917 : // mask Works like GDALDatasetCopyWholeRaster, but it does filter the input data
1918 : // based on the mask
1919 : //
1920 2 : CPLErr MRFDataset::ZenCopy(GDALDataset *poSrc, GDALProgressFunc pfnProgress,
1921 : void *pProgressData)
1922 : {
1923 2 : VALIDATE_POINTER1(poSrc, "MRF:ZenCopy", CE_Failure);
1924 :
1925 2 : if (!pfnProgress)
1926 0 : pfnProgress = GDALDummyProgress;
1927 :
1928 : /* -------------------------------------------------------------------- */
1929 : /* Confirm the datasets match in size and band counts. */
1930 : /* -------------------------------------------------------------------- */
1931 2 : const int nXSize = GetRasterXSize();
1932 2 : const int nYSize = GetRasterYSize();
1933 2 : const int nBandCount = GetRasterCount();
1934 :
1935 2 : if (poSrc->GetRasterXSize() != nXSize ||
1936 4 : poSrc->GetRasterYSize() != nYSize ||
1937 2 : poSrc->GetRasterCount() != nBandCount)
1938 : {
1939 0 : CPLError(CE_Failure, CPLE_AppDefined,
1940 : "Input and output dataset sizes or band counts do not\n"
1941 : "match in GDALDatasetCopyWholeRaster()");
1942 0 : return CE_Failure;
1943 : }
1944 :
1945 : /* -------------------------------------------------------------------- */
1946 : /* Get our prototype band, and assume the others are similarly */
1947 : /* configured. Also get the per_dataset mask */
1948 : /* -------------------------------------------------------------------- */
1949 2 : GDALRasterBand *poSrcPrototypeBand = poSrc->GetRasterBand(1);
1950 2 : GDALRasterBand *poDstPrototypeBand = GetRasterBand(1);
1951 2 : GDALRasterBand *poSrcMask = poSrcPrototypeBand->GetMaskBand();
1952 :
1953 2 : const int nPageXSize = current.pagesize.x;
1954 2 : const int nPageYSize = current.pagesize.y;
1955 2 : const double nTotalBlocks =
1956 2 : static_cast<double>(DIV_ROUND_UP(nYSize, nPageYSize)) *
1957 2 : static_cast<double>(DIV_ROUND_UP(nXSize, nPageXSize));
1958 2 : const GDALDataType eDT = poDstPrototypeBand->GetRasterDataType();
1959 :
1960 : // All the bands are done per block
1961 : // this flag tells us to apply the Zen filter to the first band only
1962 2 : const bool bFirstBandOnly = (current.order == IL_Interleaved);
1963 :
1964 2 : if (!pfnProgress(0.0, nullptr, pProgressData))
1965 : {
1966 0 : CPLError(CE_Failure, CPLE_UserInterrupt,
1967 : "User terminated CreateCopy()");
1968 0 : return CE_Failure;
1969 : }
1970 :
1971 2 : const int nPixelCount = nPageXSize * nPageYSize;
1972 2 : const int dts = GDALGetDataTypeSizeBytes(eDT);
1973 2 : void *buffer = VSI_MALLOC3_VERBOSE(nPixelCount, nBandCount, dts);
1974 2 : GByte *buffer_mask = nullptr;
1975 2 : if (buffer)
1976 : buffer_mask =
1977 2 : reinterpret_cast<GByte *>(VSI_MALLOC_VERBOSE(nPixelCount));
1978 :
1979 2 : if (!buffer || !buffer_mask)
1980 : {
1981 : // Just in case buffers did get allocated
1982 0 : CPLFree(buffer);
1983 0 : CPLFree(buffer_mask);
1984 0 : CPLError(CE_Failure, CPLE_OutOfMemory, "Can't allocate copy buffer");
1985 0 : return CE_Failure;
1986 : }
1987 :
1988 2 : int nBlocksDone = 0;
1989 2 : CPLErr eErr = CE_None;
1990 : // Advise the source that a complete read will be done
1991 2 : poSrc->AdviseRead(0, 0, nXSize, nYSize, nXSize, nYSize, eDT, nBandCount,
1992 2 : nullptr, nullptr);
1993 :
1994 : // For every block, break on error
1995 4 : for (int row = 0; row < nYSize && eErr == CE_None; row += nPageYSize)
1996 : {
1997 2 : int nRows = std::min(nPageYSize, nYSize - row);
1998 4 : for (int col = 0; col < nXSize && eErr == CE_None; col += nPageXSize)
1999 : {
2000 2 : int nCols = std::min(nPageXSize, nXSize - col);
2001 :
2002 : // Report
2003 2 : if (eErr == CE_None && !pfnProgress(nBlocksDone++ / nTotalBlocks,
2004 : nullptr, pProgressData))
2005 : {
2006 0 : eErr = CE_Failure;
2007 0 : CPLError(CE_Failure, CPLE_UserInterrupt,
2008 : "User terminated CreateCopy()");
2009 0 : break;
2010 : }
2011 :
2012 : // Get the data mask as byte
2013 2 : eErr = poSrcMask->RasterIO(GF_Read, col, row, nCols, nRows,
2014 : buffer_mask, nCols, nRows, GDT_Byte, 0,
2015 : 0, nullptr);
2016 :
2017 2 : if (eErr != CE_None)
2018 0 : break;
2019 :
2020 : // If there is no data at all, skip this block
2021 2 : if (MatchCount(buffer_mask, nPixelCount, static_cast<GByte>(0)) ==
2022 : nPixelCount)
2023 0 : continue;
2024 :
2025 : // get the data in the buffer, interleaved
2026 4 : eErr = poSrc->RasterIO(
2027 : GF_Read, col, row, nCols, nRows, buffer, nCols, nRows, eDT,
2028 2 : nBandCount, nullptr, static_cast<GSpacing>(nBands) * dts,
2029 2 : static_cast<GSpacing>(nBands) * dts * nCols, dts, nullptr);
2030 :
2031 2 : if (eErr != CE_None)
2032 0 : break;
2033 :
2034 : // This is JPEG, only 8 and 12(16) bits unsigned integer types are
2035 : // valid
2036 2 : switch (eDT)
2037 : {
2038 2 : case GDT_Byte:
2039 2 : ZenFilter(reinterpret_cast<GByte *>(buffer), buffer_mask,
2040 : nPixelCount, nBandCount, bFirstBandOnly);
2041 2 : break;
2042 0 : case GDT_UInt16:
2043 0 : ZenFilter(reinterpret_cast<GUInt16 *>(buffer), buffer_mask,
2044 : nPixelCount, nBandCount, bFirstBandOnly);
2045 0 : break;
2046 0 : default:
2047 0 : CPLError(CE_Failure, CPLE_AppDefined,
2048 : "Unsupported data type for Zen filter");
2049 0 : eErr = CE_Failure;
2050 0 : break;
2051 : }
2052 :
2053 : // Write
2054 2 : if (eErr == CE_None)
2055 2 : eErr = RasterIO(
2056 : GF_Write, col, row, nCols, nRows, buffer, nCols, nRows, eDT,
2057 2 : nBandCount, nullptr, static_cast<GSpacing>(nBands) * dts,
2058 2 : static_cast<GSpacing>(nBands) * dts * nCols, dts, nullptr);
2059 :
2060 : } // Columns
2061 2 : if (eErr != CE_None)
2062 0 : break;
2063 :
2064 : } // Rows
2065 :
2066 : // Cleanup
2067 2 : CPLFree(buffer);
2068 2 : CPLFree(buffer_mask);
2069 :
2070 : // Final report
2071 2 : if (eErr == CE_None && !pfnProgress(1.0, nullptr, pProgressData))
2072 : {
2073 0 : eErr = CE_Failure;
2074 0 : CPLError(CE_Failure, CPLE_UserInterrupt,
2075 : "User terminated CreateCopy()");
2076 : }
2077 :
2078 2 : return eErr;
2079 : }
2080 :
2081 : // Apply open options to the current dataset
2082 : // Called before the configuration is read
2083 194 : void MRFDataset::ProcessOpenOptions(char **papszOptions)
2084 : {
2085 388 : CPLStringList opt(papszOptions, FALSE);
2086 194 : no_errors = opt.FetchBoolean("NOERRORS", FALSE);
2087 194 : const char *val = opt.FetchNameValue("ZSLICE");
2088 194 : if (val)
2089 0 : zslice = atoi(val);
2090 194 : }
2091 :
2092 : // Apply create options to the current dataset, only valid during creation
2093 156 : void MRFDataset::ProcessCreateOptions(char **papszOptions)
2094 : {
2095 156 : assert(!bCrystalized);
2096 312 : CPLStringList opt(papszOptions, FALSE);
2097 156 : ILImage &img(full);
2098 :
2099 156 : const char *val = opt.FetchNameValue("COMPRESS");
2100 156 : if (val && IL_ERR_COMP == (img.comp = CompToken(val)))
2101 0 : throw CPLString("GDAL MRF: Error setting compression");
2102 :
2103 156 : val = opt.FetchNameValue("INTERLEAVE");
2104 156 : if (val && IL_ERR_ORD == (img.order = OrderToken(val)))
2105 0 : throw CPLString("GDAL MRF: Error setting interleave");
2106 :
2107 156 : val = opt.FetchNameValue("QUALITY");
2108 156 : if (val)
2109 6 : img.quality = atoi(val);
2110 :
2111 156 : val = opt.FetchNameValue("ZSIZE");
2112 156 : if (val)
2113 0 : img.size.z = atoi(val);
2114 :
2115 156 : val = opt.FetchNameValue("BLOCKXSIZE");
2116 156 : if (val)
2117 1 : img.pagesize.x = atoi(val);
2118 :
2119 156 : val = opt.FetchNameValue("BLOCKYSIZE");
2120 156 : if (val)
2121 1 : img.pagesize.y = atoi(val);
2122 :
2123 156 : val = opt.FetchNameValue("BLOCKSIZE");
2124 156 : if (val)
2125 30 : img.pagesize.x = img.pagesize.y = atoi(val);
2126 :
2127 156 : img.nbo = opt.FetchBoolean("NETBYTEORDER", FALSE) != FALSE;
2128 :
2129 156 : val = opt.FetchNameValue("CACHEDSOURCE");
2130 156 : if (val)
2131 : {
2132 2 : source = val;
2133 2 : nocopy = opt.FetchBoolean("NOCOPY", FALSE);
2134 : }
2135 :
2136 156 : val = opt.FetchNameValue("UNIFORM_SCALE");
2137 156 : if (val)
2138 0 : scale = atoi(val);
2139 :
2140 156 : val = opt.FetchNameValue("PHOTOMETRIC");
2141 156 : if (val)
2142 2 : photometric = val;
2143 :
2144 156 : val = opt.FetchNameValue("DATANAME");
2145 156 : if (val)
2146 0 : img.datfname = val;
2147 :
2148 156 : val = opt.FetchNameValue("INDEXNAME");
2149 156 : if (val)
2150 0 : img.idxfname = val;
2151 :
2152 156 : val = opt.FetchNameValue("SPACING");
2153 156 : if (val)
2154 0 : spacing = atoi(val);
2155 :
2156 : optlist.Assign(
2157 : CSLTokenizeString2(opt.FetchNameValue("OPTIONS"), " \t\n\r",
2158 156 : CSLT_STRIPLEADSPACES | CSLT_STRIPENDSPACES));
2159 :
2160 : // General Fixups
2161 156 : if (img.order == IL_Interleaved)
2162 65 : img.pagesize.c = img.size.c;
2163 :
2164 : // Compression dependent fixups
2165 156 : }
2166 :
2167 : /**
2168 : *\brief Create an MRF dataset, some settings can be changed later
2169 : * papszOptions might be anything that an MRF might take
2170 : * Still missing are the georeference ...
2171 : *
2172 : */
2173 :
2174 160 : GDALDataset *MRFDataset::Create(const char *pszName, int nXSize, int nYSize,
2175 : int nBandsIn, GDALDataType eType,
2176 : char **papszOptions)
2177 : {
2178 160 : if (nBandsIn == 0)
2179 : {
2180 1 : CPLError(CE_Failure, CPLE_NotSupported, "No bands defined");
2181 1 : return nullptr;
2182 : }
2183 :
2184 159 : MRFDataset *poDS = new MRFDataset();
2185 159 : CPLErr err = CE_None;
2186 159 : poDS->fname = pszName;
2187 159 : poDS->nBands = nBandsIn;
2188 :
2189 : // Don't know what to do with these in this call
2190 : // int level = -1;
2191 : // int version = 0;
2192 :
2193 159 : size_t pos = poDS->fname.find(":MRF:");
2194 159 : if (string::npos != pos)
2195 : { // Tokenize and pick known options
2196 0 : vector<string> tokens;
2197 0 : stringSplit(tokens, poDS->fname, pos + 5, ':');
2198 : // level = getnum(tokens, 'L', -1);
2199 : // version = getnum(tokens, 'V', 0);
2200 0 : poDS->zslice = getnum(tokens, 'Z', 0);
2201 0 : poDS->fname.resize(pos); // Cut the ornamentations
2202 : }
2203 :
2204 : // Try creating the mrf file early, to avoid failing on Crystalize later
2205 159 : if (!STARTS_WITH(poDS->fname.c_str(), "<MRF_META>"))
2206 : {
2207 : // Try opening it first, even though we still clobber it later
2208 159 : VSILFILE *mainfile = VSIFOpenL(poDS->fname.c_str(), "r+b");
2209 159 : if (!mainfile)
2210 : { // Then try creating it
2211 113 : mainfile = VSIFOpenL(poDS->fname.c_str(), "w+b");
2212 113 : if (!mainfile)
2213 : {
2214 3 : CPLError(CE_Failure, CPLE_OpenFailed,
2215 : "MRF: Can't open %s for writing", poDS->fname.c_str());
2216 3 : delete poDS;
2217 3 : return nullptr;
2218 : }
2219 : }
2220 156 : VSIFCloseL(mainfile);
2221 : }
2222 :
2223 : // Use the full, set some initial parameters
2224 156 : ILImage &img = poDS->full;
2225 156 : img.size = ILSize(nXSize, nYSize, 1, nBandsIn);
2226 : #ifdef HAVE_PNG
2227 156 : img.comp = IL_PNG;
2228 : #else
2229 : img.comp = IL_NONE;
2230 : #endif
2231 156 : img.order = (nBandsIn < 5) ? IL_Interleaved : IL_Separate;
2232 156 : img.pagesize = ILSize(512, 512, 1, 1);
2233 156 : img.quality = 85;
2234 156 : img.dt = eType;
2235 156 : img.dataoffset = 0;
2236 156 : img.idxoffset = 0;
2237 156 : img.hasNoData = false;
2238 156 : img.nbo = false;
2239 :
2240 : // Set the guard that tells us it needs saving before IO can take place
2241 156 : poDS->bCrystalized = FALSE;
2242 :
2243 : // Process the options, anything that an MRF might take
2244 :
2245 : try
2246 : {
2247 : // Adjust the dataset and the full image
2248 156 : poDS->ProcessCreateOptions(papszOptions);
2249 :
2250 : // Set default file names
2251 156 : if (img.datfname.empty())
2252 156 : img.datfname = getFname(poDS->GetFname(), ILComp_Ext[img.comp]);
2253 156 : if (img.idxfname.empty())
2254 156 : img.idxfname = getFname(poDS->GetFname(), ".idx");
2255 :
2256 156 : poDS->eAccess = GA_Update;
2257 : }
2258 :
2259 0 : catch (const CPLString &e)
2260 : {
2261 0 : CPLError(CE_Failure, CPLE_OpenFailed, "%s", e.c_str());
2262 0 : delete poDS;
2263 0 : return nullptr;
2264 : }
2265 :
2266 156 : poDS->current = poDS->full;
2267 156 : poDS->SetDescription(poDS->GetFname());
2268 :
2269 : // Build a MRF XML and initialize from it, this creates the bands
2270 156 : CPLXMLNode *config = poDS->BuildConfig();
2271 156 : err = poDS->Initialize(config);
2272 156 : CPLDestroyXMLNode(config);
2273 :
2274 156 : if (CPLE_None != err)
2275 : {
2276 31 : delete poDS;
2277 31 : return nullptr;
2278 : }
2279 :
2280 : // If not set by the band, get a pageSizeBytes buffer
2281 125 : if (poDS->GetPBufferSize() == 0 &&
2282 0 : !poDS->SetPBuffer(poDS->current.pageSizeBytes))
2283 : {
2284 0 : delete poDS;
2285 0 : return nullptr;
2286 : }
2287 :
2288 : // Tell PAM what our real file name is, to help it find the aux.xml
2289 125 : poDS->SetPhysicalFilename(poDS->GetFname());
2290 125 : return poDS;
2291 : }
2292 :
2293 166 : int MRFDataset::Crystalize()
2294 : {
2295 166 : if (bCrystalized || eAccess != GA_Update)
2296 : {
2297 0 : bCrystalized = TRUE;
2298 0 : return TRUE;
2299 : }
2300 :
2301 : // No need to write to disk if there is no filename. This is a
2302 : // memory only dataset.
2303 332 : if (strlen(GetDescription()) == 0 ||
2304 166 : EQUALN(GetDescription(), "<MRF_META>", 10))
2305 : {
2306 0 : bCrystalized = TRUE;
2307 0 : return TRUE;
2308 : }
2309 :
2310 166 : CPLXMLNode *config = BuildConfig();
2311 166 : if (!WriteConfig(config))
2312 20 : return FALSE;
2313 146 : CPLDestroyXMLNode(config);
2314 146 : if (!nocopy && (!IdxFP() || !DataFP()))
2315 0 : return FALSE;
2316 146 : bCrystalized = TRUE;
2317 146 : return TRUE;
2318 : }
2319 :
2320 : // Copy the first index at the end of the file and bump the version count
2321 1 : CPLErr MRFDataset::AddVersion()
2322 : {
2323 1 : VSILFILE *l_ifp = IdxFP();
2324 1 : void *tbuff = CPLMalloc(static_cast<size_t>(idxSize));
2325 1 : VSIFSeekL(l_ifp, 0, SEEK_SET);
2326 1 : VSIFReadL(tbuff, 1, static_cast<size_t>(idxSize), l_ifp);
2327 1 : verCount++; // The one we write
2328 1 : VSIFSeekL(l_ifp, idxSize * verCount,
2329 : SEEK_SET); // At the end, this can mess things up royally
2330 1 : VSIFWriteL(tbuff, 1, static_cast<size_t>(idxSize), l_ifp);
2331 1 : CPLFree(tbuff);
2332 1 : return CE_None;
2333 : }
2334 :
2335 : //
2336 : // Write a tile at the end of the data file
2337 : // If buff and size are zero, it is equivalent to erasing the tile
2338 : // If only size is zero, it is a special empty tile,
2339 : // when used for caching, offset should be 1
2340 : //
2341 : // To make it multi-processor safe, open the file in append mode
2342 : // and verify after write
2343 : //
2344 5079 : CPLErr MRFDataset::WriteTile(void *buff, GUIntBig infooffset, GUIntBig size)
2345 : {
2346 5079 : CPLErr ret = CE_None;
2347 5079 : ILIdx tinfo = {0, 0};
2348 :
2349 5079 : VSILFILE *l_dfp = DataFP();
2350 5079 : VSILFILE *l_ifp = IdxFP();
2351 :
2352 : // Verify buffer
2353 10158 : std::vector<GByte> tbuff;
2354 :
2355 5079 : if (l_ifp == nullptr || l_dfp == nullptr)
2356 0 : return CE_Failure;
2357 :
2358 : // Flag that versioned access requires a write even if empty
2359 5079 : int new_tile = false;
2360 : // If it has versions, might need to start a new one
2361 5079 : if (hasVersions)
2362 : {
2363 1 : int new_version = false; // Assume no need to build new version
2364 :
2365 : // Read the current tile info
2366 1 : VSIFSeekL(l_ifp, infooffset, SEEK_SET);
2367 1 : VSIFReadL(&tinfo, 1, sizeof(ILIdx), l_ifp);
2368 :
2369 1 : if (verCount == 0)
2370 1 : new_version = true; // No previous yet, might create a new version
2371 : else
2372 : { // We need at least two versions before we can test for changes
2373 0 : ILIdx prevtinfo = {0, 0};
2374 :
2375 : // Read the previous one
2376 0 : VSIFSeekL(l_ifp, infooffset + verCount * idxSize, SEEK_SET);
2377 0 : VSIFReadL(&prevtinfo, 1, sizeof(ILIdx), l_ifp);
2378 :
2379 : // current and previous tiles are different, might create version
2380 0 : if (tinfo.size != prevtinfo.size ||
2381 0 : tinfo.offset != prevtinfo.offset)
2382 0 : new_version = true;
2383 : }
2384 :
2385 : // tinfo contains the current info or 0,0
2386 1 : if (tinfo.size == GIntBig(net64(size)))
2387 : { // Might be identical
2388 0 : if (size != 0)
2389 : {
2390 : // Use the temporary buffer
2391 0 : tbuff.resize(static_cast<size_t>(size));
2392 0 : VSIFSeekL(l_dfp, infooffset, SEEK_SET);
2393 0 : VSIFReadL(tbuff.data(), 1, tbuff.size(), l_dfp);
2394 : // Need to write it if not the same
2395 0 : new_tile = !std::equal(tbuff.begin(), tbuff.end(),
2396 : static_cast<GByte *>(buff));
2397 0 : tbuff.clear();
2398 : }
2399 : else
2400 : {
2401 : // Writing a null tile on top of a null tile, does it count?
2402 0 : if (tinfo.offset != GIntBig(net64(GUIntBig(buff))))
2403 0 : new_tile = true;
2404 : }
2405 : }
2406 : else
2407 : {
2408 1 : new_tile = true; // Need to write it because it is different
2409 1 : if (verCount == 0 && tinfo.size == 0)
2410 0 : new_version = false; // Don't create a version if current is
2411 : // empty and there is no previous
2412 : }
2413 :
2414 1 : if (!new_tile)
2415 0 : return CE_None; // No reason to write
2416 :
2417 : // Do we need to start a new version before writing the tile?
2418 1 : if (new_version)
2419 1 : AddVersion();
2420 : }
2421 :
2422 5079 : bool same = true;
2423 5079 : if (size)
2424 0 : do
2425 : {
2426 : // start of critical MP section
2427 5078 : VSIFSeekL(l_dfp, 0, SEEK_END);
2428 5078 : GUIntBig offset = VSIFTellL(l_dfp) + spacing;
2429 :
2430 : // Spacing should be 0 in MP safe mode, this doesn't have much of
2431 : // effect Use the existing data, spacing content is not guaranteed
2432 5078 : for (GUIntBig pending = spacing; pending != 0;
2433 0 : pending -= std::min(pending, size))
2434 0 : VSIFWriteL(buff, 1,
2435 0 : static_cast<size_t>(std::min(pending, size)),
2436 : l_dfp); // Usually only once
2437 :
2438 5078 : if (static_cast<size_t>(size) !=
2439 5078 : VSIFWriteL(buff, 1, static_cast<size_t>(size), l_dfp))
2440 0 : ret = CE_Failure;
2441 : // End of critical section
2442 :
2443 5078 : tinfo.offset = net64(offset);
2444 : //
2445 : // For MP ops, check that we can read the same content, otherwise
2446 : // try again This makes the caching MRF MP safe on file systems that
2447 : // implement append mode fully, without using explicit locks
2448 : //
2449 5078 : if (CE_None == ret && mp_safe)
2450 : { // readback and check
2451 3 : if (tbuff.size() < size)
2452 3 : tbuff.resize(static_cast<size_t>(size));
2453 3 : VSIFSeekL(l_dfp, offset, SEEK_SET);
2454 3 : VSIFReadL(tbuff.data(), 1, tbuff.size(), l_dfp);
2455 3 : same = std::equal(tbuff.begin(), tbuff.end(),
2456 : static_cast<GByte *>(buff));
2457 : }
2458 5078 : } while (CE_None == ret && mp_safe && !same);
2459 :
2460 5079 : if (CE_None != ret)
2461 : {
2462 0 : CPLError(CE_Failure, CPLE_AppDefined, "MRF: Tile write failed");
2463 0 : return ret;
2464 : }
2465 :
2466 : // Convert index to net format, offset is set already
2467 5079 : tinfo.size = net64(size);
2468 : // Do nothing if the tile is empty and the file record is also empty
2469 5079 : if (!new_tile && 0 == size && nullptr == buff)
2470 : {
2471 0 : VSIFSeekL(l_ifp, infooffset, SEEK_SET);
2472 0 : VSIFReadL(&tinfo, 1, sizeof(ILIdx), l_ifp);
2473 0 : if (0 == tinfo.offset && 0 == tinfo.size)
2474 0 : return ret;
2475 : }
2476 :
2477 : // Special case, any non-zero offset will do
2478 5079 : if (nullptr != buff && 0 == size)
2479 0 : tinfo.offset = ~GUIntBig(0);
2480 :
2481 5079 : VSIFSeekL(l_ifp, infooffset, SEEK_SET);
2482 5079 : if (sizeof(tinfo) != VSIFWriteL(&tinfo, 1, sizeof(tinfo), l_ifp))
2483 : {
2484 0 : CPLError(CE_Failure, CPLE_AppDefined, "MRF: Index write failed");
2485 0 : ret = CE_Failure;
2486 : }
2487 :
2488 5079 : return ret;
2489 : }
2490 :
2491 121 : CPLErr MRFDataset::SetGeoTransform(double *gt)
2492 : {
2493 121 : if (GetAccess() != GA_Update || bCrystalized)
2494 : {
2495 0 : CPLError(CE_Failure, CPLE_NotSupported,
2496 : "SetGeoTransform only works during Create call");
2497 0 : return CE_Failure;
2498 : }
2499 121 : memcpy(GeoTransform, gt, 6 * sizeof(double));
2500 121 : bGeoTransformValid = TRUE;
2501 121 : return CE_None;
2502 : }
2503 :
2504 16 : bool MRFDataset::IsSingleTile()
2505 : {
2506 16 : if (current.pagecount.l != 1 || !source.empty() || nullptr == DataFP())
2507 4 : return FALSE;
2508 12 : return 0 == reinterpret_cast<MRFRasterBand *>(GetRasterBand(1))
2509 12 : ->GetOverviewCount();
2510 : }
2511 :
2512 : /*
2513 : * Returns 0,1,0,0,0,1 even if it was not set
2514 : */
2515 436 : CPLErr MRFDataset::GetGeoTransform(double *gt)
2516 : {
2517 436 : memcpy(gt, GeoTransform, 6 * sizeof(double));
2518 436 : if (GetMetadata("RPC") || GetGCPCount())
2519 0 : bGeoTransformValid = FALSE;
2520 436 : if (!bGeoTransformValid)
2521 0 : return CE_Failure;
2522 436 : return CE_None;
2523 : }
2524 :
2525 : /**
2526 : *\brief Read a tile index
2527 : *
2528 : * It handles the non-existent index case, for no compression
2529 : * The bias is non-zero only when the cloned index is read
2530 : */
2531 :
2532 2014 : CPLErr MRFDataset::ReadTileIdx(ILIdx &tinfo, const ILSize &pos,
2533 : const ILImage &img, const GIntBig bias)
2534 : {
2535 2014 : VSILFILE *l_ifp = IdxFP();
2536 :
2537 : // Initialize the tinfo structure, in case the files are missing
2538 2014 : if (missing)
2539 0 : return CE_None;
2540 :
2541 2014 : GIntBig offset = bias + IdxOffset(pos, img);
2542 2014 : if (l_ifp == nullptr && img.comp == IL_NONE)
2543 : {
2544 0 : tinfo.size = current.pageSizeBytes;
2545 0 : tinfo.offset = offset * tinfo.size;
2546 0 : return CE_None;
2547 : }
2548 :
2549 2014 : if (l_ifp == nullptr && IsSingleTile())
2550 : {
2551 6 : tinfo.offset = 0;
2552 6 : VSILFILE *l_dfp = DataFP(); // IsSingleTile() checks that fp is valid
2553 6 : VSIFSeekL(l_dfp, 0, SEEK_END);
2554 6 : tinfo.size = VSIFTellL(l_dfp);
2555 :
2556 : // It should be less than the pagebuffer
2557 6 : tinfo.size = std::min(tinfo.size, static_cast<GIntBig>(pbsize));
2558 6 : return CE_None;
2559 : }
2560 :
2561 2008 : if (l_ifp == nullptr)
2562 : {
2563 0 : CPLError(CE_Failure, CPLE_FileIO, "Can't open index file");
2564 0 : return CE_Failure;
2565 : }
2566 :
2567 2008 : VSIFSeekL(l_ifp, offset, SEEK_SET);
2568 2008 : if (1 != VSIFReadL(&tinfo, sizeof(ILIdx), 1, l_ifp))
2569 0 : return CE_Failure;
2570 : // Convert them to native form
2571 2008 : tinfo.offset = net64(tinfo.offset);
2572 2008 : tinfo.size = net64(tinfo.size);
2573 :
2574 2008 : if (0 == bias || 0 != tinfo.size || 0 != tinfo.offset)
2575 2007 : return CE_None;
2576 :
2577 : // zero size and zero offset in sourced index means that this portion is
2578 : // un-initialized
2579 :
2580 : // Should be cloned and the offset within the cloned index
2581 1 : offset -= bias;
2582 1 : assert(offset < bias);
2583 1 : assert(clonedSource);
2584 :
2585 : // Read this block from the remote index, prepare it and store it in the
2586 : // right place The block size in bytes, should be a multiple of 16, to have
2587 : // full index entries
2588 1 : const int CPYSZ = 32768;
2589 : // Adjust offset to the start of the block
2590 1 : offset = (offset / CPYSZ) * CPYSZ;
2591 1 : GIntBig size = std::min(size_t(CPYSZ), size_t(bias - offset));
2592 1 : size /= sizeof(ILIdx); // In records
2593 2 : vector<ILIdx> buf(static_cast<size_t>(size));
2594 1 : ILIdx *buffer = &buf[0]; // Buffer to copy the source to the clone index
2595 :
2596 : // Fetch the data from the cloned index
2597 1 : MRFDataset *pSrc = static_cast<MRFDataset *>(GetSrcDS());
2598 1 : if (nullptr == pSrc)
2599 : {
2600 0 : CPLError(CE_Failure, CPLE_FileIO, "Can't open cloned source index");
2601 0 : return CE_Failure; // Source reported the error
2602 : }
2603 :
2604 1 : VSILFILE *srcidx = pSrc->IdxFP();
2605 1 : if (nullptr == srcidx)
2606 : {
2607 0 : CPLError(CE_Failure, CPLE_FileIO, "Can't open cloned source index");
2608 0 : return CE_Failure; // Source reported the error
2609 : }
2610 :
2611 1 : VSIFSeekL(srcidx, offset, SEEK_SET);
2612 1 : size = VSIFReadL(buffer, sizeof(ILIdx), static_cast<size_t>(size), srcidx);
2613 1 : if (size != GIntBig(buf.size()))
2614 : {
2615 0 : CPLError(CE_Failure, CPLE_FileIO, "Can't read cloned source index");
2616 0 : return CE_Failure; // Source reported the error
2617 : }
2618 :
2619 : // Mark the empty records as checked, by making the offset non-zero
2620 2 : for (vector<ILIdx>::iterator it = buf.begin(); it != buf.end(); ++it)
2621 : {
2622 1 : if (it->offset == 0 && it->size == 0)
2623 0 : it->offset = net64(1);
2624 : }
2625 :
2626 : // Write it in the right place in the local index file
2627 1 : VSIFSeekL(l_ifp, bias + offset, SEEK_SET);
2628 1 : size = VSIFWriteL(&buf[0], sizeof(ILIdx), static_cast<size_t>(size), l_ifp);
2629 1 : if (size != GIntBig(buf.size()))
2630 : {
2631 0 : CPLError(CE_Failure, CPLE_FileIO, "Can't write to cloning MRF index");
2632 0 : return CE_Failure; // Source reported the error
2633 : }
2634 :
2635 : // Cloned index updated, restart this function, it will work now
2636 1 : return ReadTileIdx(tinfo, pos, img, bias);
2637 : }
2638 :
2639 : NAMESPACE_MRF_END
|
