Line data Source code
1 : /******************************************************************************
2 : *
3 : * Project: WMS Client Mini Driver
4 : * Purpose: Implementation of Dataset and RasterBand classes for WMS
5 : * and other similar services.
6 : * Author: Lucian Plesea
7 : *
8 : ******************************************************************************
9 : * Copyright (c) 2016, Lucian Plesea
10 : *
11 : * Copyright 2016 Esri
12 : *
13 : * Licensed under the Apache License, Version 2.0 (the "License");
14 : * you may not use this file except in compliance with the License.
15 : * You may obtain a copy of the License at
16 : *
17 : * http://www.apache.org/licenses/LICENSE-2.0
18 : *
19 : * Unless required by applicable law or agreed to in writing, software
20 : * distributed under the License is distributed on an "AS IS" BASIS,
21 : * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
22 : * See the License for the specific language governing permissions and
23 : * limitations under the License.
24 : ****************************************************************************/
25 :
26 : /*
27 : A WMS style minidriver that allows an MRF or an Esri bundle to be read from a
28 : URL, using one range request per tile All parameters have to be defined in the
29 : WMS file, especially for the MRF, so only simple MRF files work. For a bundle,
30 : the size is assumed to be 128 tiles of 256 pixels each, which is the standard
31 : size.
32 : */
33 :
34 : #include "wmsdriver.h"
35 : #include "minidriver_mrf.h"
36 :
37 : using namespace WMSMiniDriver_MRF_ns;
38 :
39 : // Copied from frmts/mrf
40 :
41 : // A tile index record, 16 bytes, big endian
42 : typedef struct
43 : {
44 : GIntBig offset;
45 : GIntBig size;
46 : } MRFIdx;
47 :
48 : // Number of pages of size psz needed to hold n elements
49 0 : static inline int pcount(const int n, const int sz)
50 : {
51 0 : return 1 + (n - 1) / sz;
52 : }
53 :
54 : // Returns a pagecount per dimension, .l will have the total number
55 0 : static inline const ILSize pcount(const ILSize &size, const ILSize &psz)
56 : {
57 0 : ILSize count;
58 0 : count.x = pcount(size.x, psz.x);
59 0 : count.y = pcount(size.y, psz.y);
60 0 : count.z = pcount(size.z, psz.z);
61 0 : count.c = pcount(size.c, psz.c);
62 0 : count.l = static_cast<GIntBig>(count.x) * count.y * count.z * count.c;
63 0 : return count;
64 : }
65 :
66 : // End copied from frmts/mrf
67 :
68 : // pread_t adapter for VSIL
69 0 : static size_t pread_VSIL(void *user_data, void *buff, size_t count,
70 : off_t offset)
71 : {
72 0 : VSILFILE *fp = reinterpret_cast<VSILFILE *>(user_data);
73 0 : VSIFSeekL(fp, offset, SEEK_SET);
74 0 : return VSIFReadL(buff, 1, count, fp);
75 : }
76 :
77 : // pread_t adapter for curl. We use the multi interface to get the same options
78 0 : static size_t pread_curl(void *user_data, void *buff, size_t count,
79 : off_t offset)
80 : {
81 : // Use a copy of the provided request, which has the options and the URL
82 : // preset
83 0 : WMSHTTPRequest &request = *(reinterpret_cast<WMSHTTPRequest *>(user_data));
84 : request.Range.Printf(CPL_FRMT_GUIB "-" CPL_FRMT_GUIB,
85 : static_cast<GUIntBig>(offset),
86 0 : static_cast<GUIntBig>(offset + count - 1));
87 0 : WMSHTTPInitializeRequest(&request);
88 0 : if (WMSHTTPFetchMulti(&request) != CE_None)
89 : {
90 0 : CPLError(CE_Failure, CPLE_AppDefined,
91 : "GDALWMS_MRF: failed to retrieve index data");
92 0 : return 0;
93 : }
94 :
95 0 : int success = (request.nStatus == 200) ||
96 0 : (!request.Range.empty() && request.nStatus == 206);
97 0 : if (!success || request.pabyData == nullptr || request.nDataLen == 0)
98 : {
99 0 : CPLError(CE_Failure, CPLE_HttpResponse,
100 : "GDALWMS: Unable to download data from %s",
101 : request.URL.c_str());
102 0 : return 0; // Error flag
103 : }
104 :
105 : // Might get less data than requested
106 0 : if (request.nDataLen < count)
107 0 : memset(buff, 0, count);
108 0 : memcpy(buff, request.pabyData, request.nDataLen);
109 0 : return request.nDataLen;
110 : }
111 :
112 0 : SectorCache::SectorCache(void *user_data, pread_t fn, unsigned int size,
113 0 : unsigned int count)
114 0 : : n(count + 2), m(size), reader(fn ? fn : pread_VSIL),
115 0 : reader_data(user_data), last_used(nullptr)
116 : {
117 0 : }
118 :
119 : // Returns an in-memory offset to the byte at the given address, within a sector
120 : // Returns NULL if the sector can't be read
121 0 : void *SectorCache::data(size_t address)
122 : {
123 0 : for (size_t i = 0; i < store.size(); i++)
124 : {
125 0 : if (store[i].uid == address / m)
126 : {
127 0 : last_used = &store[i];
128 0 : return &(last_used->range[address % m]);
129 : }
130 : }
131 :
132 : // Not found, need a target sector to replace
133 : Sector *target;
134 0 : if (store.size() < m)
135 : { // Create a new sector if there are slots available
136 0 : store.resize(store.size() + 1);
137 0 : target = &store.back();
138 : }
139 : else
140 : { // Choose a random one to replace, but not the last used, to avoid
141 : // thrashing
142 0 : do
143 : {
144 : #ifndef __COVERITY__
145 0 : target = &(store[rand() % n]);
146 : #else
147 : target = &(store[0]);
148 : #endif
149 0 : } while (target == last_used);
150 : }
151 :
152 0 : target->range.resize(m);
153 0 : if (reader(reader_data, &target->range[0], m,
154 0 : static_cast<off_t>((address / m) * m)))
155 : { // Success
156 0 : target->uid = address / m;
157 0 : last_used = target;
158 0 : return &(last_used->range[address % m]);
159 : }
160 :
161 : // Failure
162 : // If this is the last sector, it could be a new sector with invalid data,
163 : // so we remove it Otherwise, the previous content is still good
164 0 : if (target == &store.back())
165 0 : store.pop_back();
166 : // Signal invalid request
167 0 : return nullptr;
168 : }
169 :
170 : // Keep in sync with the type enum
171 : static const int ir_size[WMSMiniDriver_MRF::tEND] = {16, 8};
172 :
173 0 : WMSMiniDriver_MRF::WMSMiniDriver_MRF()
174 0 : : m_type(tMRF), fp(nullptr), m_request(nullptr), index_cache(nullptr)
175 : {
176 0 : }
177 :
178 0 : WMSMiniDriver_MRF::~WMSMiniDriver_MRF()
179 : {
180 0 : if (index_cache)
181 0 : delete index_cache;
182 0 : if (fp)
183 0 : VSIFCloseL(fp);
184 0 : delete m_request;
185 0 : }
186 :
187 0 : CPLErr WMSMiniDriver_MRF::Initialize(CPLXMLNode *config,
188 : CPL_UNUSED char **papszOpenOptions)
189 : {
190 : // This gets called before the rest of the WMS driver gets initialized
191 : // The MRF reader only works if all datawindow is defined within the WMS
192 : // file
193 :
194 0 : m_base_url = CPLGetXMLValue(config, "ServerURL", "");
195 0 : if (m_base_url.empty())
196 : {
197 0 : CPLError(CE_Failure, CPLE_AppDefined,
198 : "GDALWMS, MRF: ServerURL missing.");
199 0 : return CE_Failure;
200 : }
201 :
202 : // Index file location, in case it is different from the normal file name
203 0 : m_idxname = CPLGetXMLValue(config, "index", "");
204 :
205 0 : CPLString osType(CPLGetXMLValue(config, "type", ""));
206 :
207 0 : if (EQUAL(osType, "bundle"))
208 0 : m_type = tBundle;
209 :
210 0 : if (m_type == tBundle)
211 : {
212 0 : m_parent_dataset->WMSSetDefaultOverviewCount(0);
213 0 : m_parent_dataset->WMSSetDefaultTileCount(128, 128);
214 0 : m_parent_dataset->WMSSetDefaultBlockSize(256, 256);
215 0 : m_parent_dataset->WMSSetDefaultTileLevel(0);
216 0 : m_parent_dataset->WMSSetNeedsDataWindow(FALSE);
217 0 : offsets.push_back(64);
218 : }
219 : else
220 : { // MRF
221 0 : offsets.push_back(0);
222 : }
223 :
224 0 : return CE_None;
225 : }
226 :
227 : // Test for URL, things that curl can deal with while doing a range request
228 : // http and https should work, not sure about ftp or file
229 0 : int inline static is_url(const CPLString &value)
230 : {
231 0 : return (value.ifind("http://") == 0 || value.ifind("https://") == 0 ||
232 0 : value.ifind("ftp://") == 0 || value.ifind("file://") == 0);
233 : }
234 :
235 : // Called after the dataset is initialized by the main WMS driver
236 0 : CPLErr WMSMiniDriver_MRF::EndInit()
237 : {
238 0 : int index_is_url = 1;
239 0 : if (!m_idxname.empty())
240 : { // Provided, could be path or URL
241 0 : if (!is_url(m_idxname))
242 : {
243 0 : index_is_url = 0;
244 0 : fp = VSIFOpenL(m_idxname, "rb");
245 0 : if (fp == nullptr)
246 : {
247 0 : CPLError(CE_Failure, CPLE_FileIO, "Can't open index file %s",
248 : m_idxname.c_str());
249 0 : return CE_Failure;
250 : }
251 0 : index_cache = new SectorCache(fp);
252 : }
253 : }
254 : else
255 : { // Not provided, change extension to .idx if we can, otherwise use the
256 : // same file
257 0 : m_idxname = m_base_url;
258 : }
259 :
260 0 : if (index_is_url)
261 : { // prepare a WMS request, the pread_curl will execute it repeatedly
262 0 : m_request = new WMSHTTPRequest();
263 0 : m_request->URL = m_idxname;
264 0 : m_request->options = m_parent_dataset->GetHTTPRequestOpts();
265 0 : index_cache = new SectorCache(m_request, pread_curl);
266 : }
267 :
268 : // Set the level index offsets, assume MRF order since esri bundles don't
269 : // have overviews
270 : ILSize size(
271 0 : m_parent_dataset->GetRasterXSize(), m_parent_dataset->GetRasterYSize(),
272 : 1, // Single slice for now
273 : 1, // Ignore the c, only single or interleved data supported by WMS
274 0 : m_parent_dataset->GetRasterBand(1)->GetOverviewCount());
275 :
276 : int psx, psy;
277 0 : m_parent_dataset->GetRasterBand(1)->GetBlockSize(&psx, &psy);
278 0 : ILSize pagesize(psx, psy, 1, 1, 1);
279 :
280 0 : if (m_type == tBundle)
281 : { // A bundle contains 128x128 pages, regadless of the raster size
282 0 : size.x = psx * 128;
283 0 : size.y = psy * 128;
284 : }
285 :
286 0 : for (GIntBig l = size.l; l >= 0; l--)
287 : {
288 0 : ILSize pagecount = pcount(size, pagesize);
289 0 : pages.push_back(pagecount);
290 0 : if (l > 0) // Only for existing levels
291 0 : offsets.push_back(offsets.back() + ir_size[m_type] * pagecount.l);
292 :
293 : // Sometimes this may be a 3
294 0 : size.x = pcount(size.x, 2);
295 0 : size.y = pcount(size.y, 2);
296 : }
297 :
298 0 : return CE_None;
299 : }
300 :
301 : // Return -1 if error occurs
302 0 : size_t WMSMiniDriver_MRF::GetIndexAddress(
303 : const GDALWMSTiledImageRequestInfo &tiri) const
304 : {
305 : // Bottom level is 0
306 0 : int l = -tiri.m_level;
307 0 : if (l < 0 || l >= static_cast<int>(offsets.size()))
308 0 : return ~static_cast<size_t>(0); // Indexing error
309 0 : if (tiri.m_x >= pages[l].x || tiri.m_y >= pages[l].y)
310 0 : return ~static_cast<size_t>(0);
311 0 : return static_cast<size_t>(offsets[l] + (pages[l].x * tiri.m_y + tiri.m_x) *
312 0 : ir_size[m_type]);
313 : }
314 :
315 : // Signal errors and return error message
316 0 : CPLErr WMSMiniDriver_MRF::TiledImageRequest(
317 : WMSHTTPRequest &request, CPL_UNUSED const GDALWMSImageRequestInfo &iri,
318 : const GDALWMSTiledImageRequestInfo &tiri)
319 : {
320 0 : CPLString &url = request.URL;
321 0 : url = m_base_url;
322 :
323 0 : size_t offset = GetIndexAddress(tiri);
324 0 : if (offset == static_cast<size_t>(-1))
325 : {
326 0 : request.Error = "Invalid level requested";
327 0 : return CE_Failure;
328 : }
329 :
330 0 : void *raw_index = index_cache->data(offset);
331 0 : if (raw_index == nullptr)
332 : {
333 0 : request.Error = "Invalid indexing";
334 0 : return CE_Failure;
335 : };
336 :
337 : // Store the tile size and offset in this structure
338 : MRFIdx idx;
339 :
340 0 : if (m_type == tMRF)
341 : {
342 0 : memcpy(&idx, raw_index, sizeof(idx));
343 :
344 : #if defined(CPL_LSB) // raw index is MSB
345 0 : idx.offset = CPL_SWAP64(idx.offset);
346 0 : idx.size = CPL_SWAP64(idx.size);
347 : #endif
348 : }
349 : else
350 : { // Bundle
351 : GIntBig bidx;
352 0 : memcpy(&bidx, raw_index, sizeof(bidx));
353 :
354 : #if defined(CPL_MSB) // bundle index is LSB
355 : bidx = CPL_SWAP64(bidx);
356 : #endif
357 :
358 0 : idx.offset = bidx & ((1ULL << 40) - 1);
359 0 : idx.size = bidx >> 40;
360 : }
361 :
362 : // Set the range or flag it as missing
363 0 : if (idx.size == 0)
364 : request.Range =
365 0 : "none"; // Signal that this block doesn't exist server-side
366 : else
367 : request.Range.Printf(CPL_FRMT_GUIB "-" CPL_FRMT_GUIB, idx.offset,
368 0 : idx.offset + idx.size - 1);
369 :
370 0 : return CE_None;
371 : }
|
