Line data Source code
1 : /******************************************************************************
2 : *
3 : * Project: WMS Client Mini Driver
4 : * Purpose: Implementation of Dataset and RasterBand classes for WMS
5 : * and other similar services.
6 : * Author: Lucian Plesea
7 : *
8 : ******************************************************************************
9 : * Copyright (c) 2016, Lucian Plesea
10 : *
11 : * Copyright 2016 Esri
12 : *
13 : * Licensed under the Apache License, Version 2.0 (the "License");
14 : * you may not use this file except in compliance with the License.
15 : * You may obtain a copy of the License at
16 : *
17 : * http://www.apache.org/licenses/LICENSE-2.0
18 : *
19 : * Unless required by applicable law or agreed to in writing, software
20 : * distributed under the License is distributed on an "AS IS" BASIS,
21 : * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
22 : * See the License for the specific language governing permissions and
23 : * limitations under the License.
24 : ****************************************************************************/
25 :
26 : /*
27 : A WMS style minidriver that allows an MRF or an Esri bundle to be read from a
28 : URL, using one range request per tile All parameters have to be defined in the
29 : WMS file, especially for the MRF, so only simple MRF files work. For a bundle,
30 : the size is assumed to be 128 tiles of 256 pixels each, which is the standard
31 : size.
32 : */
33 :
34 : #include "wmsdriver.h"
35 : #include "minidriver_mrf.h"
36 :
37 : using namespace WMSMiniDriver_MRF_ns;
38 :
39 : // Copied from frmts/mrf
40 :
41 : // A tile index record, 16 bytes, big endian
42 : typedef struct
43 : {
44 : GIntBig offset;
45 : GIntBig size;
46 : } MRFIdx;
47 :
48 : // Number of pages of size psz needed to hold n elements
49 0 : static inline int pcount(const int n, const int sz)
50 : {
51 0 : return 1 + (n - 1) / sz;
52 : }
53 :
54 : // Returns a pagecount per dimension, .l will have the total number
55 0 : static inline const ILSize pcount(const ILSize &size, const ILSize &psz)
56 : {
57 0 : ILSize count;
58 0 : count.x = pcount(size.x, psz.x);
59 0 : count.y = pcount(size.y, psz.y);
60 0 : count.z = pcount(size.z, psz.z);
61 0 : count.c = pcount(size.c, psz.c);
62 0 : count.l = static_cast<GIntBig>(count.x) * count.y * count.z * count.c;
63 0 : return count;
64 : }
65 :
66 : // End copied from frmts/mrf
67 :
68 : // pread_t adapter for VSIL
69 0 : static size_t pread_VSIL(void *user_data, void *buff, size_t count,
70 : off_t offset)
71 : {
72 0 : VSILFILE *fp = reinterpret_cast<VSILFILE *>(user_data);
73 0 : VSIFSeekL(fp, offset, SEEK_SET);
74 0 : return VSIFReadL(buff, 1, count, fp);
75 : }
76 :
77 : // pread_t adapter for curl. We use the multi interface to get the same options
78 0 : static size_t pread_curl(void *user_data, void *buff, size_t count,
79 : off_t offset)
80 : {
81 : // Use a copy of the provided request, which has the options and the URL
82 : // preset
83 0 : WMSHTTPRequest request(*(reinterpret_cast<WMSHTTPRequest *>(user_data)));
84 : request.Range.Printf(CPL_FRMT_GUIB "-" CPL_FRMT_GUIB,
85 : static_cast<GUIntBig>(offset),
86 0 : static_cast<GUIntBig>(offset + count - 1));
87 0 : WMSHTTPInitializeRequest(&request);
88 0 : if (WMSHTTPFetchMulti(&request) != CE_None)
89 : {
90 0 : CPLError(CE_Failure, CPLE_AppDefined,
91 : "GDALWMS_MRF: failed to retrieve index data");
92 0 : return 0;
93 : }
94 :
95 0 : int success = (request.nStatus == 200) ||
96 0 : (!request.Range.empty() && request.nStatus == 206);
97 0 : if (!success || request.pabyData == nullptr || request.nDataLen == 0)
98 : {
99 0 : CPLError(CE_Failure, CPLE_HttpResponse,
100 : "GDALWMS: Unable to download data from %s",
101 : request.URL.c_str());
102 0 : return 0; // Error flag
103 : }
104 :
105 : // Might get less data than requested
106 0 : if (request.nDataLen < count)
107 0 : memset(buff, 0, count);
108 0 : memcpy(buff, request.pabyData, request.nDataLen);
109 0 : return request.nDataLen;
110 : }
111 :
112 0 : SectorCache::SectorCache(void *user_data, pread_t fn, unsigned int size,
113 0 : unsigned int count)
114 0 : : n(count + 2), m(size), reader(fn ? fn : pread_VSIL),
115 0 : reader_data(user_data), last_used(nullptr)
116 : {
117 0 : }
118 :
119 : // Returns an in-memory offset to the byte at the given address, within a sector
120 : // Returns NULL if the sector can't be read
121 0 : void *SectorCache::data(size_t address)
122 : {
123 0 : for (size_t i = 0; i < store.size(); i++)
124 : {
125 0 : if (store[i].uid == address / m)
126 : {
127 0 : last_used = &store[i];
128 0 : return &(last_used->range[address % m]);
129 : }
130 : }
131 :
132 : // Not found, need a target sector to replace
133 : Sector *target;
134 0 : if (store.size() < m)
135 : { // Create a new sector if there are slots available
136 0 : store.resize(store.size() + 1);
137 0 : target = &store.back();
138 : }
139 : else
140 : { // Choose a random one to replace, but not the last used, to avoid
141 : // thrashing
142 0 : do
143 : {
144 : // coverity[dont_call]
145 0 : target = &(store[rand() % n]);
146 0 : } while (target == last_used);
147 : }
148 :
149 0 : target->range.resize(m);
150 0 : if (reader(reader_data, &target->range[0], m,
151 0 : static_cast<off_t>((address / m) * m)))
152 : { // Success
153 0 : target->uid = address / m;
154 0 : last_used = target;
155 0 : return &(last_used->range[address % m]);
156 : }
157 :
158 : // Failure
159 : // If this is the last sector, it could be a new sector with invalid data,
160 : // so we remove it Otherwise, the previous content is still good
161 0 : if (target == &store.back())
162 0 : store.pop_back();
163 : // Signal invalid request
164 0 : return nullptr;
165 : }
166 :
167 : // Keep in sync with the type enum
168 : static const int ir_size[WMSMiniDriver_MRF::tEND] = {16, 8};
169 :
170 0 : WMSMiniDriver_MRF::WMSMiniDriver_MRF()
171 0 : : m_type(tMRF), fp(nullptr), m_request(nullptr), index_cache(nullptr)
172 : {
173 0 : }
174 :
175 0 : WMSMiniDriver_MRF::~WMSMiniDriver_MRF()
176 : {
177 0 : if (index_cache)
178 0 : delete index_cache;
179 0 : if (fp)
180 0 : VSIFCloseL(fp);
181 0 : delete m_request;
182 0 : }
183 :
184 0 : CPLErr WMSMiniDriver_MRF::Initialize(CPLXMLNode *config,
185 : CPL_UNUSED char **papszOpenOptions)
186 : {
187 : // This gets called before the rest of the WMS driver gets initialized
188 : // The MRF reader only works if all datawindow is defined within the WMS
189 : // file
190 :
191 0 : m_base_url = CPLGetXMLValue(config, "ServerURL", "");
192 0 : if (m_base_url.empty())
193 : {
194 0 : CPLError(CE_Failure, CPLE_AppDefined,
195 : "GDALWMS, MRF: ServerURL missing.");
196 0 : return CE_Failure;
197 : }
198 :
199 : // Index file location, in case it is different from the normal file name
200 0 : m_idxname = CPLGetXMLValue(config, "index", "");
201 :
202 0 : CPLString osType(CPLGetXMLValue(config, "type", ""));
203 :
204 0 : if (EQUAL(osType, "bundle"))
205 0 : m_type = tBundle;
206 :
207 0 : if (m_type == tBundle)
208 : {
209 0 : m_parent_dataset->WMSSetDefaultOverviewCount(0);
210 0 : m_parent_dataset->WMSSetDefaultTileCount(128, 128);
211 0 : m_parent_dataset->WMSSetDefaultBlockSize(256, 256);
212 0 : m_parent_dataset->WMSSetDefaultTileLevel(0);
213 0 : m_parent_dataset->WMSSetNeedsDataWindow(FALSE);
214 0 : offsets.push_back(64);
215 : }
216 : else
217 : { // MRF
218 0 : offsets.push_back(0);
219 : }
220 :
221 0 : return CE_None;
222 : }
223 :
224 : // Test for URL, things that curl can deal with while doing a range request
225 : // http and https should work, not sure about ftp or file
226 0 : int inline static is_url(const CPLString &value)
227 : {
228 0 : return (value.ifind("http://") == 0 || value.ifind("https://") == 0 ||
229 0 : value.ifind("ftp://") == 0 || value.ifind("file://") == 0);
230 : }
231 :
232 : // Called after the dataset is initialized by the main WMS driver
233 0 : CPLErr WMSMiniDriver_MRF::EndInit()
234 : {
235 0 : int index_is_url = 1;
236 0 : if (!m_idxname.empty())
237 : { // Provided, could be path or URL
238 0 : if (!is_url(m_idxname))
239 : {
240 0 : index_is_url = 0;
241 0 : fp = VSIFOpenL(m_idxname, "rb");
242 0 : if (fp == nullptr)
243 : {
244 0 : CPLError(CE_Failure, CPLE_FileIO, "Can't open index file %s",
245 : m_idxname.c_str());
246 0 : return CE_Failure;
247 : }
248 0 : index_cache = new SectorCache(fp);
249 : }
250 : }
251 : else
252 : { // Not provided, change extension to .idx if we can, otherwise use the
253 : // same file
254 0 : m_idxname = m_base_url;
255 : }
256 :
257 0 : if (index_is_url)
258 : { // prepare a WMS request, the pread_curl will execute it repeatedly
259 0 : m_request = new WMSHTTPRequest();
260 0 : m_request->URL = m_idxname;
261 0 : m_request->options = m_parent_dataset->GetHTTPRequestOpts();
262 0 : index_cache = new SectorCache(m_request, pread_curl);
263 : }
264 :
265 : // Set the level index offsets, assume MRF order since esri bundles don't
266 : // have overviews
267 : ILSize size(
268 0 : m_parent_dataset->GetRasterXSize(), m_parent_dataset->GetRasterYSize(),
269 : 1, // Single slice for now
270 : 1, // Ignore the c, only single or interleved data supported by WMS
271 0 : m_parent_dataset->GetRasterBand(1)->GetOverviewCount());
272 :
273 : int psx, psy;
274 0 : m_parent_dataset->GetRasterBand(1)->GetBlockSize(&psx, &psy);
275 0 : ILSize pagesize(psx, psy, 1, 1, 1);
276 :
277 0 : if (m_type == tBundle)
278 : { // A bundle contains 128x128 pages, regadless of the raster size
279 0 : size.x = psx * 128;
280 0 : size.y = psy * 128;
281 : }
282 :
283 0 : for (GIntBig l = size.l; l >= 0; l--)
284 : {
285 0 : ILSize pagecount = pcount(size, pagesize);
286 0 : pages.push_back(pagecount);
287 0 : if (l > 0) // Only for existing levels
288 0 : offsets.push_back(offsets.back() + ir_size[m_type] * pagecount.l);
289 :
290 : // Sometimes this may be a 3
291 0 : size.x = pcount(size.x, 2);
292 0 : size.y = pcount(size.y, 2);
293 : }
294 :
295 0 : return CE_None;
296 : }
297 :
298 : // Return -1 if error occurs
299 0 : size_t WMSMiniDriver_MRF::GetIndexAddress(
300 : const GDALWMSTiledImageRequestInfo &tiri) const
301 : {
302 : // Bottom level is 0
303 0 : int l = -tiri.m_level;
304 0 : if (l < 0 || l >= static_cast<int>(offsets.size()))
305 0 : return ~static_cast<size_t>(0); // Indexing error
306 0 : if (tiri.m_x >= pages[l].x || tiri.m_y >= pages[l].y)
307 0 : return ~static_cast<size_t>(0);
308 0 : return static_cast<size_t>(offsets[l] + (pages[l].x * tiri.m_y + tiri.m_x) *
309 0 : ir_size[m_type]);
310 : }
311 :
312 : // Signal errors and return error message
313 0 : CPLErr WMSMiniDriver_MRF::TiledImageRequest(
314 : WMSHTTPRequest &request, CPL_UNUSED const GDALWMSImageRequestInfo &iri,
315 : const GDALWMSTiledImageRequestInfo &tiri)
316 : {
317 0 : CPLString &url = request.URL;
318 0 : url = m_base_url;
319 :
320 0 : size_t offset = GetIndexAddress(tiri);
321 0 : if (offset == static_cast<size_t>(-1))
322 : {
323 0 : request.Error = "Invalid level requested";
324 0 : return CE_Failure;
325 : }
326 :
327 0 : void *raw_index = index_cache->data(offset);
328 0 : if (raw_index == nullptr)
329 : {
330 0 : request.Error = "Invalid indexing";
331 0 : return CE_Failure;
332 : };
333 :
334 : // Store the tile size and offset in this structure
335 : MRFIdx idx;
336 :
337 0 : if (m_type == tMRF)
338 : {
339 0 : memcpy(&idx, raw_index, sizeof(idx));
340 :
341 : #if defined(CPL_LSB) // raw index is MSB
342 0 : idx.offset = CPL_SWAP64(idx.offset);
343 0 : idx.size = CPL_SWAP64(idx.size);
344 : #endif
345 : }
346 : else
347 : { // Bundle
348 : GIntBig bidx;
349 0 : memcpy(&bidx, raw_index, sizeof(bidx));
350 :
351 : #if defined(CPL_MSB) // bundle index is LSB
352 : bidx = CPL_SWAP64(bidx);
353 : #endif
354 :
355 0 : idx.offset = bidx & ((1ULL << 40) - 1);
356 0 : idx.size = bidx >> 40;
357 : }
358 :
359 : // Set the range or flag it as missing
360 0 : if (idx.size == 0)
361 : request.Range =
362 0 : "none"; // Signal that this block doesn't exist server-side
363 : else
364 : request.Range.Printf(CPL_FRMT_GUIB "-" CPL_FRMT_GUIB, idx.offset,
365 0 : idx.offset + idx.size - 1);
366 :
367 0 : return CE_None;
368 : }
|
