Line data Source code
1 : /******************************************************************************
2 : * $Id$
3 : *
4 : * Project: WMS Client Mini Driver
5 : * Purpose: Implementation of Dataset and RasterBand classes for WMS
6 : * and other similar services.
7 : * Author: Lucian Plesea
8 : *
9 : ******************************************************************************
10 : * Copyright (c) 2016, Lucian Plesea
11 : *
12 : * Copyright 2016 Esri
13 : *
14 : * Licensed under the Apache License, Version 2.0 (the "License");
15 : * you may not use this file except in compliance with the License.
16 : * You may obtain a copy of the License at
17 : *
18 : * http://www.apache.org/licenses/LICENSE-2.0
19 : *
20 : * Unless required by applicable law or agreed to in writing, software
21 : * distributed under the License is distributed on an "AS IS" BASIS,
22 : * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
23 : * See the License for the specific language governing permissions and
24 : * limitations under the License.
25 : ****************************************************************************/
26 :
27 : /*
28 : A WMS style minidriver that allows an MRF or an Esri bundle to be read from a
29 : URL, using one range request per tile All parameters have to be defined in the
30 : WMS file, especially for the MRF, so only simple MRF files work. For a bundle,
31 : the size is assumed to be 128 tiles of 256 pixels each, which is the standard
32 : size.
33 : */
34 :
35 : #include "wmsdriver.h"
36 : #include "minidriver_mrf.h"
37 :
38 : using namespace WMSMiniDriver_MRF_ns;
39 :
40 : // Copied from frmts/mrf
41 :
42 : // A tile index record, 16 bytes, big endian
43 : typedef struct
44 : {
45 : GIntBig offset;
46 : GIntBig size;
47 : } MRFIdx;
48 :
49 : // Number of pages of size psz needed to hold n elements
50 0 : static inline int pcount(const int n, const int sz)
51 : {
52 0 : return 1 + (n - 1) / sz;
53 : }
54 :
55 : // Returns a pagecount per dimension, .l will have the total number
56 0 : static inline const ILSize pcount(const ILSize &size, const ILSize &psz)
57 : {
58 0 : ILSize count;
59 0 : count.x = pcount(size.x, psz.x);
60 0 : count.y = pcount(size.y, psz.y);
61 0 : count.z = pcount(size.z, psz.z);
62 0 : count.c = pcount(size.c, psz.c);
63 0 : count.l = static_cast<GIntBig>(count.x) * count.y * count.z * count.c;
64 0 : return count;
65 : }
66 :
67 : // End copied from frmts/mrf
68 :
69 : // pread_t adapter for VSIL
70 0 : static size_t pread_VSIL(void *user_data, void *buff, size_t count,
71 : off_t offset)
72 : {
73 0 : VSILFILE *fp = reinterpret_cast<VSILFILE *>(user_data);
74 0 : VSIFSeekL(fp, offset, SEEK_SET);
75 0 : return VSIFReadL(buff, 1, count, fp);
76 : }
77 :
78 : // pread_t adapter for curl. We use the multi interface to get the same options
79 0 : static size_t pread_curl(void *user_data, void *buff, size_t count,
80 : off_t offset)
81 : {
82 : // Use a copy of the provided request, which has the options and the URL
83 : // preset
84 0 : WMSHTTPRequest request(*(reinterpret_cast<WMSHTTPRequest *>(user_data)));
85 : request.Range.Printf(CPL_FRMT_GUIB "-" CPL_FRMT_GUIB,
86 : static_cast<GUIntBig>(offset),
87 0 : static_cast<GUIntBig>(offset + count - 1));
88 0 : WMSHTTPInitializeRequest(&request);
89 0 : if (WMSHTTPFetchMulti(&request) != CE_None)
90 : {
91 0 : CPLError(CE_Failure, CPLE_AppDefined,
92 : "GDALWMS_MRF: failed to retrieve index data");
93 0 : return 0;
94 : }
95 :
96 0 : int success = (request.nStatus == 200) ||
97 0 : (!request.Range.empty() && request.nStatus == 206);
98 0 : if (!success || request.pabyData == nullptr || request.nDataLen == 0)
99 : {
100 0 : CPLError(CE_Failure, CPLE_HttpResponse,
101 : "GDALWMS: Unable to download data from %s",
102 : request.URL.c_str());
103 0 : return 0; // Error flag
104 : }
105 :
106 : // Might get less data than requested
107 0 : if (request.nDataLen < count)
108 0 : memset(buff, 0, count);
109 0 : memcpy(buff, request.pabyData, request.nDataLen);
110 0 : return request.nDataLen;
111 : }
112 :
113 0 : SectorCache::SectorCache(void *user_data, pread_t fn, unsigned int size,
114 0 : unsigned int count)
115 0 : : n(count + 2), m(size), reader(fn ? fn : pread_VSIL),
116 0 : reader_data(user_data), last_used(nullptr)
117 : {
118 0 : }
119 :
120 : // Returns an in-memory offset to the byte at the given address, within a sector
121 : // Returns NULL if the sector can't be read
122 0 : void *SectorCache::data(size_t address)
123 : {
124 0 : for (size_t i = 0; i < store.size(); i++)
125 : {
126 0 : if (store[i].uid == address / m)
127 : {
128 0 : last_used = &store[i];
129 0 : return &(last_used->range[address % m]);
130 : }
131 : }
132 :
133 : // Not found, need a target sector to replace
134 : Sector *target;
135 0 : if (store.size() < m)
136 : { // Create a new sector if there are slots available
137 0 : store.resize(store.size() + 1);
138 0 : target = &store.back();
139 : }
140 : else
141 : { // Choose a random one to replace, but not the last used, to avoid
142 : // thrashing
143 0 : do
144 : {
145 : // coverity[dont_call]
146 0 : target = &(store[rand() % n]);
147 0 : } while (target == last_used);
148 : }
149 :
150 0 : target->range.resize(m);
151 0 : if (reader(reader_data, &target->range[0], m,
152 0 : static_cast<off_t>((address / m) * m)))
153 : { // Success
154 0 : target->uid = address / m;
155 0 : last_used = target;
156 0 : return &(last_used->range[address % m]);
157 : }
158 :
159 : // Failure
160 : // If this is the last sector, it could be a new sector with invalid data,
161 : // so we remove it Otherwise, the previous content is still good
162 0 : if (target == &store.back())
163 0 : store.pop_back();
164 : // Signal invalid request
165 0 : return nullptr;
166 : }
167 :
168 : // Keep in sync with the type enum
169 : static const int ir_size[WMSMiniDriver_MRF::tEND] = {16, 8};
170 :
171 0 : WMSMiniDriver_MRF::WMSMiniDriver_MRF()
172 0 : : m_type(tMRF), fp(nullptr), m_request(nullptr), index_cache(nullptr)
173 : {
174 0 : }
175 :
176 0 : WMSMiniDriver_MRF::~WMSMiniDriver_MRF()
177 : {
178 0 : if (index_cache)
179 0 : delete index_cache;
180 0 : if (fp)
181 0 : VSIFCloseL(fp);
182 0 : delete m_request;
183 0 : }
184 :
185 0 : CPLErr WMSMiniDriver_MRF::Initialize(CPLXMLNode *config,
186 : CPL_UNUSED char **papszOpenOptions)
187 : {
188 : // This gets called before the rest of the WMS driver gets initialized
189 : // The MRF reader only works if all datawindow is defined within the WMS
190 : // file
191 :
192 0 : m_base_url = CPLGetXMLValue(config, "ServerURL", "");
193 0 : if (m_base_url.empty())
194 : {
195 0 : CPLError(CE_Failure, CPLE_AppDefined,
196 : "GDALWMS, MRF: ServerURL missing.");
197 0 : return CE_Failure;
198 : }
199 :
200 : // Index file location, in case it is different from the normal file name
201 0 : m_idxname = CPLGetXMLValue(config, "index", "");
202 :
203 0 : CPLString osType(CPLGetXMLValue(config, "type", ""));
204 :
205 0 : if (EQUAL(osType, "bundle"))
206 0 : m_type = tBundle;
207 :
208 0 : if (m_type == tBundle)
209 : {
210 0 : m_parent_dataset->WMSSetDefaultOverviewCount(0);
211 0 : m_parent_dataset->WMSSetDefaultTileCount(128, 128);
212 0 : m_parent_dataset->WMSSetDefaultBlockSize(256, 256);
213 0 : m_parent_dataset->WMSSetDefaultTileLevel(0);
214 0 : m_parent_dataset->WMSSetNeedsDataWindow(FALSE);
215 0 : offsets.push_back(64);
216 : }
217 : else
218 : { // MRF
219 0 : offsets.push_back(0);
220 : }
221 :
222 0 : return CE_None;
223 : }
224 :
225 : // Test for URL, things that curl can deal with while doing a range request
226 : // http and https should work, not sure about ftp or file
227 0 : int inline static is_url(const CPLString &value)
228 : {
229 0 : return (value.ifind("http://") == 0 || value.ifind("https://") == 0 ||
230 0 : value.ifind("ftp://") == 0 || value.ifind("file://") == 0);
231 : }
232 :
233 : // Called after the dataset is initialized by the main WMS driver
234 0 : CPLErr WMSMiniDriver_MRF::EndInit()
235 : {
236 0 : int index_is_url = 1;
237 0 : if (!m_idxname.empty())
238 : { // Provided, could be path or URL
239 0 : if (!is_url(m_idxname))
240 : {
241 0 : index_is_url = 0;
242 0 : fp = VSIFOpenL(m_idxname, "rb");
243 0 : if (fp == nullptr)
244 : {
245 0 : CPLError(CE_Failure, CPLE_FileIO, "Can't open index file %s",
246 : m_idxname.c_str());
247 0 : return CE_Failure;
248 : }
249 0 : index_cache = new SectorCache(fp);
250 : }
251 : }
252 : else
253 : { // Not provided, change extension to .idx if we can, otherwise use the
254 : // same file
255 0 : m_idxname = m_base_url;
256 : }
257 :
258 0 : if (index_is_url)
259 : { // prepare a WMS request, the pread_curl will execute it repeatedly
260 0 : m_request = new WMSHTTPRequest();
261 0 : m_request->URL = m_idxname;
262 0 : m_request->options = m_parent_dataset->GetHTTPRequestOpts();
263 0 : index_cache = new SectorCache(m_request, pread_curl);
264 : }
265 :
266 : // Set the level index offsets, assume MRF order since esri bundles don't
267 : // have overviews
268 : ILSize size(
269 0 : m_parent_dataset->GetRasterXSize(), m_parent_dataset->GetRasterYSize(),
270 : 1, // Single slice for now
271 : 1, // Ignore the c, only single or interleved data supported by WMS
272 0 : m_parent_dataset->GetRasterBand(1)->GetOverviewCount());
273 :
274 : int psx, psy;
275 0 : m_parent_dataset->GetRasterBand(1)->GetBlockSize(&psx, &psy);
276 0 : ILSize pagesize(psx, psy, 1, 1, 1);
277 :
278 0 : if (m_type == tBundle)
279 : { // A bundle contains 128x128 pages, regadless of the raster size
280 0 : size.x = psx * 128;
281 0 : size.y = psy * 128;
282 : }
283 :
284 0 : for (GIntBig l = size.l; l >= 0; l--)
285 : {
286 0 : ILSize pagecount = pcount(size, pagesize);
287 0 : pages.push_back(pagecount);
288 0 : if (l > 0) // Only for existing levels
289 0 : offsets.push_back(offsets.back() + ir_size[m_type] * pagecount.l);
290 :
291 : // Sometimes this may be a 3
292 0 : size.x = pcount(size.x, 2);
293 0 : size.y = pcount(size.y, 2);
294 : }
295 :
296 0 : return CE_None;
297 : }
298 :
299 : // Return -1 if error occurs
300 0 : size_t WMSMiniDriver_MRF::GetIndexAddress(
301 : const GDALWMSTiledImageRequestInfo &tiri) const
302 : {
303 : // Bottom level is 0
304 0 : int l = -tiri.m_level;
305 0 : if (l < 0 || l >= static_cast<int>(offsets.size()))
306 0 : return ~static_cast<size_t>(0); // Indexing error
307 0 : if (tiri.m_x >= pages[l].x || tiri.m_y >= pages[l].y)
308 0 : return ~static_cast<size_t>(0);
309 0 : return static_cast<size_t>(offsets[l] + (pages[l].x * tiri.m_y + tiri.m_x) *
310 0 : ir_size[m_type]);
311 : }
312 :
313 : // Signal errors and return error message
314 0 : CPLErr WMSMiniDriver_MRF::TiledImageRequest(
315 : WMSHTTPRequest &request, CPL_UNUSED const GDALWMSImageRequestInfo &iri,
316 : const GDALWMSTiledImageRequestInfo &tiri)
317 : {
318 0 : CPLString &url = request.URL;
319 0 : url = m_base_url;
320 :
321 0 : size_t offset = GetIndexAddress(tiri);
322 0 : if (offset == static_cast<size_t>(-1))
323 : {
324 0 : request.Error = "Invalid level requested";
325 0 : return CE_Failure;
326 : }
327 :
328 0 : void *raw_index = index_cache->data(offset);
329 0 : if (raw_index == nullptr)
330 : {
331 0 : request.Error = "Invalid indexing";
332 0 : return CE_Failure;
333 : };
334 :
335 : // Store the tile size and offset in this structure
336 : MRFIdx idx;
337 :
338 0 : if (m_type == tMRF)
339 : {
340 0 : memcpy(&idx, raw_index, sizeof(idx));
341 :
342 : #if defined(CPL_LSB) // raw index is MSB
343 0 : idx.offset = CPL_SWAP64(idx.offset);
344 0 : idx.size = CPL_SWAP64(idx.size);
345 : #endif
346 : }
347 : else
348 : { // Bundle
349 : GIntBig bidx;
350 0 : memcpy(&bidx, raw_index, sizeof(bidx));
351 :
352 : #if defined(CPL_MSB) // bundle index is LSB
353 : bidx = CPL_SWAP64(bidx);
354 : #endif
355 :
356 0 : idx.offset = bidx & ((1ULL << 40) - 1);
357 0 : idx.size = bidx >> 40;
358 : }
359 :
360 : // Set the range or flag it as missing
361 0 : if (idx.size == 0)
362 : request.Range =
363 0 : "none"; // Signal that this block doesn't exist server-side
364 : else
365 : request.Range.Printf(CPL_FRMT_GUIB "-" CPL_FRMT_GUIB, idx.offset,
366 0 : idx.offset + idx.size - 1);
367 :
368 0 : return CE_None;
369 : }
|
