Line data Source code
1 : /******************************************************************************
2 : * Project: OGR
3 : * Purpose: OGRGMLASDriver implementation
4 : * Author: Even Rouault, <even dot rouault at spatialys dot com>
5 : *
6 : * Initial development funded by the European Earth observation programme
7 : * Copernicus
8 : *
9 : ******************************************************************************
10 : * Copyright (c) 2016, Even Rouault, <even dot rouault at spatialys dot com>
11 : *
12 : * Permission is hereby granted, free of charge, to any person obtaining a
13 : * copy of this software and associated documentation files (the "Software"),
14 : * to deal in the Software without restriction, including without limitation
15 : * the rights to use, copy, modify, merge, publish, distribute, sublicense,
16 : * and/or sell copies of the Software, and to permit persons to whom the
17 : * Software is furnished to do so, subject to the following conditions:
18 : *
19 : * The above copyright notice and this permission notice shall be included
20 : * in all copies or substantial portions of the Software.
21 : *
22 : * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
23 : * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
24 : * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
25 : * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
26 : * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
27 : * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
28 : * DEALINGS IN THE SOFTWARE.
29 : ****************************************************************************/
30 :
31 : #include "ogr_gmlas.h"
32 :
33 : #include "cpl_http.h"
34 :
35 : #include <time.h>
36 :
37 : /************************************************************************/
38 : /* GMLASXLinkResolver() */
39 : /************************************************************************/
40 :
41 187 : GMLASXLinkResolver::GMLASXLinkResolver()
42 : : m_nMaxRAMCacheSize(
43 187 : atoi(CPLGetConfigOption("GMLAS_XLINK_RAM_CACHE_SIZE", "10000000")))
44 : {
45 187 : }
46 :
47 : /************************************************************************/
48 : /* SetConf() */
49 : /************************************************************************/
50 :
51 171 : void GMLASXLinkResolver::SetConf(const GMLASXLinkResolutionConf &oConf)
52 : {
53 171 : m_oConf = oConf;
54 171 : SetCacheDirectory(m_oConf.m_osCacheDirectory);
55 171 : }
56 :
57 : /************************************************************************/
58 : /* FetchRawContent() */
59 : /************************************************************************/
60 :
61 10 : CPLString GMLASXLinkResolver::FetchRawContent(const CPLString &osURL,
62 : const char *pszHeaders)
63 : {
64 10 : char **papszOptions = nullptr;
65 10 : if (m_oConf.m_nMaxGlobalResolutionTime > 0 &&
66 0 : m_nGlobalResolutionTime > m_oConf.m_nMaxGlobalResolutionTime)
67 : {
68 0 : CPLError(CE_Failure, CPLE_AppDefined,
69 : "Maximum global resolution time has been reached. "
70 : "No remote resource will be fetched");
71 0 : return CPLString();
72 : }
73 10 : if (m_oConf.m_nTimeOut > 0 || m_oConf.m_nMaxGlobalResolutionTime > 0)
74 : {
75 0 : int nTimeout = m_oConf.m_nTimeOut;
76 0 : if (m_oConf.m_nTimeOut > 0 && m_oConf.m_nMaxGlobalResolutionTime > 0)
77 : {
78 : // Select the minimum between the individual timeout and the
79 : // remaining time granted by the max global resolution time.
80 0 : int nRemaining =
81 0 : m_oConf.m_nMaxGlobalResolutionTime - m_nGlobalResolutionTime;
82 0 : if (nRemaining < nTimeout)
83 0 : nTimeout = nRemaining;
84 : }
85 0 : else if (m_oConf.m_nMaxGlobalResolutionTime > 0)
86 : {
87 0 : nTimeout =
88 0 : m_oConf.m_nMaxGlobalResolutionTime - m_nGlobalResolutionTime;
89 : }
90 0 : papszOptions = CSLSetNameValue(papszOptions, "TIMEOUT",
91 : CPLSPrintf("%d", nTimeout));
92 : }
93 10 : if (m_oConf.m_nMaxFileSize > 0)
94 : {
95 : papszOptions =
96 10 : CSLSetNameValue(papszOptions, "MAX_FILE_SIZE",
97 : CPLSPrintf("%d", m_oConf.m_nMaxFileSize));
98 : }
99 10 : if (!m_oConf.m_osProxyServerPort.empty())
100 : {
101 : papszOptions =
102 0 : CSLSetNameValue(papszOptions, "PROXY", m_oConf.m_osProxyServerPort);
103 : }
104 10 : if (!m_oConf.m_osProxyUserPassword.empty())
105 : {
106 0 : papszOptions = CSLSetNameValue(papszOptions, "PROXYUSERPWD",
107 : m_oConf.m_osProxyUserPassword);
108 : }
109 10 : if (!m_oConf.m_osProxyAuth.empty())
110 : {
111 : papszOptions =
112 0 : CSLSetNameValue(papszOptions, "PROXYAUTH", m_oConf.m_osProxyAuth);
113 : }
114 10 : if (pszHeaders != nullptr)
115 : {
116 1 : papszOptions = CSLSetNameValue(papszOptions, "HEADERS", pszHeaders);
117 : }
118 10 : time_t nTimeStart = time(nullptr);
119 10 : CPLHTTPResult *psResult = CPLHTTPFetch(osURL, papszOptions);
120 10 : time_t nTimeStop = time(nullptr);
121 10 : m_nGlobalResolutionTime += static_cast<int>(nTimeStop - nTimeStart);
122 10 : CSLDestroy(papszOptions);
123 10 : if (psResult == nullptr)
124 0 : return CPLString();
125 :
126 10 : if (psResult->nStatus != 0 || psResult->pabyData == nullptr)
127 : {
128 3 : CPLHTTPDestroyResult(psResult);
129 3 : return CPLString();
130 : }
131 :
132 14 : CPLString osResult;
133 7 : osResult.assign(reinterpret_cast<char *>(psResult->pabyData),
134 7 : psResult->nDataLen);
135 7 : CPLHTTPDestroyResult(psResult);
136 7 : return osResult;
137 : }
138 :
139 : /************************************************************************/
140 : /* GetRawContent() */
141 : /************************************************************************/
142 :
143 17 : CPLString GMLASXLinkResolver::GetRawContent(const CPLString &osURL,
144 : const char *pszHeaders,
145 : bool bAllowRemoteDownload,
146 : bool bCacheResults)
147 : {
148 17 : bool bDiskCacheAvailable = false;
149 17 : if (!m_osCacheDirectory.empty() && RecursivelyCreateDirectoryIfNeeded())
150 : {
151 17 : bDiskCacheAvailable = true;
152 :
153 17 : CPLString osCachedFileName(GetCachedFilename(osURL));
154 17 : VSILFILE *fp = nullptr;
155 18 : if (!m_bRefresh || m_aoSetRefreshedFiles.find(osCachedFileName) !=
156 18 : m_aoSetRefreshedFiles.end())
157 : {
158 16 : fp = VSIFOpenL(osCachedFileName, "rb");
159 : }
160 17 : if (fp != nullptr)
161 : {
162 4 : CPLDebug("GMLAS", "Use cached %s", osCachedFileName.c_str());
163 4 : GByte *pabyRet = nullptr;
164 4 : vsi_l_offset nSize = 0;
165 8 : CPLString osContent;
166 4 : if (VSIIngestFile(fp, nullptr, &pabyRet, &nSize, -1))
167 : {
168 : osContent.assign(reinterpret_cast<const char *>(pabyRet),
169 4 : static_cast<size_t>(nSize));
170 : }
171 4 : VSIFree(pabyRet);
172 4 : VSIFCloseL(fp);
173 4 : return osContent;
174 : }
175 13 : else if (bAllowRemoteDownload)
176 : {
177 12 : if (m_bRefresh)
178 1 : m_aoSetRefreshedFiles.insert(osCachedFileName);
179 : }
180 : else
181 : {
182 1 : CPLDebug("GMLAS",
183 : "Could not find locally cached %s, and not allowed to"
184 : "download it",
185 : osURL.c_str());
186 1 : return CPLString();
187 : }
188 : }
189 :
190 : // Check memory cache first
191 : {
192 12 : const auto oIter = m_oMapURLToContent.find(osURL);
193 12 : if (oIter != m_oMapURLToContent.end())
194 2 : return oIter->second;
195 : }
196 :
197 20 : const CPLString osContent(FetchRawContent(osURL, pszHeaders));
198 : // Cache to disk if possible
199 10 : if (bDiskCacheAvailable && bCacheResults && !osContent.empty())
200 : {
201 10 : CPLString osCachedFileName(GetCachedFilename(osURL));
202 10 : CPLString osTmpfilename(osCachedFileName + ".tmp");
203 5 : VSILFILE *fpTemp = VSIFOpenL(osTmpfilename, "wb");
204 5 : if (fpTemp != nullptr)
205 : {
206 : const bool bSuccess =
207 5 : VSIFWriteL(osContent.data(), osContent.size(), 1, fpTemp) == 1;
208 5 : VSIFCloseL(fpTemp);
209 5 : if (bSuccess)
210 5 : VSIRename(osTmpfilename, osCachedFileName);
211 : }
212 : }
213 : // Otherwise to RAM
214 5 : else if (!osContent.empty() && osContent.size() < m_nMaxRAMCacheSize)
215 : {
216 : // If cache is going to be saturated, evict larger objects first
217 3 : while (osContent.size() + m_nCurrentRAMCacheSize > m_nMaxRAMCacheSize)
218 : {
219 : std::map<size_t, std::vector<CPLString>>::reverse_iterator oIter =
220 1 : m_oMapFileSizeToURLs.rbegin();
221 1 : const size_t nSizeToEvict = oIter->first;
222 1 : m_nCurrentRAMCacheSize -= nSizeToEvict;
223 2 : const CPLString osURLToEvict(oIter->second.front());
224 1 : m_oMapURLToContent.erase(osURLToEvict);
225 1 : oIter->second.erase(oIter->second.begin());
226 1 : if (oIter->second.empty())
227 1 : m_oMapFileSizeToURLs.erase(nSizeToEvict);
228 : }
229 2 : m_oMapURLToContent[osURL] = osContent;
230 2 : m_oMapFileSizeToURLs[osContent.size()].push_back(osURL);
231 2 : m_nCurrentRAMCacheSize += osContent.size();
232 : }
233 10 : return osContent;
234 : }
235 :
236 : /************************************************************************/
237 : /* IsRawContentResolutionEnabled() */
238 : /************************************************************************/
239 :
240 21 : bool GMLASXLinkResolver::IsRawContentResolutionEnabled() const
241 : {
242 32 : return m_oConf.m_bDefaultResolutionEnabled &&
243 11 : m_oConf.m_eDefaultResolutionMode ==
244 21 : GMLASXLinkResolutionConf::RawContent;
245 : }
246 :
247 : /************************************************************************/
248 : /* GetMatchingResolutionRule() */
249 : /************************************************************************/
250 :
251 35 : int GMLASXLinkResolver::GetMatchingResolutionRule(const CPLString &osURL) const
252 : {
253 45 : for (size_t i = 0; i < m_oConf.m_aoURLSpecificRules.size(); ++i)
254 : {
255 24 : if (osURL.compare(0,
256 24 : m_oConf.m_aoURLSpecificRules[i].m_osURLPrefix.size(),
257 48 : m_oConf.m_aoURLSpecificRules[i].m_osURLPrefix) == 0)
258 : {
259 14 : return static_cast<int>(i);
260 : }
261 : }
262 :
263 : // No match
264 21 : return -1;
265 : }
266 :
267 : /************************************************************************/
268 : /* GetRawContent() */
269 : /************************************************************************/
270 :
271 11 : CPLString GMLASXLinkResolver::GetRawContent(const CPLString &osURL)
272 : {
273 11 : return GetRawContent(osURL, nullptr, m_oConf.m_bDefaultAllowRemoteDownload,
274 11 : m_oConf.m_bDefaultCacheResults);
275 : }
276 :
277 : /************************************************************************/
278 : /* GetRawContentForRule() */
279 : /************************************************************************/
280 :
281 6 : CPLString GMLASXLinkResolver::GetRawContentForRule(const CPLString &osURL,
282 : int nIdxRule)
283 : {
284 : const GMLASXLinkResolutionConf::URLSpecificResolution &oRule(
285 6 : m_oConf.m_aoURLSpecificRules[nIdxRule]);
286 :
287 12 : CPLString osHeaders;
288 12 : for (size_t i = 0; i < oRule.m_aosNameValueHTTPHeaders.size(); ++i)
289 : {
290 6 : if (!osHeaders.empty())
291 3 : osHeaders += "\r\n";
292 6 : osHeaders += oRule.m_aosNameValueHTTPHeaders[i].first;
293 6 : osHeaders += ": ";
294 6 : osHeaders += oRule.m_aosNameValueHTTPHeaders[i].second;
295 : }
296 3 : return GetRawContent(osURL, osHeaders.empty() ? nullptr : osHeaders.c_str(),
297 15 : oRule.m_bAllowRemoteDownload, oRule.m_bCacheResults);
298 : }
|
