Line data Source code
1 : /******************************************************************************
2 : *
3 : * Project: libcurl based HTTP client
4 : * Purpose: libcurl based HTTP client
5 : * Author: Frank Warmerdam, warmerdam@pobox.com
6 : *
7 : ******************************************************************************
8 : * Copyright (c) 2006, Frank Warmerdam
9 : * Copyright (c) 2008-2013, Even Rouault <even dot rouault at spatialys.com>
10 : *
11 : * Permission is hereby granted, free of charge, to any person obtaining a
12 : * copy of this software and associated documentation files (the "Software"),
13 : * to deal in the Software without restriction, including without limitation
14 : * the rights to use, copy, modify, merge, publish, distribute, sublicense,
15 : * and/or sell copies of the Software, and to permit persons to whom the
16 : * Software is furnished to do so, subject to the following conditions:
17 : *
18 : * The above copyright notice and this permission notice shall be included
19 : * in all copies or substantial portions of the Software.
20 : *
21 : * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
22 : * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
23 : * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
24 : * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
25 : * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
26 : * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
27 : * DEALINGS IN THE SOFTWARE.
28 : ****************************************************************************/
29 :
30 : #include "cpl_port.h"
31 : #include "cpl_http.h"
32 :
33 : #include <cstddef>
34 : #include <cstring>
35 :
36 : #include <algorithm>
37 : #include <array>
38 : #include <map>
39 : #include <mutex>
40 : #include <string>
41 : #include <vector>
42 :
43 : #include "cpl_http.h"
44 : #include "cpl_error.h"
45 : #include "cpl_multiproc.h"
46 : #include "cpl_vsil_curl_class.h"
47 :
48 : // gcc or clang complains about C-style cast in #define like
49 : // CURL_ZERO_TERMINATED
50 : #if defined(__GNUC__)
51 : #pragma GCC diagnostic push
52 : #pragma GCC diagnostic ignored "-Wold-style-cast"
53 : #endif
54 :
55 : #ifdef HAVE_CURL
56 :
57 : #include "cpl_curl_priv.h"
58 :
59 : #ifdef HAVE_OPENSSL_CRYPTO
60 : #include <openssl/err.h>
61 : #include <openssl/ssl.h>
62 : #include <openssl/engine.h>
63 : #include <openssl/x509v3.h>
64 :
65 : #if defined(_WIN32)
66 : #include <wincrypt.h>
67 : #endif
68 :
69 : #endif
70 :
71 : #ifdef HAVE_SIGACTION
72 : #include <signal.h>
73 : #endif
74 :
75 : #define unchecked_curl_easy_setopt(handle, opt, param) \
76 : CPL_IGNORE_RET_VAL(curl_easy_setopt(handle, opt, param))
77 :
78 : #endif // HAVE_CURL
79 :
80 : // list of named persistent http sessions
81 :
82 : #ifdef HAVE_CURL
83 : static std::map<CPLString, CURL *> *poSessionMap = nullptr;
84 : static std::map<CPLString, CURLM *> *poSessionMultiMap = nullptr;
85 : static CPLMutex *hSessionMapMutex = nullptr;
86 : static bool bHasCheckVersion = false;
87 : static bool bSupportGZip = false;
88 : static bool bSupportHTTP2 = false;
89 : #if defined(_WIN32) && defined(HAVE_OPENSSL_CRYPTO)
90 : static std::vector<X509 *> *poWindowsCertificateList = nullptr;
91 :
92 : #if (OPENSSL_VERSION_NUMBER < 0x10100000L)
93 : #define EVP_PKEY_get0_RSA(x) (x->pkey.rsa)
94 : #define EVP_PKEY_get0_DSA(x) (x->pkey.dsa)
95 : #define X509_get_extension_flags(x) (x->ex_flags)
96 : #define X509_get_key_usage(x) (x->ex_kusage)
97 : #define X509_get_extended_key_usage(x) (x->ex_xkusage)
98 : #endif
99 :
100 : #endif // defined(_WIN32) && defined(HAVE_OPENSSL_CRYPTO)
101 :
102 : #if defined(HAVE_OPENSSL_CRYPTO) && OPENSSL_VERSION_NUMBER < 0x10100000
103 :
104 : // Ported from https://curl.haxx.se/libcurl/c/opensslthreadlock.html
105 : static CPLMutex **pahSSLMutex = nullptr;
106 :
107 : static void CPLOpenSSLLockingFunction(int mode, int n, const char * /*file*/,
108 : int /*line*/)
109 : {
110 : if (mode & CRYPTO_LOCK)
111 : {
112 : CPLAcquireMutex(pahSSLMutex[n], 3600.0);
113 : }
114 : else
115 : {
116 : CPLReleaseMutex(pahSSLMutex[n]);
117 : }
118 : }
119 :
120 : static unsigned long CPLOpenSSLIdCallback(void)
121 : {
122 : return static_cast<unsigned long>(CPLGetPID());
123 : }
124 :
125 : static void CPLOpenSSLInit()
126 : {
127 : if (strstr(curl_version(), "OpenSSL") &&
128 : CPLTestBool(CPLGetConfigOption("CPL_OPENSSL_INIT_ENABLED", "YES")) &&
129 : CRYPTO_get_id_callback() == nullptr)
130 : {
131 : pahSSLMutex = static_cast<CPLMutex **>(
132 : CPLMalloc(CRYPTO_num_locks() * sizeof(CPLMutex *)));
133 : for (int i = 0; i < CRYPTO_num_locks(); i++)
134 : {
135 : pahSSLMutex[i] = CPLCreateMutex();
136 : CPLReleaseMutex(pahSSLMutex[i]);
137 : }
138 : CRYPTO_set_id_callback(CPLOpenSSLIdCallback);
139 : CRYPTO_set_locking_callback(CPLOpenSSLLockingFunction);
140 : }
141 : }
142 :
143 : static void CPLOpenSSLCleanup()
144 : {
145 : if (pahSSLMutex)
146 : {
147 : for (int i = 0; i < CRYPTO_num_locks(); i++)
148 : {
149 : CPLDestroyMutex(pahSSLMutex[i]);
150 : }
151 : CPLFree(pahSSLMutex);
152 : pahSSLMutex = nullptr;
153 : CRYPTO_set_id_callback(nullptr);
154 : CRYPTO_set_locking_callback(nullptr);
155 : }
156 : }
157 :
158 : #endif
159 :
160 : #if defined(_WIN32) && defined(HAVE_OPENSSL_CRYPTO)
161 :
162 : /************************************************************************/
163 : /* CPLWindowsCertificateListCleanup() */
164 : /************************************************************************/
165 :
166 : static void CPLWindowsCertificateListCleanup()
167 : {
168 : if (poWindowsCertificateList)
169 : {
170 : for (auto &&pX509 : *poWindowsCertificateList)
171 : {
172 : X509_free(pX509);
173 : }
174 : delete poWindowsCertificateList;
175 : poWindowsCertificateList = nullptr;
176 : }
177 : }
178 :
179 : /************************************************************************/
180 : /* LoadCAPICertificates() */
181 : /************************************************************************/
182 :
183 : static CPLErr LoadCAPICertificates(const char *pszName,
184 : std::vector<X509 *> *poCertificateList)
185 : {
186 : CPLAssert(pszName);
187 : CPLAssert(poCertificateList);
188 :
189 : HCERTSTORE pCertStore = CertOpenSystemStore(
190 : reinterpret_cast<HCRYPTPROV_LEGACY>(nullptr), pszName);
191 : if (pCertStore == nullptr)
192 : {
193 : CPLError(CE_Failure, CPLE_AppDefined,
194 : "CPLLoadCAPICertificates(): Unable open system "
195 : "certificate store %s.",
196 : pszName);
197 : return CE_Failure;
198 : }
199 :
200 : PCCERT_CONTEXT pCertificate =
201 : CertEnumCertificatesInStore(pCertStore, nullptr);
202 : while (pCertificate != nullptr)
203 : {
204 : X509 *pX509 = d2i_X509(
205 : nullptr,
206 : const_cast<unsigned char const **>(&pCertificate->pbCertEncoded),
207 : pCertificate->cbCertEncoded);
208 : if (pX509 == nullptr)
209 : {
210 : CPLError(CE_Warning, CPLE_AppDefined,
211 : "CPLLoadCAPICertificates(): CertEnumCertificatesInStore() "
212 : "returned a null certificate, skipping.");
213 : }
214 : else
215 : {
216 : #ifdef DEBUG_VERBOSE
217 : char szSubject[256] = {0};
218 : CPLString osSubject;
219 : X509_NAME *pName = X509_get_subject_name(pX509);
220 : if (pName)
221 : {
222 : X509_NAME_oneline(pName, szSubject, sizeof(szSubject));
223 : osSubject = szSubject;
224 : }
225 : if (!osSubject.empty())
226 : CPLDebug("HTTP", "SSL Certificate: %s", osSubject.c_str());
227 : #endif
228 : poCertificateList->push_back(pX509);
229 : }
230 : pCertificate = CertEnumCertificatesInStore(pCertStore, pCertificate);
231 : }
232 : CertCloseStore(pCertStore, 0);
233 : return CE_None;
234 : }
235 :
236 : /************************************************************************/
237 : /* CPL_ssl_ctx_callback() */
238 : /************************************************************************/
239 :
240 : // Load certificates from Windows Crypto API store.
241 : static CURLcode CPL_ssl_ctx_callback(CURL *, void *pSSL, void *)
242 : {
243 : SSL_CTX *pSSL_CTX = static_cast<SSL_CTX *>(pSSL);
244 : if (pSSL_CTX == nullptr)
245 : {
246 : CPLError(CE_Failure, CPLE_AppDefined,
247 : "CPL_ssl_ctx_callback(): OpenSSL context pointer is NULL.");
248 : return CURLE_ABORTED_BY_CALLBACK;
249 : }
250 :
251 : static std::mutex goMutex;
252 : {
253 : std::lock_guard<std::mutex> oLock(goMutex);
254 : if (poWindowsCertificateList == nullptr)
255 : {
256 : poWindowsCertificateList = new std::vector<X509 *>();
257 : if (!poWindowsCertificateList)
258 : {
259 : CPLError(CE_Failure, CPLE_AppDefined,
260 : "CPL_ssl_ctx_callback(): Unable to allocate "
261 : "structure to hold certificates.");
262 : return CURLE_FAILED_INIT;
263 : }
264 :
265 : const std::array<const char *, 3> aszStores{
266 : {"CA", "AuthRoot", "ROOT"}};
267 : for (auto &&pszStore : aszStores)
268 : {
269 : if (LoadCAPICertificates(pszStore, poWindowsCertificateList) ==
270 : CE_Failure)
271 : {
272 : CPLError(
273 : CE_Failure, CPLE_AppDefined,
274 : "CPL_ssl_ctx_callback(): Unable to load certificates "
275 : "from '%s' store.",
276 : pszStore);
277 : return CURLE_FAILED_INIT;
278 : }
279 : }
280 :
281 : CPLDebug("HTTP", "Loading %d certificates from Windows store.",
282 : static_cast<int>(poWindowsCertificateList->size()));
283 : }
284 : }
285 :
286 : X509_STORE *pX509Store = SSL_CTX_get_cert_store(pSSL_CTX);
287 : for (X509 *x509 : *poWindowsCertificateList)
288 : X509_STORE_add_cert(pX509Store, x509);
289 :
290 : return CURLE_OK;
291 : }
292 :
293 : #endif // defined(_WIN32) && defined (HAVE_OPENSSL_CRYPTO)
294 :
295 : /************************************************************************/
296 : /* CheckCurlFeatures() */
297 : /************************************************************************/
298 :
299 2241 : static void CheckCurlFeatures()
300 : {
301 4482 : CPLMutexHolder oHolder(&hSessionMapMutex);
302 2241 : if (!bHasCheckVersion)
303 : {
304 6 : const char *pszVersion = curl_version();
305 6 : CPLDebug("HTTP", "%s", pszVersion);
306 6 : bSupportGZip = strstr(pszVersion, "zlib/") != nullptr;
307 6 : bSupportHTTP2 = strstr(curl_version(), "nghttp2/") != nullptr;
308 6 : bHasCheckVersion = true;
309 :
310 6 : curl_version_info_data *data = curl_version_info(CURLVERSION_NOW);
311 6 : if (data->version_num < LIBCURL_VERSION_NUM)
312 : {
313 0 : CPLError(CE_Warning, CPLE_AppDefined,
314 : "GDAL was built against curl %d.%d.%d, but is "
315 : "running against %s. Runtime failure is likely !",
316 : LIBCURL_VERSION_MAJOR, LIBCURL_VERSION_MINOR,
317 : LIBCURL_VERSION_PATCH, data->version);
318 : }
319 6 : else if (data->version_num > LIBCURL_VERSION_NUM)
320 : {
321 0 : CPLDebug("HTTP",
322 : "GDAL was built against curl %d.%d.%d, but is "
323 : "running against %s.",
324 : LIBCURL_VERSION_MAJOR, LIBCURL_VERSION_MINOR,
325 : LIBCURL_VERSION_PATCH, data->version);
326 : }
327 :
328 : #if defined(HAVE_OPENSSL_CRYPTO) && OPENSSL_VERSION_NUMBER < 0x10100000
329 : CPLOpenSSLInit();
330 : #endif
331 : }
332 2241 : }
333 :
334 : /************************************************************************/
335 : /* CPLWriteFct() */
336 : /* */
337 : /* Append incoming text to our collection buffer, reallocating */
338 : /* it larger as needed. */
339 : /************************************************************************/
340 :
341 : class CPLHTTPResultWithLimit
342 : {
343 : public:
344 : CPLHTTPResult *psResult = nullptr;
345 : int nMaxFileSize = 0;
346 : };
347 :
348 12049 : static size_t CPLWriteFct(void *buffer, size_t size, size_t nmemb,
349 : void *reqInfo)
350 :
351 : {
352 12049 : CPLHTTPResultWithLimit *psResultWithLimit =
353 : static_cast<CPLHTTPResultWithLimit *>(reqInfo);
354 12049 : CPLHTTPResult *psResult = psResultWithLimit->psResult;
355 :
356 12049 : int nBytesToWrite = static_cast<int>(nmemb) * static_cast<int>(size);
357 12049 : int nNewSize = psResult->nDataLen + nBytesToWrite + 1;
358 12049 : if (nNewSize > psResult->nDataAlloc)
359 : {
360 1552 : psResult->nDataAlloc = static_cast<int>(nNewSize * 1.25 + 100);
361 : GByte *pabyNewData = static_cast<GByte *>(
362 1552 : VSIRealloc(psResult->pabyData, psResult->nDataAlloc));
363 1552 : if (pabyNewData == nullptr)
364 : {
365 0 : VSIFree(psResult->pabyData);
366 0 : psResult->pabyData = nullptr;
367 0 : psResult->pszErrBuf = CPLStrdup(CPLString().Printf(
368 : "Out of memory allocating %d bytes for HTTP data buffer.",
369 0 : psResult->nDataAlloc));
370 0 : psResult->nDataAlloc = psResult->nDataLen = 0;
371 :
372 0 : return 0;
373 : }
374 1552 : psResult->pabyData = pabyNewData;
375 : }
376 :
377 12049 : memcpy(psResult->pabyData + psResult->nDataLen, buffer, nBytesToWrite);
378 :
379 12049 : psResult->nDataLen += nBytesToWrite;
380 12049 : psResult->pabyData[psResult->nDataLen] = 0;
381 :
382 12049 : if (psResultWithLimit->nMaxFileSize > 0 &&
383 8 : psResult->nDataLen > psResultWithLimit->nMaxFileSize)
384 : {
385 1 : CPLError(CE_Failure, CPLE_AppDefined, "Maximum file size reached");
386 1 : return 0;
387 : }
388 :
389 12048 : return nmemb;
390 : }
391 :
392 : /************************************************************************/
393 : /* CPLHdrWriteFct() */
394 : /************************************************************************/
395 7530 : static size_t CPLHdrWriteFct(void *buffer, size_t size, size_t nmemb,
396 : void *reqInfo)
397 : {
398 7530 : CPLHTTPResult *psResult = static_cast<CPLHTTPResult *>(reqInfo);
399 : // Copy the buffer to a char* and initialize with zeros (zero
400 : // terminate as well).
401 7530 : size_t nBytes = size * nmemb;
402 7530 : char *pszHdr = static_cast<char *>(CPLCalloc(1, nBytes + 1));
403 7530 : memcpy(pszHdr, buffer, nBytes);
404 7530 : size_t nIdx = nBytes - 1;
405 : // Remove end of line characters
406 21208 : while (nIdx > 0 && (pszHdr[nIdx] == '\r' || pszHdr[nIdx] == '\n'))
407 : {
408 13678 : pszHdr[nIdx] = 0;
409 13678 : nIdx--;
410 : }
411 7530 : char *pszKey = nullptr;
412 7530 : const char *pszValue = CPLParseNameValue(pszHdr, &pszKey);
413 7530 : if (pszKey && pszValue)
414 : {
415 5311 : psResult->papszHeaders =
416 5311 : CSLAddNameValue(psResult->papszHeaders, pszKey, pszValue);
417 : }
418 7530 : CPLFree(pszHdr);
419 7530 : CPLFree(pszKey);
420 7530 : return nmemb;
421 : }
422 :
423 : /************************************************************************/
424 : /* CPLHTTPReadFunction() */
425 : /************************************************************************/
426 0 : static size_t CPLHTTPReadFunction(char *buffer, size_t size, size_t nitems,
427 : void *arg)
428 : {
429 0 : return VSIFReadL(buffer, size, nitems, static_cast<VSILFILE *>(arg));
430 : }
431 :
432 : /************************************************************************/
433 : /* CPLHTTPSeekFunction() */
434 : /************************************************************************/
435 0 : static int CPLHTTPSeekFunction(void *arg, curl_off_t offset, int origin)
436 : {
437 0 : if (VSIFSeekL(static_cast<VSILFILE *>(arg), offset, origin) == 0)
438 0 : return CURL_SEEKFUNC_OK;
439 : else
440 0 : return CURL_SEEKFUNC_FAIL;
441 : }
442 :
443 : /************************************************************************/
444 : /* CPLHTTPFreeFunction() */
445 : /************************************************************************/
446 0 : static void CPLHTTPFreeFunction(void *arg)
447 : {
448 0 : VSIFCloseL(static_cast<VSILFILE *>(arg));
449 0 : }
450 :
451 : typedef struct
452 : {
453 : GDALProgressFunc pfnProgress;
454 : void *pProgressArg;
455 : } CurlProcessData, *CurlProcessDataL;
456 :
457 0 : static int NewProcessFunction(void *p, curl_off_t dltotal, curl_off_t dlnow,
458 : curl_off_t ultotal, curl_off_t ulnow)
459 : {
460 0 : CurlProcessDataL pData = static_cast<CurlProcessDataL>(p);
461 0 : if (nullptr != pData && pData->pfnProgress)
462 : {
463 0 : if (dltotal > 0)
464 : {
465 0 : const double dfDone = double(dlnow) / dltotal;
466 0 : return pData->pfnProgress(dfDone, "Downloading ...",
467 : pData->pProgressArg) == TRUE
468 0 : ? 0
469 0 : : 1;
470 : }
471 0 : else if (ultotal > 0)
472 : {
473 0 : const double dfDone = double(ulnow) / ultotal;
474 0 : return pData->pfnProgress(dfDone, "Uploading ...",
475 : pData->pProgressArg) == TRUE
476 0 : ? 0
477 0 : : 1;
478 : }
479 : }
480 0 : return 0;
481 : }
482 :
483 : #endif /* def HAVE_CURL */
484 :
485 : /************************************************************************/
486 : /* CPLHTTPSetDefaultUserAgent() */
487 : /************************************************************************/
488 :
489 : static std::string gosDefaultUserAgent;
490 :
491 : /**
492 : * \brief Set the default user agent.
493 : *
494 : * GDAL core will by default call this method with "GDAL/x.y.z" where x.y.z
495 : * is the GDAL version number (during driver initialization). Applications may
496 : * override it.
497 : *
498 : * @param pszUserAgent String (or nullptr to cancel the default user agent)
499 : *
500 : * @since GDAL 3.7
501 : */
502 1219 : void CPLHTTPSetDefaultUserAgent(const char *pszUserAgent)
503 : {
504 1219 : gosDefaultUserAgent = pszUserAgent ? pszUserAgent : "";
505 1219 : }
506 :
507 : /************************************************************************/
508 : /* CPLHTTPGetOptionsFromEnv() */
509 : /************************************************************************/
510 :
511 : typedef struct
512 : {
513 : const char *pszEnvVar;
514 : const char *pszOptionName;
515 : } TupleEnvVarOptionName;
516 :
517 : constexpr TupleEnvVarOptionName asAssocEnvVarOptionName[] = {
518 : {"GDAL_HTTP_VERSION", "HTTP_VERSION"},
519 : {"GDAL_HTTP_CONNECTTIMEOUT", "CONNECTTIMEOUT"},
520 : {"GDAL_HTTP_TIMEOUT", "TIMEOUT"},
521 : {"GDAL_HTTP_LOW_SPEED_TIME", "LOW_SPEED_TIME"},
522 : {"GDAL_HTTP_LOW_SPEED_LIMIT", "LOW_SPEED_LIMIT"},
523 : {"GDAL_HTTP_USERPWD", "USERPWD"},
524 : {"GDAL_HTTP_PROXY", "PROXY"},
525 : {"GDAL_HTTPS_PROXY", "HTTPS_PROXY"},
526 : {"GDAL_HTTP_PROXYUSERPWD", "PROXYUSERPWD"},
527 : {"GDAL_PROXY_AUTH", "PROXYAUTH"},
528 : {"GDAL_HTTP_NETRC", "NETRC"},
529 : {"GDAL_HTTP_NETRC_FILE", "NETRC_FILE"},
530 : {"GDAL_HTTP_MAX_RETRY", "MAX_RETRY"},
531 : {"GDAL_HTTP_RETRY_DELAY", "RETRY_DELAY"},
532 : {"GDAL_CURL_CA_BUNDLE", "CAINFO"},
533 : {"CURL_CA_BUNDLE", "CAINFO"},
534 : {"SSL_CERT_FILE", "CAINFO"},
535 : {"GDAL_HTTP_HEADER_FILE", "HEADER_FILE"},
536 : {"GDAL_HTTP_CAPATH", "CAPATH"},
537 : {"GDAL_HTTP_SSL_VERIFYSTATUS", "SSL_VERIFYSTATUS"},
538 : {"GDAL_HTTP_USE_CAPI_STORE", "USE_CAPI_STORE"},
539 : {"GDAL_HTTP_HEADERS", "HEADERS"},
540 : {"GDAL_HTTP_HEADER_FILE", "HEADER_FILE"},
541 : {"GDAL_HTTP_AUTH", "HTTPAUTH"},
542 : {"GDAL_GSSAPI_DELEGATION", "GSSAPI_DELEGATION"},
543 : {"GDAL_HTTP_BEARER", "HTTP_BEARER"},
544 : {"GDAL_HTTP_COOKIE", "COOKIE"},
545 : {"GDAL_HTTP_COOKIEFILE", "COOKIEFILE"},
546 : {"GDAL_HTTP_COOKIEJAR", "COOKIEJAR"},
547 : {"GDAL_HTTP_MAX_RETRY", "MAX_RETRY"},
548 : {"GDAL_HTTP_RETRY_DELAY", "RETRY_DELAY"},
549 : {"GDAL_HTTP_TCP_KEEPALIVE", "TCP_KEEPALIVE"},
550 : {"GDAL_HTTP_TCP_KEEPIDLE", "TCP_KEEPIDLE"},
551 : {"GDAL_HTTP_TCP_KEEPINTVL", "TCP_KEEPINTVL"},
552 : };
553 :
554 1152 : char **CPLHTTPGetOptionsFromEnv(const char *pszFilename)
555 : {
556 1152 : char **papszOptions = nullptr;
557 40320 : for (size_t i = 0; i < CPL_ARRAYSIZE(asAssocEnvVarOptionName); ++i)
558 : {
559 : const char *pszVal =
560 : pszFilename
561 78336 : ? VSIGetPathSpecificOption(pszFilename,
562 39168 : asAssocEnvVarOptionName[i].pszEnvVar,
563 : nullptr)
564 0 : : CPLGetConfigOption(asAssocEnvVarOptionName[i].pszEnvVar,
565 39168 : nullptr);
566 39168 : if (pszVal != nullptr)
567 : {
568 129 : papszOptions = CSLSetNameValue(
569 129 : papszOptions, asAssocEnvVarOptionName[i].pszOptionName, pszVal);
570 : }
571 : }
572 1152 : return papszOptions;
573 : }
574 :
575 : /************************************************************************/
576 : /* CPLHTTPGetNewRetryDelay() */
577 : /************************************************************************/
578 :
579 386 : double CPLHTTPGetNewRetryDelay(int response_code, double dfOldDelay,
580 : const char *pszErrBuf, const char *pszCurlError)
581 : {
582 386 : if (response_code == 429 || response_code == 500 ||
583 309 : (response_code >= 502 && response_code <= 504) ||
584 : // S3 sends some client timeout errors as 400 Client Error
585 24 : (response_code == 400 && pszErrBuf &&
586 293 : strstr(pszErrBuf, "RequestTimeout")) ||
587 293 : (pszCurlError && (strstr(pszCurlError, "Connection timed out") ||
588 293 : strstr(pszCurlError, "Operation timed out") ||
589 293 : strstr(pszCurlError, "Connection reset by peer") ||
590 293 : strstr(pszCurlError, "Connection was reset"))))
591 : {
592 : // 'Operation tmied out': seen during some long running operation 'hang'
593 : // no error but no response from server and we are in the cURL loop
594 : // infinitely.
595 :
596 : // 'Connection was reset': was found with Azure: server resets
597 : // connection during TLS handshake (10054 error code). It seems like
598 : // the server process crashed or something forced TCP reset;
599 : // the request succeeds on retry.
600 :
601 : // Use an exponential backoff factor of 2 plus some random jitter
602 : // We don't care about cryptographic quality randomness, hence:
603 : // coverity[dont_call]
604 98 : return dfOldDelay * (2 + rand() * 0.5 / RAND_MAX);
605 : }
606 : else
607 : {
608 288 : return 0;
609 : }
610 : }
611 :
612 : #ifdef HAVE_CURL
613 :
614 : /************************************************************************/
615 : /* CPLHTTPEmitFetchDebug() */
616 : /************************************************************************/
617 :
618 1150 : static void CPLHTTPEmitFetchDebug(const char *pszURL,
619 : const char *pszExtraDebug = "")
620 : {
621 1150 : const char *pszArobase = strchr(pszURL, '@');
622 1150 : const char *pszSlash = strchr(pszURL, '/');
623 1150 : const char *pszColon = (pszSlash) ? strchr(pszSlash, ':') : nullptr;
624 1150 : if (pszArobase != nullptr && pszColon != nullptr &&
625 0 : pszArobase - pszColon > 0)
626 : {
627 : /* http://user:password@www.example.com */
628 0 : char *pszSanitizedURL = CPLStrdup(pszURL);
629 0 : pszSanitizedURL[pszColon - pszURL] = 0;
630 0 : CPLDebug("HTTP", "Fetch(%s:#password#%s%s)", pszSanitizedURL,
631 : pszArobase, pszExtraDebug);
632 0 : CPLFree(pszSanitizedURL);
633 : }
634 : else
635 : {
636 1150 : CPLDebug("HTTP", "Fetch(%s%s)", pszURL, pszExtraDebug);
637 : }
638 1150 : }
639 :
640 : #endif
641 :
642 : #ifdef HAVE_CURL
643 :
644 : /************************************************************************/
645 : /* class CPLHTTPPostFields */
646 : /************************************************************************/
647 :
648 : class CPLHTTPPostFields
649 : {
650 : public:
651 1150 : CPLHTTPPostFields() = default;
652 : CPLHTTPPostFields &operator=(const CPLHTTPPostFields &) = delete;
653 : CPLHTTPPostFields(const CPLHTTPPostFields &) = delete;
654 :
655 1150 : CPLErr Fill(CURL *http_handle, CSLConstList papszOptions)
656 : {
657 : // Fill POST form if present
658 : const char *pszFormFilePath =
659 1150 : CSLFetchNameValue(papszOptions, "FORM_FILE_PATH");
660 : const char *pszParametersCount =
661 1150 : CSLFetchNameValue(papszOptions, "FORM_ITEM_COUNT");
662 :
663 1150 : if (pszFormFilePath != nullptr || pszParametersCount != nullptr)
664 : {
665 2 : mime = curl_mime_init(http_handle);
666 2 : curl_mimepart *mimepart = curl_mime_addpart(mime);
667 2 : if (pszFormFilePath != nullptr)
668 : {
669 : const char *pszFormFileName =
670 1 : CSLFetchNameValue(papszOptions, "FORM_FILE_NAME");
671 1 : const char *pszFilename = CPLGetFilename(pszFormFilePath);
672 1 : if (pszFormFileName == nullptr)
673 : {
674 1 : pszFormFileName = pszFilename;
675 : }
676 :
677 : VSIStatBufL sStat;
678 1 : if (VSIStatL(pszFormFilePath, &sStat) == 0)
679 : {
680 0 : VSILFILE *mime_fp = VSIFOpenL(pszFormFilePath, "rb");
681 0 : if (mime_fp != nullptr)
682 : {
683 0 : curl_mime_name(mimepart, pszFormFileName);
684 0 : CPL_IGNORE_RET_VAL(
685 0 : curl_mime_filename(mimepart, pszFilename));
686 0 : curl_mime_data_cb(
687 : mimepart, sStat.st_size, CPLHTTPReadFunction,
688 : CPLHTTPSeekFunction, CPLHTTPFreeFunction, mime_fp);
689 : }
690 : else
691 : {
692 : osErrMsg = CPLSPrintf("Failed to open file %s",
693 0 : pszFormFilePath);
694 1 : return CE_Failure;
695 : }
696 :
697 0 : CPLDebug("HTTP", "Send file: %s, COPYNAME: %s",
698 : pszFormFilePath, pszFormFileName);
699 : }
700 : else
701 : {
702 : osErrMsg =
703 1 : CPLSPrintf("File '%s' not found", pszFormFilePath);
704 1 : return CE_Failure;
705 : }
706 : }
707 :
708 1 : int nParametersCount = 0;
709 1 : if (pszParametersCount != nullptr)
710 : {
711 1 : nParametersCount = atoi(pszParametersCount);
712 : }
713 :
714 2 : for (int i = 0; i < nParametersCount; ++i)
715 : {
716 2 : const char *pszKey = CSLFetchNameValue(
717 : papszOptions, CPLSPrintf("FORM_KEY_%d", i));
718 2 : const char *pszValue = CSLFetchNameValue(
719 : papszOptions, CPLSPrintf("FORM_VALUE_%d", i));
720 :
721 2 : if (nullptr == pszKey)
722 : {
723 : osErrMsg = CPLSPrintf("Key #%d is not exists. Maybe wrong "
724 : "count of form items",
725 1 : i);
726 1 : return CE_Failure;
727 : }
728 :
729 1 : if (nullptr == pszValue)
730 : {
731 : osErrMsg = CPLSPrintf("Value #%d is not exists. Maybe "
732 : "wrong count of form items",
733 0 : i);
734 0 : return CE_Failure;
735 : }
736 :
737 1 : mimepart = curl_mime_addpart(mime);
738 1 : curl_mime_name(mimepart, pszKey);
739 1 : CPL_IGNORE_RET_VAL(
740 1 : curl_mime_data(mimepart, pszValue, CURL_ZERO_TERMINATED));
741 :
742 1 : CPLDebug("HTTP", "COPYNAME: %s, COPYCONTENTS: %s", pszKey,
743 : pszValue);
744 : }
745 :
746 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_MIMEPOST, mime);
747 : }
748 1148 : return CE_None;
749 : }
750 :
751 1150 : ~CPLHTTPPostFields()
752 1150 : {
753 1150 : if (mime != nullptr)
754 : {
755 2 : curl_mime_free(mime);
756 : }
757 1150 : }
758 :
759 2 : std::string GetErrorMessage() const
760 : {
761 2 : return osErrMsg;
762 : }
763 :
764 : private:
765 : curl_mime *mime = nullptr;
766 : std::string osErrMsg{};
767 : };
768 :
769 : /************************************************************************/
770 : /* CPLHTTPFetchCleanup() */
771 : /************************************************************************/
772 :
773 1150 : static void CPLHTTPFetchCleanup(CURL *http_handle, struct curl_slist *headers,
774 : const char *pszPersistent,
775 : CSLConstList papszOptions)
776 : {
777 1150 : if (CSLFetchNameValue(papszOptions, "POSTFIELDS"))
778 157 : unchecked_curl_easy_setopt(http_handle, CURLOPT_POST, 0);
779 1150 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTPHEADER, nullptr);
780 :
781 1150 : if (!pszPersistent)
782 662 : curl_easy_cleanup(http_handle);
783 :
784 1150 : curl_slist_free_all(headers);
785 1150 : }
786 : #endif // HAVE_CURL
787 :
788 : struct CPLHTTPFetchContext
789 : {
790 : std::vector<std::pair<CPLHTTPFetchCallbackFunc, void *>> stack{};
791 : };
792 :
793 : /************************************************************************/
794 : /* GetHTTPFetchContext() */
795 : /************************************************************************/
796 :
797 1340 : static CPLHTTPFetchContext *GetHTTPFetchContext(bool bAlloc)
798 : {
799 1340 : int bError = FALSE;
800 : CPLHTTPFetchContext *psCtx = static_cast<CPLHTTPFetchContext *>(
801 1340 : CPLGetTLSEx(CTLS_HTTPFETCHCALLBACK, &bError));
802 1340 : if (bError)
803 0 : return nullptr;
804 :
805 1340 : if (psCtx == nullptr && bAlloc)
806 : {
807 1 : const auto FreeFunc = [](void *pData)
808 1 : { delete static_cast<CPLHTTPFetchContext *>(pData); };
809 1 : psCtx = new CPLHTTPFetchContext();
810 1 : CPLSetTLSWithFreeFuncEx(CTLS_HTTPFETCHCALLBACK, psCtx, FreeFunc,
811 : &bError);
812 1 : if (bError)
813 : {
814 0 : delete psCtx;
815 0 : psCtx = nullptr;
816 : }
817 : }
818 1340 : return psCtx;
819 : }
820 :
821 : /************************************************************************/
822 : /* CPLHTTPSetFetchCallback() */
823 : /************************************************************************/
824 :
825 : static CPLHTTPFetchCallbackFunc gpsHTTPFetchCallbackFunc = nullptr;
826 : static void *gpHTTPFetchCallbackUserData = nullptr;
827 :
828 : /** Installs an alternate callback to the default implementation of
829 : * CPLHTTPFetchEx().
830 : *
831 : * This callback will be used by all threads, unless contextual callbacks are
832 : * installed with CPLHTTPPushFetchCallback().
833 : *
834 : * It is the responsibility of the caller to make sure this function is not
835 : * called concurrently, or during CPLHTTPFetchEx() execution.
836 : *
837 : * @param pFunc Callback function to be called with CPLHTTPFetchEx() is called
838 : * (or NULL to restore default handler)
839 : * @param pUserData Last argument to provide to the pFunc callback.
840 : *
841 : * @since GDAL 3.2
842 : */
843 2 : void CPLHTTPSetFetchCallback(CPLHTTPFetchCallbackFunc pFunc, void *pUserData)
844 : {
845 2 : gpsHTTPFetchCallbackFunc = pFunc;
846 2 : gpHTTPFetchCallbackUserData = pUserData;
847 2 : }
848 :
849 : /************************************************************************/
850 : /* CPLHTTPPushFetchCallback() */
851 : /************************************************************************/
852 :
853 : /** Installs an alternate callback to the default implementation of
854 : * CPLHTTPFetchEx().
855 : *
856 : * This callback will only be used in the thread where this function has been
857 : * called. It must be un-installed by CPLHTTPPopFetchCallback(), which must also
858 : * be called from the same thread.
859 : *
860 : * @param pFunc Callback function to be called with CPLHTTPFetchEx() is called.
861 : * @param pUserData Last argument to provide to the pFunc callback.
862 : * @return TRUE in case of success.
863 : *
864 : * @since GDAL 3.2
865 : */
866 1 : int CPLHTTPPushFetchCallback(CPLHTTPFetchCallbackFunc pFunc, void *pUserData)
867 : {
868 1 : auto psCtx = GetHTTPFetchContext(true);
869 1 : if (psCtx == nullptr)
870 0 : return false;
871 : psCtx->stack.emplace_back(
872 1 : std::pair<CPLHTTPFetchCallbackFunc, void *>(pFunc, pUserData));
873 1 : return true;
874 : }
875 :
876 : /************************************************************************/
877 : /* CPLHTTPPopFetchCallback() */
878 : /************************************************************************/
879 :
880 : /** Uninstalls a callback set by CPLHTTPPushFetchCallback().
881 : *
882 : * @see CPLHTTPPushFetchCallback()
883 : * @return TRUE in case of success.
884 : * @since GDAL 3.2
885 : */
886 2 : int CPLHTTPPopFetchCallback(void)
887 : {
888 2 : auto psCtx = GetHTTPFetchContext(false);
889 2 : if (psCtx == nullptr || psCtx->stack.empty())
890 : {
891 1 : CPLError(
892 : CE_Failure, CPLE_AppDefined,
893 : "CPLHTTPPushFetchCallback / CPLHTTPPopFetchCallback not balanced");
894 1 : return false;
895 : }
896 : else
897 : {
898 1 : psCtx->stack.pop_back();
899 1 : return true;
900 : }
901 : }
902 :
903 : /************************************************************************/
904 : /* CPLHTTPFetch() */
905 : /************************************************************************/
906 :
907 : // NOTE: when adding an option below, add it in asAssocEnvVarOptionName[]
908 :
909 : // clang-format off
910 : /**
911 : * \brief Fetch a document from an url and return in a string.
912 : *
913 : * @param pszURL valid URL recognized by underlying download library (libcurl)
914 : * @param papszOptions option list as a NULL-terminated array of strings. May be
915 : * NULL. The following options are handled : <ul>
916 : * <li>CONNECTTIMEOUT=val, where
917 : * val is in seconds (possibly with decimals). This is the maximum delay for the
918 : * connection to be established before being aborted (GDAL >= 2.2).</li>
919 : * <li>TIMEOUT=val, where val is in seconds. This is the maximum delay for the
920 : * whole request to complete before being aborted.</li>
921 : * <li>LOW_SPEED_TIME=val,
922 : * where val is in seconds. This is the maximum time where the transfer speed
923 : * should be below the LOW_SPEED_LIMIT (if not specified 1b/s), before the
924 : * transfer to be considered too slow and aborted. (GDAL >= 2.1)</li>
925 : * <li>LOW_SPEED_LIMIT=val, where val is in bytes/second. See LOW_SPEED_TIME.
926 : * Has only effect if LOW_SPEED_TIME is specified too. (GDAL >= 2.1)</li>
927 : * <li>HEADERS=val, where val is an extra header to use when getting a web page.
928 : * For example "Accept: application/x-ogcwkt"</li>
929 : * <li>HEADER_FILE=filename: filename of a text file with "key: value" headers.
930 : * The content of the file is not cached, and thus it is read again before
931 : * issuing each HTTP request. (GDAL >= 2.2)</li>
932 : * <li>HTTPAUTH=[BASIC/NTLM/NEGOTIATE/ANY/ANYSAFE/BEARER] to specify an
933 : * authentication scheme to use.</li>
934 : * <li>USERPWD=userid:password to specify a user and password for
935 : * authentication</li>
936 : * <li>GSSAPI_DELEGATION=[NONE/POLICY/ALWAYS] set allowed
937 : * GSS-API delegation. Relevant only with HTTPAUTH=NEGOTIATE (GDAL >= 3.3).</li>
938 : * <li>HTTP_BEARER=val set OAuth 2.0 Bearer Access Token.
939 : * Relevant only with HTTPAUTH=BEARER (GDAL >= 3.9).</li>
940 : * <li>POSTFIELDS=val, where val is a nul-terminated string to be passed to the
941 : * server with a POST request.</li>
942 : * <li>PROXY=val, to make requests go through a
943 : * proxy server, where val is of the form proxy.server.com:port_number. This
944 : * option affects both HTTP and HTTPS URLs.</li>
945 : * <li>HTTPS_PROXY=val (GDAL
946 : * >= 2.4), the same meaning as PROXY, but this option is taken into account
947 : * only for HTTPS URLs.</li>
948 : * <li>PROXYUSERPWD=val, where val is of the form
949 : * username:password</li>
950 : * <li>PROXYAUTH=[BASIC/NTLM/DIGEST/NEGOTIATE/ANY/ANYSAFE] to
951 : * specify an proxy authentication scheme to use.</li>
952 : * <li>NETRC=[YES/NO] to
953 : * enable or disable use of $HOME/.netrc (or NETRC_FILE), default YES.</li>
954 : * <li>NETRC_FILE=file name to read .netrc info from (GDAL >= 3.7).</li>
955 : * <li>CUSTOMREQUEST=val, where val is GET, PUT, POST, DELETE, etc.. (GDAL
956 : * >= 1.9.0)</li>
957 : * <li>FORM_FILE_NAME=val, where val is upload file name. If this
958 : * option and FORM_FILE_PATH present, request type will set to POST.</li>
959 : * <li>FORM_FILE_PATH=val, where val is upload file path.</li>
960 : * <li>FORM_KEY_0=val...FORM_KEY_N, where val is name of form item.</li>
961 : * <li>FORM_VALUE_0=val...FORM_VALUE_N, where val is value of the form
962 : * item.</li>
963 : * <li>FORM_ITEM_COUNT=val, where val is count of form items.</li>
964 : * <li>COOKIE=val, where val is formatted as COOKIE1=VALUE1; COOKIE2=VALUE2;
965 : * ...</li>
966 : * <li>COOKIEFILE=val, where val is file name to read cookies from
967 : * (GDAL >= 2.4)</li>
968 : * <li>COOKIEJAR=val, where val is file name to store cookies
969 : * to (GDAL >= 2.4)</li>
970 : * <li>MAX_RETRY=val, where val is the maximum number of
971 : * retry attempts if a 429, 502, 503 or 504 HTTP error occurs. Default is 0.
972 : * (GDAL >= 2.0)</li>
973 : * <li>RETRY_DELAY=val, where val is the number of seconds
974 : * between retry attempts. Default is 30. (GDAL >= 2.0)</li>
975 : * <li>MAX_FILE_SIZE=val, where val is a number of bytes (GDAL >= 2.2)</li>
976 : * <li>CAINFO=/path/to/bundle.crt. This is path to Certificate Authority (CA)
977 : * bundle file. By default, it will be looked for in a system location. If
978 : * the CAINFO option is not defined, GDAL will also look in the
979 : * CURL_CA_BUNDLE and SSL_CERT_FILE environment variables respectively
980 : * and use the first one found as the CAINFO value (GDAL >= 2.1.3). The
981 : * GDAL_CURL_CA_BUNDLE environment variable may also be used to set the
982 : * CAINFO value in GDAL >= 3.2.</li>
983 : * <li>HTTP_VERSION=1.0/1.1/2/2TLS (GDAL >= 2.3). Specify HTTP version to use.
984 : * Will default to 1.1 generally (except on some controlled environments,
985 : * like Google Compute Engine VMs, where 2TLS will be the default).
986 : * Support for HTTP/2 requires curl 7.33 or later, built against nghttp2.
987 : * "2TLS" means that HTTP/2 will be attempted for HTTPS connections only.
988 : * Whereas "2" means that HTTP/2 will be attempted for HTTP or HTTPS.</li>
989 : * <li>SSL_VERIFYSTATUS=YES/NO (GDAL >= 2.3, and curl >= 7.41): determines
990 : * whether the status of the server cert using the "Certificate Status Request"
991 : * TLS extension (aka. OCSP stapling) should be checked. If this option is
992 : * enabled but the server does not support the TLS extension, the verification
993 : * will fail. Default to NO.</li>
994 : * <li>USE_CAPI_STORE=YES/NO (GDAL >= 2.3,
995 : * Windows only): whether CA certificates from the Windows certificate store.
996 : * Defaults to NO.</li>
997 : * <li>TCP_KEEPALIVE=YES/NO (GDAL >= 3.6): whether to
998 : * enable TCP keep-alive. Defaults to NO</li> <li>TCP_KEEPIDLE=integer, in
999 : * seconds (GDAL >= 3.6): keep-alive idle time. Defaults to 60. Only taken into
1000 : * account if TCP_KEEPALIVE=YES.</li>
1001 : * <li>TCP_KEEPINTVL=integer, in seconds
1002 : * (GDAL >= 3.6): interval time between keep-alive probes. Defaults to 60. Only
1003 : * taken into account if TCP_KEEPALIVE=YES.</li>
1004 : * <li>USERAGENT=string: value of User-Agent header. Starting with GDAL 3.7,
1005 : * GDAL core sets it by default (during driver initialization) to GDAL/x.y.z
1006 : * where x.y.z is the GDAL version number. Applications may override it with the
1007 : * CPLHTTPSetDefaultUserAgent() function.</li>
1008 : * <li>SSLCERT=filename (GDAL >= 3.7): Filename of the the SSL client certificate.
1009 : * Cf https://curl.se/libcurl/c/CURLOPT_SSLCERT.html</li>
1010 : * <li>SSLCERTTYPE=string (GDAL >= 3.7): Format of the SSL certificate: "PEM"
1011 : * or "DER". Cf https://curl.se/libcurl/c/CURLOPT_SSLCERTTYPE.html</li>
1012 : * <li>SSLKEY=filename (GDAL >= 3.7): Private key file for TLS and SSL client
1013 : * certificate. Cf https://curl.se/libcurl/c/CURLOPT_SSLKEY.html</li>
1014 : * <li>KEYPASSWD=string (GDAL >= 3.7): Passphrase to private key.
1015 : * Cf https://curl.se/libcurl/c/CURLOPT_KEYPASSWD.html</li>
1016 : * </ul>
1017 : *
1018 : * Alternatively, if not defined in the papszOptions arguments, the following
1019 : * CONNECTTIMEOUT, TIMEOUT,
1020 : * LOW_SPEED_TIME, LOW_SPEED_LIMIT, USERPWD, PROXY, HTTPS_PROXY, PROXYUSERPWD,
1021 : * PROXYAUTH, NETRC, NETRC_FILE, MAX_RETRY and RETRY_DELAY, HEADER_FILE, HTTP_VERSION,
1022 : * SSL_VERIFYSTATUS, USE_CAPI_STORE, GSSAPI_DELEGATION, HTTP_BEARER, TCP_KEEPALIVE,
1023 : * TCP_KEEPIDLE, TCP_KEEPINTVL, USERAGENT, SSLCERT, SSLCERTTYPE, SSLKEY,
1024 : * KEYPASSWD values are searched in the configuration options
1025 : * respectively named GDAL_HTTP_CONNECTTIMEOUT, GDAL_HTTP_TIMEOUT,
1026 : * GDAL_HTTP_LOW_SPEED_TIME, GDAL_HTTP_LOW_SPEED_LIMIT, GDAL_HTTP_USERPWD,
1027 : * GDAL_HTTP_PROXY, GDAL_HTTPS_PROXY, GDAL_HTTP_PROXYUSERPWD, GDAL_PROXY_AUTH,
1028 : * GDAL_HTTP_NETRC, GDAL_HTTP_NETC_FILE, GDAL_HTTP_MAX_RETRY, GDAL_HTTP_RETRY_DELAY,
1029 : * GDAL_HTTP_HEADER_FILE, GDAL_HTTP_VERSION, GDAL_HTTP_SSL_VERIFYSTATUS,
1030 : * GDAL_HTTP_USE_CAPI_STORE, GDAL_GSSAPI_DELEGATION, GDAL_HTTP_BEARER,
1031 : * GDAL_HTTP_TCP_KEEPALIVE, GDAL_HTTP_TCP_KEEPIDLE, GDAL_HTTP_TCP_KEEPINTVL,
1032 : * GDAL_HTTP_USERAGENT, GDAL_HTTP_SSLCERT, GDAL_HTTP_SSLCERTTYPE,
1033 : * GDAL_HTTP_SSLKEY and GDAL_HTTP_KEYPASSWD.
1034 : *
1035 : * Starting with GDAL 3.6, the GDAL_HTTP_HEADERS configuration option can also
1036 : * be used to specify a comma separated list of key: value pairs. This is an
1037 : * alternative to the GDAL_HTTP_HEADER_FILE mechanism. If a comma or a
1038 : * double-quote character is needed in the value, then the key: value pair must
1039 : * be enclosed in double-quote characters. In that situation, backslash and
1040 : * double quote character must be backslash-escaped. e.g GDAL_HTTP_HEADERS=Foo:
1041 : * Bar,"Baz: escaped backslash \\, escaped double-quote \", end of
1042 : * value",Another: Header
1043 : *
1044 : * Starting with GDAL 3.7, the above configuration options can also be specified
1045 : * as path-specific options with VSISetPathSpecificOption().
1046 : *
1047 : * @return a CPLHTTPResult* structure that must be freed by
1048 : * CPLHTTPDestroyResult(), or NULL if libcurl support is disabled
1049 : */
1050 : // clang-format on
1051 2105 : CPLHTTPResult *CPLHTTPFetch(const char *pszURL, CSLConstList papszOptions)
1052 : {
1053 2105 : return CPLHTTPFetchEx(pszURL, papszOptions, nullptr, nullptr, nullptr,
1054 2105 : nullptr);
1055 : }
1056 :
1057 : /**
1058 : * Fetch a document from an url and return in a string.
1059 : * @param pszURL Url to fetch document from web.
1060 : * @param papszOptions Option list as a NULL-terminated array of strings.
1061 : * Available keys see in CPLHTTPFetch.
1062 : * @param pfnProgress Callback for reporting algorithm progress matching the
1063 : * GDALProgressFunc() semantics. May be NULL.
1064 : * @param pProgressArg Callback argument passed to pfnProgress.
1065 : * @param pfnWrite Write function pointer matching the CPLHTTPWriteFunc()
1066 : * semantics. May be NULL.
1067 : * @param pWriteArg Argument which will pass to a write function.
1068 : * @return A CPLHTTPResult* structure that must be freed by
1069 : * CPLHTTPDestroyResult(), or NULL if libcurl support is disabled.
1070 : */
1071 2111 : CPLHTTPResult *CPLHTTPFetchEx(const char *pszURL, CSLConstList papszOptions,
1072 : GDALProgressFunc pfnProgress, void *pProgressArg,
1073 : CPLHTTPFetchWriteFunc pfnWrite, void *pWriteArg)
1074 :
1075 : {
1076 2910 : if (STARTS_WITH(pszURL, "/vsimem/") &&
1077 : // Disabled by default for potential security issues.
1078 799 : CPLTestBool(CPLGetConfigOption("CPL_CURL_ENABLE_VSIMEM", "FALSE")))
1079 : {
1080 774 : CPLString osURL(pszURL);
1081 : const char *pszCustomRequest =
1082 774 : CSLFetchNameValue(papszOptions, "CUSTOMREQUEST");
1083 774 : if (pszCustomRequest != nullptr)
1084 : {
1085 5 : osURL += "&CUSTOMREQUEST=";
1086 5 : osURL += pszCustomRequest;
1087 : }
1088 774 : const char *pszUserPwd = CSLFetchNameValue(papszOptions, "USERPWD");
1089 774 : if (pszUserPwd != nullptr)
1090 : {
1091 1 : osURL += "&USERPWD=";
1092 1 : osURL += pszUserPwd;
1093 : }
1094 774 : const char *pszPost = CSLFetchNameValue(papszOptions, "POSTFIELDS");
1095 774 : if (pszPost != nullptr) // Hack: We append post content to filename.
1096 : {
1097 281 : osURL += "&POSTFIELDS=";
1098 281 : osURL += pszPost;
1099 : }
1100 774 : const char *pszHeaders = CSLFetchNameValue(papszOptions, "HEADERS");
1101 906 : if (pszHeaders != nullptr &&
1102 132 : CPLTestBool(
1103 : CPLGetConfigOption("CPL_CURL_VSIMEM_PRINT_HEADERS", "FALSE")))
1104 : {
1105 5 : osURL += "&HEADERS=";
1106 5 : osURL += pszHeaders;
1107 : }
1108 774 : vsi_l_offset nLength = 0;
1109 : CPLHTTPResult *psResult =
1110 774 : static_cast<CPLHTTPResult *>(CPLCalloc(1, sizeof(CPLHTTPResult)));
1111 774 : GByte *pabyData = VSIGetMemFileBuffer(osURL, &nLength, FALSE);
1112 774 : if (pabyData == nullptr)
1113 : {
1114 119 : CPLDebug("HTTP", "Cannot find %s", osURL.c_str());
1115 119 : psResult->nStatus = 1;
1116 119 : psResult->pszErrBuf =
1117 119 : CPLStrdup(CPLSPrintf("HTTP error code : %d", 404));
1118 119 : CPLError(CE_Failure, CPLE_AppDefined, "%s", psResult->pszErrBuf);
1119 : }
1120 655 : else if (nLength != 0)
1121 : {
1122 613 : psResult->nDataLen = static_cast<int>(nLength);
1123 613 : psResult->pabyData = static_cast<GByte *>(
1124 613 : CPLMalloc(static_cast<size_t>(nLength) + 1));
1125 613 : memcpy(psResult->pabyData, pabyData, static_cast<size_t>(nLength));
1126 613 : psResult->pabyData[static_cast<size_t>(nLength)] = 0;
1127 : }
1128 :
1129 774 : if (psResult->pabyData != nullptr &&
1130 613 : STARTS_WITH(reinterpret_cast<char *>(psResult->pabyData),
1131 : "Content-Type: "))
1132 : {
1133 7 : const char *pszContentType =
1134 7 : reinterpret_cast<char *>(psResult->pabyData) +
1135 : strlen("Content-type: ");
1136 7 : const char *pszEOL = strchr(pszContentType, '\r');
1137 7 : if (pszEOL)
1138 7 : pszEOL = strchr(pszContentType, '\n');
1139 7 : if (pszEOL)
1140 : {
1141 7 : size_t nContentLength = pszEOL - pszContentType;
1142 7 : psResult->pszContentType =
1143 7 : static_cast<char *>(CPLMalloc(nContentLength + 1));
1144 7 : memcpy(psResult->pszContentType, pszContentType,
1145 : nContentLength);
1146 7 : psResult->pszContentType[nContentLength] = 0;
1147 : }
1148 : }
1149 :
1150 774 : return psResult;
1151 : }
1152 :
1153 : // Try to user alternate network layer if set.
1154 1337 : auto pCtx = GetHTTPFetchContext(false);
1155 1337 : if (pCtx)
1156 : {
1157 3 : for (size_t i = pCtx->stack.size(); i > 0;)
1158 : {
1159 1 : --i;
1160 1 : const auto &cbk = pCtx->stack[i];
1161 1 : auto cbkFunc = cbk.first;
1162 1 : auto pUserData = cbk.second;
1163 1 : auto res = cbkFunc(pszURL, papszOptions, pfnProgress, pProgressArg,
1164 : pfnWrite, pWriteArg, pUserData);
1165 1 : if (res)
1166 : {
1167 1 : if (CSLFetchNameValue(papszOptions, "CLOSE_PERSISTENT"))
1168 : {
1169 0 : CPLHTTPDestroyResult(res);
1170 0 : res = nullptr;
1171 : }
1172 1 : return res;
1173 : }
1174 : }
1175 : }
1176 :
1177 1336 : if (gpsHTTPFetchCallbackFunc)
1178 : {
1179 1 : auto res = gpsHTTPFetchCallbackFunc(pszURL, papszOptions, pfnProgress,
1180 : pProgressArg, pfnWrite, pWriteArg,
1181 : gpHTTPFetchCallbackUserData);
1182 1 : if (res)
1183 : {
1184 1 : if (CSLFetchNameValue(papszOptions, "CLOSE_PERSISTENT"))
1185 : {
1186 0 : CPLHTTPDestroyResult(res);
1187 0 : res = nullptr;
1188 : }
1189 1 : return res;
1190 : }
1191 : }
1192 :
1193 : #ifndef HAVE_CURL
1194 : CPLError(CE_Failure, CPLE_NotSupported,
1195 : "GDAL/OGR not compiled with libcurl support, "
1196 : "remote requests not supported.");
1197 : return nullptr;
1198 : #else
1199 :
1200 : /* -------------------------------------------------------------------- */
1201 : /* Are we using a persistent named session? If so, search for */
1202 : /* or create it. */
1203 : /* */
1204 : /* Currently this code does not attempt to protect against */
1205 : /* multiple threads asking for the same named session. If that */
1206 : /* occurs it will be in use in multiple threads at once, which */
1207 : /* will lead to potential crashes in libcurl. */
1208 : /* -------------------------------------------------------------------- */
1209 1335 : CURL *http_handle = nullptr;
1210 :
1211 1335 : const char *pszPersistent = CSLFetchNameValue(papszOptions, "PERSISTENT");
1212 : const char *pszClosePersistent =
1213 1335 : CSLFetchNameValue(papszOptions, "CLOSE_PERSISTENT");
1214 1335 : if (pszPersistent)
1215 : {
1216 976 : CPLString osSessionName = pszPersistent;
1217 488 : CPLMutexHolder oHolder(&hSessionMapMutex);
1218 :
1219 488 : if (poSessionMap == nullptr)
1220 132 : poSessionMap = new std::map<CPLString, CURL *>;
1221 488 : if (poSessionMap->count(osSessionName) == 0)
1222 : {
1223 155 : (*poSessionMap)[osSessionName] = curl_easy_init();
1224 155 : CPLDebug("HTTP", "Establish persistent session named '%s'.",
1225 : osSessionName.c_str());
1226 : }
1227 :
1228 488 : http_handle = (*poSessionMap)[osSessionName];
1229 : }
1230 : /* -------------------------------------------------------------------- */
1231 : /* Are we requested to close a persistent named session? */
1232 : /* -------------------------------------------------------------------- */
1233 847 : else if (pszClosePersistent)
1234 : {
1235 370 : CPLString osSessionName = pszClosePersistent;
1236 185 : CPLMutexHolder oHolder(&hSessionMapMutex);
1237 :
1238 185 : if (poSessionMap)
1239 : {
1240 : std::map<CPLString, CURL *>::iterator oIter =
1241 155 : poSessionMap->find(osSessionName);
1242 155 : if (oIter != poSessionMap->end())
1243 : {
1244 155 : curl_easy_cleanup(oIter->second);
1245 155 : poSessionMap->erase(oIter);
1246 155 : if (poSessionMap->empty())
1247 : {
1248 132 : delete poSessionMap;
1249 132 : poSessionMap = nullptr;
1250 : }
1251 155 : CPLDebug("HTTP", "Ended persistent session named '%s'.",
1252 : osSessionName.c_str());
1253 : }
1254 : else
1255 : {
1256 0 : CPLDebug("HTTP",
1257 : "Could not find persistent session named '%s'.",
1258 : osSessionName.c_str());
1259 : }
1260 : }
1261 :
1262 185 : return nullptr;
1263 : }
1264 : else
1265 662 : http_handle = curl_easy_init();
1266 :
1267 : /* -------------------------------------------------------------------- */
1268 : /* Setup the request. */
1269 : /* -------------------------------------------------------------------- */
1270 1150 : char szCurlErrBuf[CURL_ERROR_SIZE + 1] = {};
1271 :
1272 1150 : CPLHTTPEmitFetchDebug(pszURL);
1273 :
1274 : CPLHTTPResult *psResult =
1275 1150 : static_cast<CPLHTTPResult *>(CPLCalloc(1, sizeof(CPLHTTPResult)));
1276 :
1277 : struct curl_slist *headers = reinterpret_cast<struct curl_slist *>(
1278 1150 : CPLHTTPSetOptions(http_handle, pszURL, papszOptions));
1279 1150 : if (headers != nullptr)
1280 591 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTPHEADER, headers);
1281 :
1282 : // Are we making a head request.
1283 1150 : const char *pszNoBody = nullptr;
1284 1150 : if ((pszNoBody = CSLFetchNameValue(papszOptions, "NO_BODY")) != nullptr)
1285 : {
1286 0 : if (CPLTestBool(pszNoBody))
1287 : {
1288 0 : CPLDebug("HTTP", "HEAD Request: %s", pszURL);
1289 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_NOBODY, 1L);
1290 : }
1291 : }
1292 :
1293 : // Capture response headers.
1294 1150 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HEADERDATA, psResult);
1295 1150 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HEADERFUNCTION,
1296 : CPLHdrWriteFct);
1297 :
1298 1150 : CPLHTTPResultWithLimit sResultWithLimit;
1299 1150 : if (nullptr == pfnWrite)
1300 : {
1301 1149 : pfnWrite = CPLWriteFct;
1302 :
1303 1149 : sResultWithLimit.psResult = psResult;
1304 1149 : sResultWithLimit.nMaxFileSize = 0;
1305 : const char *pszMaxFileSize =
1306 1149 : CSLFetchNameValue(papszOptions, "MAX_FILE_SIZE");
1307 1149 : if (pszMaxFileSize != nullptr)
1308 : {
1309 10 : sResultWithLimit.nMaxFileSize = atoi(pszMaxFileSize);
1310 : // Only useful if size is returned by server before actual download.
1311 10 : unchecked_curl_easy_setopt(http_handle, CURLOPT_MAXFILESIZE,
1312 : sResultWithLimit.nMaxFileSize);
1313 : }
1314 1149 : pWriteArg = &sResultWithLimit;
1315 : }
1316 :
1317 1150 : unchecked_curl_easy_setopt(http_handle, CURLOPT_WRITEDATA, pWriteArg);
1318 1150 : unchecked_curl_easy_setopt(http_handle, CURLOPT_WRITEFUNCTION, pfnWrite);
1319 :
1320 1150 : CurlProcessData stProcessData = {pfnProgress, pProgressArg};
1321 1150 : if (nullptr != pfnProgress)
1322 : {
1323 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_XFERINFOFUNCTION,
1324 : NewProcessFunction);
1325 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_XFERINFODATA,
1326 : &stProcessData);
1327 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_NOPROGRESS, 0L);
1328 : }
1329 :
1330 1150 : szCurlErrBuf[0] = '\0';
1331 :
1332 1150 : unchecked_curl_easy_setopt(http_handle, CURLOPT_ERRORBUFFER, szCurlErrBuf);
1333 :
1334 1150 : bool bGZipRequested = false;
1335 1150 : if (bSupportGZip && CPLTestBool(CPLGetConfigOption("CPL_CURL_GZIP", "YES")))
1336 : {
1337 1150 : bGZipRequested = true;
1338 1150 : unchecked_curl_easy_setopt(http_handle, CURLOPT_ENCODING, "gzip");
1339 : }
1340 :
1341 2300 : CPLHTTPPostFields oPostFields;
1342 1150 : if (oPostFields.Fill(http_handle, papszOptions) != CE_None)
1343 : {
1344 2 : psResult->nStatus = 34; // CURLE_HTTP_POST_ERROR
1345 2 : psResult->pszErrBuf = CPLStrdup(oPostFields.GetErrorMessage().c_str());
1346 2 : CPLError(CE_Failure, CPLE_AppDefined, "%s", psResult->pszErrBuf);
1347 2 : CPLHTTPFetchCleanup(http_handle, headers, pszPersistent, papszOptions);
1348 2 : return psResult;
1349 : }
1350 :
1351 : /* -------------------------------------------------------------------- */
1352 : /* If 429, 502, 503 or 504 status code retry this HTTP call until max
1353 : */
1354 : /* retry has been reached */
1355 : /* -------------------------------------------------------------------- */
1356 1148 : const char *pszRetryDelay = CSLFetchNameValue(papszOptions, "RETRY_DELAY");
1357 1148 : if (pszRetryDelay == nullptr)
1358 1148 : pszRetryDelay = CPLGetConfigOption(
1359 : "GDAL_HTTP_RETRY_DELAY", CPLSPrintf("%f", CPL_HTTP_RETRY_DELAY));
1360 1148 : const char *pszMaxRetries = CSLFetchNameValue(papszOptions, "MAX_RETRY");
1361 1148 : if (pszMaxRetries == nullptr)
1362 1084 : pszMaxRetries = CPLGetConfigOption(
1363 : "GDAL_HTTP_MAX_RETRY", CPLSPrintf("%d", CPL_HTTP_MAX_RETRY));
1364 : // coverity[tainted_data]
1365 1148 : double dfRetryDelaySecs = CPLAtof(pszRetryDelay);
1366 1148 : int nMaxRetries = atoi(pszMaxRetries);
1367 1148 : int nRetryCount = 0;
1368 :
1369 : while (true)
1370 : {
1371 : /* --------------------------------------------------------------------
1372 : */
1373 : /* Execute the request, waiting for results. */
1374 : /* --------------------------------------------------------------------
1375 : */
1376 1148 : void *old_handler = CPLHTTPIgnoreSigPipe();
1377 1148 : psResult->nStatus = static_cast<int>(curl_easy_perform(http_handle));
1378 1148 : CPLHTTPRestoreSigPipeHandler(old_handler);
1379 :
1380 : /* --------------------------------------------------------------------
1381 : */
1382 : /* Fetch content-type if possible. */
1383 : /* --------------------------------------------------------------------
1384 : */
1385 1148 : psResult->pszContentType = nullptr;
1386 1148 : curl_easy_getinfo(http_handle, CURLINFO_CONTENT_TYPE,
1387 : &(psResult->pszContentType));
1388 1148 : if (psResult->pszContentType != nullptr)
1389 556 : psResult->pszContentType = CPLStrdup(psResult->pszContentType);
1390 :
1391 1148 : long response_code = 0;
1392 1148 : curl_easy_getinfo(http_handle, CURLINFO_RESPONSE_CODE, &response_code);
1393 1148 : if (response_code != 200)
1394 : {
1395 203 : const double dfNewRetryDelay = CPLHTTPGetNewRetryDelay(
1396 : static_cast<int>(response_code), dfRetryDelaySecs,
1397 203 : reinterpret_cast<const char *>(psResult->pabyData),
1398 : szCurlErrBuf);
1399 203 : if (dfNewRetryDelay > 0 && nRetryCount < nMaxRetries)
1400 : {
1401 0 : CPLError(CE_Warning, CPLE_AppDefined,
1402 : "HTTP error code: %d - %s. "
1403 : "Retrying again in %.1f secs",
1404 : static_cast<int>(response_code), pszURL,
1405 : dfRetryDelaySecs);
1406 0 : CPLSleep(dfRetryDelaySecs);
1407 0 : dfRetryDelaySecs = dfNewRetryDelay;
1408 0 : nRetryCount++;
1409 :
1410 0 : CPLFree(psResult->pszContentType);
1411 0 : psResult->pszContentType = nullptr;
1412 0 : CSLDestroy(psResult->papszHeaders);
1413 0 : psResult->papszHeaders = nullptr;
1414 0 : CPLFree(psResult->pabyData);
1415 0 : psResult->pabyData = nullptr;
1416 0 : psResult->nDataLen = 0;
1417 0 : psResult->nDataAlloc = 0;
1418 :
1419 0 : continue;
1420 : }
1421 : }
1422 :
1423 : /* --------------------------------------------------------------------
1424 : */
1425 : /* Have we encountered some sort of error? */
1426 : /* --------------------------------------------------------------------
1427 : */
1428 1148 : if (strlen(szCurlErrBuf) > 0)
1429 : {
1430 7 : bool bSkipError = false;
1431 : const char *pszContentLength =
1432 7 : CSLFetchNameValue(psResult->papszHeaders, "Content-Length");
1433 : // Some servers such as
1434 : // http://115.113.193.14/cgi-bin/world/qgis_mapserv.fcgi?VERSION=1.1.1&SERVICE=WMS&REQUEST=GetCapabilities
1435 : // invalidly return Content-Length as the uncompressed size, with
1436 : // makes curl to wait for more data and time-out finally. If we got
1437 : // the expected data size, then we don't emit an error but turn off
1438 : // GZip requests.
1439 7 : if (bGZipRequested &&
1440 7 : strstr(szCurlErrBuf, "transfer closed with") &&
1441 0 : strstr(szCurlErrBuf, "bytes remaining to read"))
1442 : {
1443 0 : if (pszContentLength && psResult->nDataLen != 0 &&
1444 0 : atoi(pszContentLength) == psResult->nDataLen)
1445 : {
1446 : const char *pszCurlGZIPOption =
1447 0 : CPLGetConfigOption("CPL_CURL_GZIP", nullptr);
1448 0 : if (pszCurlGZIPOption == nullptr)
1449 : {
1450 0 : CPLSetConfigOption("CPL_CURL_GZIP", "NO");
1451 0 : CPLDebug("HTTP",
1452 : "Disabling CPL_CURL_GZIP, "
1453 : "because %s doesn't support it properly",
1454 : pszURL);
1455 : }
1456 0 : psResult->nStatus = 0;
1457 0 : bSkipError = true;
1458 0 : }
1459 : }
1460 :
1461 : // Ignore SSL errors about non-properly terminated connection,
1462 : // often due to HTTP proxies
1463 7 : else if (pszContentLength == nullptr &&
1464 : // Cf https://github.com/curl/curl/pull/3148
1465 7 : (strstr(szCurlErrBuf,
1466 : "GnuTLS recv error (-110): The TLS connection was "
1467 7 : "non-properly terminated") != nullptr ||
1468 : // Cf https://github.com/curl/curl/issues/9024
1469 7 : strstr(szCurlErrBuf,
1470 : "SSL_read: error:0A000126:SSL "
1471 : "routines::unexpected eof while reading") !=
1472 : nullptr))
1473 : {
1474 0 : psResult->nStatus = 0;
1475 0 : bSkipError = true;
1476 : }
1477 7 : else if (CPLTestBool(
1478 : CPLGetConfigOption("CPL_CURL_IGNORE_ERROR", "NO")))
1479 : {
1480 0 : psResult->nStatus = 0;
1481 0 : bSkipError = true;
1482 : }
1483 :
1484 7 : if (!bSkipError)
1485 : {
1486 7 : psResult->pszErrBuf = CPLStrdup(szCurlErrBuf);
1487 7 : if (psResult->nDataLen > 0)
1488 : {
1489 1 : CPLError(CE_Failure, CPLE_AppDefined,
1490 : "%s. You may set the CPL_CURL_IGNORE_ERROR "
1491 : "configuration option to YES to try to ignore it.",
1492 : szCurlErrBuf);
1493 : }
1494 : else
1495 : {
1496 6 : CPLError(CE_Failure, CPLE_AppDefined, "%s", szCurlErrBuf);
1497 : }
1498 : }
1499 : }
1500 : else
1501 : {
1502 1141 : if (response_code >= 400 && response_code < 600)
1503 : {
1504 193 : psResult->pszErrBuf = CPLStrdup(CPLSPrintf(
1505 : "HTTP error code : %d", static_cast<int>(response_code)));
1506 193 : CPLError(CE_Failure, CPLE_AppDefined, "%s",
1507 : psResult->pszErrBuf);
1508 : }
1509 : }
1510 1148 : break;
1511 0 : }
1512 :
1513 1148 : CPLHTTPFetchCleanup(http_handle, headers, pszPersistent, papszOptions);
1514 :
1515 1148 : return psResult;
1516 : #endif /* def HAVE_CURL */
1517 : }
1518 :
1519 : #ifdef HAVE_CURL
1520 : /************************************************************************/
1521 : /* CPLMultiPerformWait() */
1522 : /************************************************************************/
1523 :
1524 1853 : bool CPLMultiPerformWait(void *hCurlMultiHandleIn, int & /*repeats*/)
1525 : {
1526 1853 : CURLM *hCurlMultiHandle = static_cast<CURLM *>(hCurlMultiHandleIn);
1527 :
1528 : // Wait for events on the sockets
1529 :
1530 : // Using curl_multi_poll() is preferred to avoid hitting the 1024 file
1531 : // descriptor limit
1532 :
1533 1853 : int numfds = 0;
1534 1853 : if (curl_multi_poll(hCurlMultiHandle, nullptr, 0, 1000, &numfds) !=
1535 : CURLM_OK)
1536 : {
1537 0 : CPLError(CE_Failure, CPLE_AppDefined, "curl_multi_poll() failed");
1538 0 : return false;
1539 : }
1540 1853 : return true;
1541 : }
1542 :
1543 : class CPLHTTPErrorBuffer
1544 : {
1545 : public:
1546 : char szBuffer[CURL_ERROR_SIZE + 1];
1547 :
1548 0 : CPLHTTPErrorBuffer()
1549 0 : {
1550 0 : szBuffer[0] = '\0';
1551 0 : }
1552 : };
1553 :
1554 : #endif // HAVE_CURL
1555 :
1556 : /************************************************************************/
1557 : /* CPLHTTPMultiFetch() */
1558 : /************************************************************************/
1559 :
1560 : /**
1561 : * \brief Fetch several documents at once
1562 : *
1563 : * @param papszURL array of valid URLs recognized by underlying download library
1564 : * (libcurl)
1565 : * @param nURLCount number of URLs of papszURL
1566 : * @param nMaxSimultaneous maximum number of downloads to issue simultaneously.
1567 : * Any negative or zer value means unlimited.
1568 : * @param papszOptions option list as a NULL-terminated array of strings. May be
1569 : * NULL. Refer to CPLHTTPFetch() for valid options.
1570 : * @return an array of CPLHTTPResult* structures that must be freed by
1571 : * CPLHTTPDestroyMultiResult() or NULL if libcurl support is disabled
1572 : *
1573 : * @since GDAL 2.3
1574 : */
1575 0 : CPLHTTPResult **CPLHTTPMultiFetch(const char *const *papszURL, int nURLCount,
1576 : int nMaxSimultaneous,
1577 : CSLConstList papszOptions)
1578 : {
1579 : #ifndef HAVE_CURL
1580 : (void)papszURL;
1581 : (void)nURLCount;
1582 : (void)nMaxSimultaneous;
1583 : (void)papszOptions;
1584 :
1585 : CPLError(CE_Failure, CPLE_NotSupported,
1586 : "GDAL/OGR not compiled with libcurl support, "
1587 : "remote requests not supported.");
1588 : return nullptr;
1589 : #else /* def HAVE_CURL */
1590 :
1591 : /* -------------------------------------------------------------------- */
1592 : /* Are we using a persistent named session? If so, search for */
1593 : /* or create it. */
1594 : /* */
1595 : /* Currently this code does not attempt to protect against */
1596 : /* multiple threads asking for the same named session. If that */
1597 : /* occurs it will be in use in multiple threads at once, which */
1598 : /* will lead to potential crashes in libcurl. */
1599 : /* -------------------------------------------------------------------- */
1600 0 : CURLM *hCurlMultiHandle = nullptr;
1601 :
1602 0 : const char *pszPersistent = CSLFetchNameValue(papszOptions, "PERSISTENT");
1603 : const char *pszClosePersistent =
1604 0 : CSLFetchNameValue(papszOptions, "CLOSE_PERSISTENT");
1605 0 : if (pszPersistent)
1606 : {
1607 0 : CPLString osSessionName = pszPersistent;
1608 0 : CPLMutexHolder oHolder(&hSessionMapMutex);
1609 :
1610 0 : if (poSessionMultiMap == nullptr)
1611 0 : poSessionMultiMap = new std::map<CPLString, CURLM *>;
1612 0 : if (poSessionMultiMap->count(osSessionName) == 0)
1613 : {
1614 0 : (*poSessionMultiMap)[osSessionName] = curl_multi_init();
1615 0 : CPLDebug("HTTP", "Establish persistent session named '%s'.",
1616 : osSessionName.c_str());
1617 : }
1618 :
1619 0 : hCurlMultiHandle = (*poSessionMultiMap)[osSessionName];
1620 : }
1621 : /* -------------------------------------------------------------------- */
1622 : /* Are we requested to close a persistent named session? */
1623 : /* -------------------------------------------------------------------- */
1624 0 : else if (pszClosePersistent)
1625 : {
1626 0 : CPLString osSessionName = pszClosePersistent;
1627 0 : CPLMutexHolder oHolder(&hSessionMapMutex);
1628 :
1629 0 : if (poSessionMultiMap)
1630 : {
1631 0 : auto oIter = poSessionMultiMap->find(osSessionName);
1632 0 : if (oIter != poSessionMultiMap->end())
1633 : {
1634 0 : VSICURLMultiCleanup(oIter->second);
1635 0 : poSessionMultiMap->erase(oIter);
1636 0 : if (poSessionMultiMap->empty())
1637 : {
1638 0 : delete poSessionMultiMap;
1639 0 : poSessionMultiMap = nullptr;
1640 : }
1641 0 : CPLDebug("HTTP", "Ended persistent session named '%s'.",
1642 : osSessionName.c_str());
1643 : }
1644 : else
1645 : {
1646 0 : CPLDebug("HTTP",
1647 : "Could not find persistent session named '%s'.",
1648 : osSessionName.c_str());
1649 : }
1650 : }
1651 :
1652 0 : return nullptr;
1653 : }
1654 : else
1655 : {
1656 0 : hCurlMultiHandle = curl_multi_init();
1657 : }
1658 :
1659 : CPLHTTPResult **papsResults = static_cast<CPLHTTPResult **>(
1660 0 : CPLCalloc(nURLCount, sizeof(CPLHTTPResult *)));
1661 0 : std::vector<CURL *> asHandles;
1662 0 : std::vector<CPLHTTPResultWithLimit> asResults;
1663 0 : asResults.resize(nURLCount);
1664 0 : std::vector<struct curl_slist *> aHeaders;
1665 0 : aHeaders.resize(nURLCount);
1666 0 : std::vector<CPLHTTPErrorBuffer> asErrorBuffers;
1667 0 : asErrorBuffers.resize(nURLCount);
1668 :
1669 0 : for (int i = 0; i < nURLCount; i++)
1670 : {
1671 0 : papsResults[i] =
1672 0 : static_cast<CPLHTTPResult *>(CPLCalloc(1, sizeof(CPLHTTPResult)));
1673 :
1674 0 : const char *pszURL = papszURL[i];
1675 0 : CURL *http_handle = curl_easy_init();
1676 :
1677 0 : aHeaders[i] = reinterpret_cast<struct curl_slist *>(
1678 0 : CPLHTTPSetOptions(http_handle, pszURL, papszOptions));
1679 :
1680 : // Set Headers.
1681 0 : const char *pszHeaders = CSLFetchNameValue(papszOptions, "HEADERS");
1682 0 : if (pszHeaders != nullptr)
1683 : {
1684 : char **papszTokensHeaders =
1685 0 : CSLTokenizeString2(pszHeaders, "\r\n", 0);
1686 0 : for (int j = 0; papszTokensHeaders[j] != nullptr; ++j)
1687 0 : aHeaders[i] =
1688 0 : curl_slist_append(aHeaders[i], papszTokensHeaders[j]);
1689 0 : CSLDestroy(papszTokensHeaders);
1690 : }
1691 :
1692 0 : if (aHeaders[i] != nullptr)
1693 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTPHEADER,
1694 : aHeaders[i]);
1695 :
1696 : // Capture response headers.
1697 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HEADERDATA,
1698 : papsResults[i]);
1699 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HEADERFUNCTION,
1700 : CPLHdrWriteFct);
1701 :
1702 0 : asResults[i].psResult = papsResults[i];
1703 : const char *pszMaxFileSize =
1704 0 : CSLFetchNameValue(papszOptions, "MAX_FILE_SIZE");
1705 0 : if (pszMaxFileSize != nullptr)
1706 : {
1707 0 : asResults[i].nMaxFileSize = atoi(pszMaxFileSize);
1708 : // Only useful if size is returned by server before actual download.
1709 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_MAXFILESIZE,
1710 : asResults[i].nMaxFileSize);
1711 : }
1712 :
1713 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_WRITEDATA,
1714 : &asResults[i]);
1715 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_WRITEFUNCTION,
1716 : CPLWriteFct);
1717 :
1718 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_ERRORBUFFER,
1719 : asErrorBuffers[i].szBuffer);
1720 :
1721 0 : if (bSupportGZip &&
1722 0 : CPLTestBool(CPLGetConfigOption("CPL_CURL_GZIP", "YES")))
1723 : {
1724 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_ENCODING, "gzip");
1725 : }
1726 :
1727 0 : asHandles.push_back(http_handle);
1728 : }
1729 :
1730 0 : int iCurRequest = 0;
1731 0 : for (;
1732 0 : iCurRequest <
1733 0 : std::min(nURLCount, nMaxSimultaneous > 0 ? nMaxSimultaneous : INT_MAX);
1734 : iCurRequest++)
1735 : {
1736 0 : CPLHTTPEmitFetchDebug(papszURL[iCurRequest],
1737 : CPLSPrintf(" %d/%d", iCurRequest + 1, nURLCount));
1738 0 : curl_multi_add_handle(hCurlMultiHandle, asHandles[iCurRequest]);
1739 : }
1740 :
1741 0 : int repeats = 0;
1742 0 : void *old_handler = CPLHTTPIgnoreSigPipe();
1743 : while (true)
1744 : {
1745 0 : int still_running = 0;
1746 0 : while (curl_multi_perform(hCurlMultiHandle, &still_running) ==
1747 : CURLM_CALL_MULTI_PERFORM)
1748 : {
1749 : // loop
1750 : }
1751 0 : if (!still_running && iCurRequest == nURLCount)
1752 : {
1753 0 : break;
1754 : }
1755 :
1756 0 : bool bRequestsAdded = false;
1757 : CURLMsg *msg;
1758 0 : do
1759 : {
1760 0 : int msgq = 0;
1761 0 : msg = curl_multi_info_read(hCurlMultiHandle, &msgq);
1762 0 : if (msg && (msg->msg == CURLMSG_DONE))
1763 : {
1764 0 : if (iCurRequest < nURLCount)
1765 : {
1766 0 : CPLHTTPEmitFetchDebug(
1767 0 : papszURL[iCurRequest],
1768 : CPLSPrintf(" %d/%d", iCurRequest + 1, nURLCount));
1769 0 : curl_multi_add_handle(hCurlMultiHandle,
1770 0 : asHandles[iCurRequest]);
1771 0 : iCurRequest++;
1772 0 : bRequestsAdded = true;
1773 : }
1774 : }
1775 0 : } while (msg);
1776 :
1777 0 : if (!bRequestsAdded)
1778 0 : CPLMultiPerformWait(hCurlMultiHandle, repeats);
1779 0 : }
1780 0 : CPLHTTPRestoreSigPipeHandler(old_handler);
1781 :
1782 0 : for (int i = 0; i < nURLCount; i++)
1783 : {
1784 0 : if (asErrorBuffers[i].szBuffer[0] != '\0')
1785 0 : papsResults[i]->pszErrBuf = CPLStrdup(asErrorBuffers[i].szBuffer);
1786 : else
1787 : {
1788 0 : long response_code = 0;
1789 0 : curl_easy_getinfo(asHandles[i], CURLINFO_RESPONSE_CODE,
1790 : &response_code);
1791 :
1792 0 : if (response_code >= 400 && response_code < 600)
1793 : {
1794 0 : papsResults[i]->pszErrBuf = CPLStrdup(CPLSPrintf(
1795 : "HTTP error code : %d", static_cast<int>(response_code)));
1796 : }
1797 : }
1798 :
1799 0 : curl_easy_getinfo(asHandles[i], CURLINFO_CONTENT_TYPE,
1800 : &(papsResults[i]->pszContentType));
1801 0 : if (papsResults[i]->pszContentType != nullptr)
1802 0 : papsResults[i]->pszContentType =
1803 0 : CPLStrdup(papsResults[i]->pszContentType);
1804 :
1805 0 : curl_multi_remove_handle(hCurlMultiHandle, asHandles[i]);
1806 0 : curl_easy_cleanup(asHandles[i]);
1807 : }
1808 :
1809 0 : if (!pszPersistent)
1810 0 : VSICURLMultiCleanup(hCurlMultiHandle);
1811 :
1812 0 : for (size_t i = 0; i < aHeaders.size(); i++)
1813 0 : curl_slist_free_all(aHeaders[i]);
1814 :
1815 0 : return papsResults;
1816 : #endif /* def HAVE_CURL */
1817 : }
1818 :
1819 : /************************************************************************/
1820 : /* CPLHTTPDestroyMultiResult() */
1821 : /************************************************************************/
1822 : /**
1823 : * \brief Clean the memory associated with the return value of
1824 : * CPLHTTPMultiFetch()
1825 : *
1826 : * @param papsResults pointer to the return value of CPLHTTPMultiFetch()
1827 : * @param nCount value of the nURLCount parameter passed to CPLHTTPMultiFetch()
1828 : * @since GDAL 2.3
1829 : */
1830 0 : void CPLHTTPDestroyMultiResult(CPLHTTPResult **papsResults, int nCount)
1831 : {
1832 0 : if (papsResults)
1833 : {
1834 0 : for (int i = 0; i < nCount; i++)
1835 : {
1836 0 : CPLHTTPDestroyResult(papsResults[i]);
1837 : }
1838 0 : CPLFree(papsResults);
1839 : }
1840 0 : }
1841 :
1842 : #ifdef HAVE_CURL
1843 :
1844 : #ifdef _WIN32
1845 :
1846 : #include <windows.h>
1847 :
1848 : /************************************************************************/
1849 : /* CPLFindWin32CurlCaBundleCrt() */
1850 : /************************************************************************/
1851 :
1852 : static const char *CPLFindWin32CurlCaBundleCrt()
1853 : {
1854 : DWORD nResLen;
1855 : const DWORD nBufSize = MAX_PATH + 1;
1856 : char *pszFilePart = nullptr;
1857 :
1858 : char *pszPath = static_cast<char *>(CPLCalloc(1, nBufSize));
1859 :
1860 : nResLen = SearchPathA(nullptr, "curl-ca-bundle.crt", nullptr, nBufSize,
1861 : pszPath, &pszFilePart);
1862 : if (nResLen > 0)
1863 : {
1864 : const char *pszRet = CPLSPrintf("%s", pszPath);
1865 : CPLFree(pszPath);
1866 : return pszRet;
1867 : }
1868 : CPLFree(pszPath);
1869 : return nullptr;
1870 : }
1871 :
1872 : #endif // WIN32
1873 :
1874 : /************************************************************************/
1875 : /* CPLHTTPCurlDebugFunction() */
1876 : /************************************************************************/
1877 :
1878 13 : static int CPLHTTPCurlDebugFunction(CURL *handle, curl_infotype type,
1879 : char *data, size_t size, void *userp)
1880 : {
1881 : (void)handle;
1882 : (void)userp;
1883 :
1884 13 : const char *pszDebugKey = nullptr;
1885 13 : if (type == CURLINFO_TEXT)
1886 : {
1887 7 : pszDebugKey = "CURL_INFO_TEXT";
1888 : }
1889 6 : else if (type == CURLINFO_HEADER_OUT)
1890 : {
1891 1 : pszDebugKey = "CURL_INFO_HEADER_OUT";
1892 : }
1893 5 : else if (type == CURLINFO_HEADER_IN)
1894 : {
1895 5 : pszDebugKey = "CURL_INFO_HEADER_IN";
1896 : }
1897 0 : else if (type == CURLINFO_DATA_IN &&
1898 0 : CPLTestBool(CPLGetConfigOption("CPL_CURL_VERBOSE_DATA_IN", "NO")))
1899 : {
1900 0 : pszDebugKey = "CURL_INFO_DATA_IN";
1901 : }
1902 :
1903 13 : if (pszDebugKey)
1904 : {
1905 26 : std::string osMsg(data, size);
1906 13 : if (!osMsg.empty() && osMsg.back() == '\n')
1907 13 : osMsg.resize(osMsg.size() - 1);
1908 13 : CPLDebug(pszDebugKey, "%s", osMsg.c_str());
1909 : }
1910 13 : return 0;
1911 : }
1912 :
1913 : /************************************************************************/
1914 : /* CPLHTTPSetOptions() */
1915 : /************************************************************************/
1916 :
1917 : // Note: papszOptions must be kept alive until curl_easy/multi_perform()
1918 : // has completed, and we must be careful not to set short lived strings
1919 : // with unchecked_curl_easy_setopt(), as long as we need to support curl < 7.17
1920 : // see https://curl.haxx.se/libcurl/c/unchecked_curl_easy_setopt.html
1921 : // caution: if we remove that assumption, we'll needto use
1922 : // CURLOPT_COPYPOSTFIELDS
1923 :
1924 2241 : void *CPLHTTPSetOptions(void *pcurl, const char *pszURL,
1925 : const char *const *papszOptions)
1926 : {
1927 2241 : CheckCurlFeatures();
1928 :
1929 2241 : CURL *http_handle = reinterpret_cast<CURL *>(pcurl);
1930 :
1931 2241 : unchecked_curl_easy_setopt(http_handle, CURLOPT_URL, pszURL);
1932 :
1933 2241 : if (CPLTestBool(CPLGetConfigOption("CPL_CURL_VERBOSE", "NO")))
1934 : {
1935 1 : unchecked_curl_easy_setopt(http_handle, CURLOPT_VERBOSE, 1);
1936 :
1937 1 : if (CPLGetConfigOption("CPL_DEBUG", nullptr) != nullptr)
1938 : {
1939 1 : unchecked_curl_easy_setopt(http_handle, CURLOPT_DEBUGFUNCTION,
1940 : CPLHTTPCurlDebugFunction);
1941 : }
1942 : }
1943 :
1944 : const char *pszHttpVersion =
1945 2241 : CSLFetchNameValue(papszOptions, "HTTP_VERSION");
1946 2241 : if (pszHttpVersion == nullptr)
1947 2241 : pszHttpVersion = CPLGetConfigOption("GDAL_HTTP_VERSION", nullptr);
1948 2241 : if (pszHttpVersion && strcmp(pszHttpVersion, "1.0") == 0)
1949 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTP_VERSION,
1950 : CURL_HTTP_VERSION_1_0);
1951 2241 : else if (pszHttpVersion && strcmp(pszHttpVersion, "1.1") == 0)
1952 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTP_VERSION,
1953 : CURL_HTTP_VERSION_1_1);
1954 2241 : else if (pszHttpVersion && (strcmp(pszHttpVersion, "2") == 0 ||
1955 0 : strcmp(pszHttpVersion, "2.0") == 0))
1956 : {
1957 0 : if (bSupportHTTP2)
1958 : {
1959 : // Try HTTP/2 both for HTTP and HTTPS. With fallback to HTTP/1.1
1960 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTP_VERSION,
1961 : CURL_HTTP_VERSION_2_0);
1962 : }
1963 : }
1964 2241 : else if (pszHttpVersion == nullptr || strcmp(pszHttpVersion, "2TLS") == 0)
1965 : {
1966 2241 : if (bSupportHTTP2)
1967 : {
1968 : // Only enable this mode if explicitly required, or if the
1969 : // machine is a GCE instance. On other networks, requesting a
1970 : // file in HTTP/2 is found to be significantly slower than HTTP/1.1
1971 : // for unknown reasons.
1972 2241 : if (pszHttpVersion != nullptr || CPLIsMachineForSureGCEInstance())
1973 : {
1974 : static bool bDebugEmitted = false;
1975 0 : if (!bDebugEmitted)
1976 : {
1977 0 : CPLDebug("HTTP", "Using HTTP/2 for HTTPS when possible");
1978 0 : bDebugEmitted = true;
1979 : }
1980 :
1981 : // CURL_HTTP_VERSION_2TLS means for HTTPS connection, try to
1982 : // negotiate HTTP/2 with the server (and fallback to HTTP/1.1
1983 : // otherwise), and for HTTP connection do HTTP/1
1984 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTP_VERSION,
1985 : CURL_HTTP_VERSION_2TLS);
1986 : }
1987 2241 : }
1988 : }
1989 : else
1990 : {
1991 0 : CPLError(CE_Warning, CPLE_NotSupported, "HTTP_VERSION=%s not supported",
1992 : pszHttpVersion);
1993 : }
1994 :
1995 : // Default value is 1 since curl 7.50.2. But worth applying it on
1996 : // previous versions as well.
1997 : const char *pszTCPNoDelay =
1998 2241 : CSLFetchNameValueDef(papszOptions, "TCP_NODELAY", "1");
1999 2241 : unchecked_curl_easy_setopt(http_handle, CURLOPT_TCP_NODELAY,
2000 : atoi(pszTCPNoDelay));
2001 :
2002 : /* Support control over HTTPAUTH */
2003 2241 : const char *pszHttpAuth = CSLFetchNameValue(papszOptions, "HTTPAUTH");
2004 2241 : if (pszHttpAuth == nullptr)
2005 2239 : pszHttpAuth = CPLGetConfigOption("GDAL_HTTP_AUTH", nullptr);
2006 2241 : if (pszHttpAuth == nullptr)
2007 : {
2008 : /* do nothing */;
2009 : }
2010 3 : else if (EQUAL(pszHttpAuth, "BASIC"))
2011 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTPAUTH,
2012 : CURLAUTH_BASIC);
2013 3 : else if (EQUAL(pszHttpAuth, "NTLM"))
2014 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTPAUTH,
2015 : CURLAUTH_NTLM);
2016 3 : else if (EQUAL(pszHttpAuth, "ANY"))
2017 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTPAUTH, CURLAUTH_ANY);
2018 3 : else if (EQUAL(pszHttpAuth, "ANYSAFE"))
2019 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTPAUTH,
2020 : CURLAUTH_ANYSAFE);
2021 3 : else if (EQUAL(pszHttpAuth, "BEARER"))
2022 : {
2023 3 : const char *pszBearer = CSLFetchNameValue(papszOptions, "HTTP_BEARER");
2024 3 : if (pszBearer == nullptr)
2025 1 : pszBearer = CPLGetConfigOption("GDAL_HTTP_BEARER", nullptr);
2026 3 : if (pszBearer != nullptr)
2027 3 : unchecked_curl_easy_setopt(http_handle, CURLOPT_XOAUTH2_BEARER,
2028 : pszBearer);
2029 3 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTPAUTH,
2030 : CURLAUTH_BEARER);
2031 : }
2032 0 : else if (EQUAL(pszHttpAuth, "NEGOTIATE"))
2033 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTPAUTH,
2034 : CURLAUTH_NEGOTIATE);
2035 : else
2036 : {
2037 0 : CPLError(CE_Warning, CPLE_AppDefined,
2038 : "Unsupported HTTPAUTH value '%s', ignored.", pszHttpAuth);
2039 : }
2040 :
2041 : const char *pszGssDelegation =
2042 2241 : CSLFetchNameValue(papszOptions, "GSSAPI_DELEGATION");
2043 2241 : if (pszGssDelegation == nullptr)
2044 : pszGssDelegation =
2045 2241 : CPLGetConfigOption("GDAL_GSSAPI_DELEGATION", nullptr);
2046 2241 : if (pszGssDelegation == nullptr)
2047 : {
2048 : }
2049 0 : else if (EQUAL(pszGssDelegation, "NONE"))
2050 : {
2051 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_GSSAPI_DELEGATION,
2052 : CURLGSSAPI_DELEGATION_NONE);
2053 : }
2054 0 : else if (EQUAL(pszGssDelegation, "POLICY"))
2055 : {
2056 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_GSSAPI_DELEGATION,
2057 : CURLGSSAPI_DELEGATION_POLICY_FLAG);
2058 : }
2059 0 : else if (EQUAL(pszGssDelegation, "ALWAYS"))
2060 : {
2061 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_GSSAPI_DELEGATION,
2062 : CURLGSSAPI_DELEGATION_FLAG);
2063 : }
2064 : else
2065 : {
2066 0 : CPLError(CE_Warning, CPLE_AppDefined,
2067 : "Unsupported GSSAPI_DELEGATION value '%s', ignored.",
2068 : pszGssDelegation);
2069 : }
2070 :
2071 : // Support use of .netrc - default enabled.
2072 2241 : const char *pszHttpNetrc = CSLFetchNameValue(papszOptions, "NETRC");
2073 2241 : if (pszHttpNetrc == nullptr)
2074 2241 : pszHttpNetrc = CPLGetConfigOption("GDAL_HTTP_NETRC", "YES");
2075 2241 : if (pszHttpNetrc == nullptr || CPLTestBool(pszHttpNetrc))
2076 2241 : unchecked_curl_easy_setopt(http_handle, CURLOPT_NETRC, 1L);
2077 :
2078 : // Custom .netrc file location
2079 : const char *pszHttpNetrcFile =
2080 2241 : CSLFetchNameValue(papszOptions, "NETRC_FILE");
2081 2241 : if (pszHttpNetrcFile == nullptr)
2082 2241 : pszHttpNetrcFile = CPLGetConfigOption("GDAL_HTTP_NETRC_FILE", nullptr);
2083 2241 : if (pszHttpNetrcFile)
2084 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_NETRC_FILE,
2085 : pszHttpNetrcFile);
2086 :
2087 : // Support setting userid:password.
2088 2241 : const char *pszUserPwd = CSLFetchNameValue(papszOptions, "USERPWD");
2089 2241 : if (pszUserPwd == nullptr)
2090 2203 : pszUserPwd = CPLGetConfigOption("GDAL_HTTP_USERPWD", nullptr);
2091 2241 : if (pszUserPwd != nullptr)
2092 38 : unchecked_curl_easy_setopt(http_handle, CURLOPT_USERPWD, pszUserPwd);
2093 :
2094 : // Set Proxy parameters.
2095 2241 : const char *pszProxy = CSLFetchNameValue(papszOptions, "PROXY");
2096 2241 : if (pszProxy == nullptr)
2097 2241 : pszProxy = CPLGetConfigOption("GDAL_HTTP_PROXY", nullptr);
2098 2241 : if (pszProxy)
2099 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_PROXY, pszProxy);
2100 :
2101 2241 : const char *pszHttpsProxy = CSLFetchNameValue(papszOptions, "HTTPS_PROXY");
2102 2241 : if (pszHttpsProxy == nullptr)
2103 2241 : pszHttpsProxy = CPLGetConfigOption("GDAL_HTTPS_PROXY", nullptr);
2104 2241 : if (pszHttpsProxy && (STARTS_WITH(pszURL, "https")))
2105 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_PROXY, pszHttpsProxy);
2106 :
2107 : const char *pszProxyUserPwd =
2108 2241 : CSLFetchNameValue(papszOptions, "PROXYUSERPWD");
2109 2241 : if (pszProxyUserPwd == nullptr)
2110 2241 : pszProxyUserPwd = CPLGetConfigOption("GDAL_HTTP_PROXYUSERPWD", nullptr);
2111 2241 : if (pszProxyUserPwd)
2112 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_PROXYUSERPWD,
2113 : pszProxyUserPwd);
2114 :
2115 : // Support control over PROXYAUTH.
2116 2241 : const char *pszProxyAuth = CSLFetchNameValue(papszOptions, "PROXYAUTH");
2117 2241 : if (pszProxyAuth == nullptr)
2118 2241 : pszProxyAuth = CPLGetConfigOption("GDAL_PROXY_AUTH", nullptr);
2119 2241 : if (pszProxyAuth == nullptr)
2120 : {
2121 : // Do nothing.
2122 : }
2123 0 : else if (EQUAL(pszProxyAuth, "BASIC"))
2124 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_PROXYAUTH,
2125 : CURLAUTH_BASIC);
2126 0 : else if (EQUAL(pszProxyAuth, "NTLM"))
2127 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_PROXYAUTH,
2128 : CURLAUTH_NTLM);
2129 0 : else if (EQUAL(pszProxyAuth, "DIGEST"))
2130 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_PROXYAUTH,
2131 : CURLAUTH_DIGEST);
2132 0 : else if (EQUAL(pszProxyAuth, "ANY"))
2133 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_PROXYAUTH,
2134 : CURLAUTH_ANY);
2135 0 : else if (EQUAL(pszProxyAuth, "ANYSAFE"))
2136 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_PROXYAUTH,
2137 : CURLAUTH_ANYSAFE);
2138 0 : else if (EQUAL(pszProxyAuth, "NEGOTIATE"))
2139 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_PROXYAUTH,
2140 : CURLAUTH_NEGOTIATE);
2141 : else
2142 : {
2143 0 : CPLError(CE_Warning, CPLE_AppDefined,
2144 : "Unsupported PROXYAUTH value '%s', ignored.", pszProxyAuth);
2145 : }
2146 :
2147 2241 : unchecked_curl_easy_setopt(http_handle, CURLOPT_SUPPRESS_CONNECT_HEADERS,
2148 : 1L);
2149 :
2150 2241 : unchecked_curl_easy_setopt(http_handle, CURLOPT_FOLLOWLOCATION, 1);
2151 2241 : unchecked_curl_easy_setopt(http_handle, CURLOPT_MAXREDIRS, 10);
2152 2241 : unchecked_curl_easy_setopt(http_handle, CURLOPT_POSTREDIR,
2153 : CURL_REDIR_POST_ALL);
2154 :
2155 : // Set connect timeout.
2156 : const char *pszConnectTimeout =
2157 2241 : CSLFetchNameValue(papszOptions, "CONNECTTIMEOUT");
2158 2241 : if (pszConnectTimeout == nullptr)
2159 : pszConnectTimeout =
2160 2241 : CPLGetConfigOption("GDAL_HTTP_CONNECTTIMEOUT", nullptr);
2161 2241 : if (pszConnectTimeout != nullptr)
2162 : {
2163 : // coverity[tainted_data]
2164 1 : unchecked_curl_easy_setopt(
2165 : http_handle, CURLOPT_CONNECTTIMEOUT_MS,
2166 : static_cast<int>(1000 * CPLAtof(pszConnectTimeout)));
2167 : }
2168 :
2169 : // Set timeout.
2170 2241 : const char *pszTimeout = CSLFetchNameValue(papszOptions, "TIMEOUT");
2171 2241 : if (pszTimeout == nullptr)
2172 1952 : pszTimeout = CPLGetConfigOption("GDAL_HTTP_TIMEOUT", nullptr);
2173 2241 : if (pszTimeout != nullptr)
2174 : {
2175 : // coverity[tainted_data]
2176 301 : unchecked_curl_easy_setopt(
2177 : http_handle, CURLOPT_TIMEOUT_MS,
2178 : static_cast<int>(1000 * CPLAtof(pszTimeout)));
2179 : }
2180 :
2181 : // Set low speed time and limit.
2182 : const char *pszLowSpeedTime =
2183 2241 : CSLFetchNameValue(papszOptions, "LOW_SPEED_TIME");
2184 2241 : if (pszLowSpeedTime == nullptr)
2185 : pszLowSpeedTime =
2186 2241 : CPLGetConfigOption("GDAL_HTTP_LOW_SPEED_TIME", nullptr);
2187 2241 : if (pszLowSpeedTime != nullptr)
2188 : {
2189 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_LOW_SPEED_TIME,
2190 : atoi(pszLowSpeedTime));
2191 :
2192 : const char *pszLowSpeedLimit =
2193 0 : CSLFetchNameValue(papszOptions, "LOW_SPEED_LIMIT");
2194 0 : if (pszLowSpeedLimit == nullptr)
2195 : pszLowSpeedLimit =
2196 0 : CPLGetConfigOption("GDAL_HTTP_LOW_SPEED_LIMIT", "1");
2197 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_LOW_SPEED_LIMIT,
2198 : atoi(pszLowSpeedLimit));
2199 : }
2200 :
2201 : /* Disable some SSL verification */
2202 2241 : const char *pszUnsafeSSL = CSLFetchNameValue(papszOptions, "UNSAFESSL");
2203 2241 : if (pszUnsafeSSL == nullptr)
2204 2221 : pszUnsafeSSL = CPLGetConfigOption("GDAL_HTTP_UNSAFESSL", nullptr);
2205 2241 : if (pszUnsafeSSL != nullptr && CPLTestBool(pszUnsafeSSL))
2206 : {
2207 20 : unchecked_curl_easy_setopt(http_handle, CURLOPT_SSL_VERIFYPEER, 0L);
2208 20 : unchecked_curl_easy_setopt(http_handle, CURLOPT_SSL_VERIFYHOST, 0L);
2209 : }
2210 :
2211 : const char *pszUseCAPIStore =
2212 2241 : CSLFetchNameValue(papszOptions, "USE_CAPI_STORE");
2213 2241 : if (pszUseCAPIStore == nullptr)
2214 2241 : pszUseCAPIStore = CPLGetConfigOption("GDAL_HTTP_USE_CAPI_STORE", "NO");
2215 2241 : if (CPLTestBool(pszUseCAPIStore))
2216 : {
2217 : #if defined(_WIN32) && defined(HAVE_OPENSSL_CRYPTO)
2218 : // Use certificates from Windows certificate store; requires
2219 : // crypt32.lib, OpenSSL crypto and ssl libraries.
2220 : unchecked_curl_easy_setopt(http_handle, CURLOPT_SSL_CTX_FUNCTION,
2221 : *CPL_ssl_ctx_callback);
2222 : #else // defined(_WIN32) && defined(HAVE_OPENSSL_CRYPTO)
2223 2 : CPLError(CE_Warning, CPLE_NotSupported,
2224 : "GDAL_HTTP_USE_CAPI_STORE requested, but libcurl too old, "
2225 : "non-Windows platform or OpenSSL missing.");
2226 : #endif // defined(_WIN32) && defined(HAVE_OPENSSL_CRYPTO)
2227 : }
2228 :
2229 : // Enable OCSP stapling if requested.
2230 : const char *pszSSLVerifyStatus =
2231 2241 : CSLFetchNameValue(papszOptions, "SSL_VERIFYSTATUS");
2232 2241 : if (pszSSLVerifyStatus == nullptr)
2233 : pszSSLVerifyStatus =
2234 2241 : CPLGetConfigOption("GDAL_HTTP_SSL_VERIFYSTATUS", "NO");
2235 2241 : if (CPLTestBool(pszSSLVerifyStatus))
2236 : {
2237 1 : unchecked_curl_easy_setopt(http_handle, CURLOPT_SSL_VERIFYSTATUS, 1L);
2238 : }
2239 :
2240 : // Custom path to SSL certificates.
2241 2241 : const char *pszCAInfo = CSLFetchNameValue(papszOptions, "CAINFO");
2242 2241 : if (pszCAInfo == nullptr)
2243 : // Name of GDAL environment variable for the CA Bundle path
2244 2241 : pszCAInfo = CPLGetConfigOption("GDAL_CURL_CA_BUNDLE", nullptr);
2245 2241 : if (pszCAInfo == nullptr)
2246 : // Name of environment variable used by the curl binary
2247 2241 : pszCAInfo = CPLGetConfigOption("CURL_CA_BUNDLE", nullptr);
2248 2241 : if (pszCAInfo == nullptr)
2249 : // Name of environment variable used by the curl binary (tested
2250 : // after CURL_CA_BUNDLE
2251 2241 : pszCAInfo = CPLGetConfigOption("SSL_CERT_FILE", nullptr);
2252 : #ifdef _WIN32
2253 : if (pszCAInfo == nullptr)
2254 : {
2255 : pszCAInfo = CPLFindWin32CurlCaBundleCrt();
2256 : }
2257 : #endif
2258 2241 : if (pszCAInfo != nullptr)
2259 : {
2260 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_CAINFO, pszCAInfo);
2261 : }
2262 :
2263 2241 : const char *pszCAPath = CSLFetchNameValue(papszOptions, "CAPATH");
2264 2241 : if (pszCAPath != nullptr)
2265 : {
2266 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_CAPATH, pszCAPath);
2267 : }
2268 :
2269 : // Support for SSL client certificates
2270 :
2271 : // Filename of the the client certificate
2272 2241 : const char *pszSSLCert = CSLFetchNameValue(papszOptions, "SSLCERT");
2273 2241 : if (!pszSSLCert)
2274 2241 : pszSSLCert = CPLGetConfigOption("GDAL_HTTP_SSLCERT", nullptr);
2275 2241 : if (pszSSLCert)
2276 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_SSLCERT, pszSSLCert);
2277 :
2278 : // private key file for TLS and SSL client cert
2279 2241 : const char *pszSSLKey = CSLFetchNameValue(papszOptions, "SSLKEY");
2280 2241 : if (!pszSSLKey)
2281 2241 : pszSSLKey = CPLGetConfigOption("GDAL_HTTP_SSLKEY", nullptr);
2282 2241 : if (pszSSLKey)
2283 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_SSLKEY, pszSSLKey);
2284 :
2285 : // type of client SSL certificate ("PEM", "DER", ...)
2286 2241 : const char *pszSSLCertType = CSLFetchNameValue(papszOptions, "SSLCERTTYPE");
2287 2241 : if (!pszSSLCertType)
2288 2241 : pszSSLCertType = CPLGetConfigOption("GDAL_HTTP_SSLCERTTYPE", nullptr);
2289 2241 : if (pszSSLCertType)
2290 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_SSLCERTTYPE,
2291 : pszSSLCertType);
2292 :
2293 : // passphrase to private key
2294 2241 : const char *pszKeyPasswd = CSLFetchNameValue(papszOptions, "KEYPASSWD");
2295 2241 : if (!pszKeyPasswd)
2296 2241 : pszKeyPasswd = CPLGetConfigOption("GDAL_HTTP_KEYPASSWD", nullptr);
2297 2241 : if (pszKeyPasswd)
2298 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_KEYPASSWD,
2299 : pszKeyPasswd);
2300 :
2301 : /* Set Referer */
2302 2241 : const char *pszReferer = CSLFetchNameValue(papszOptions, "REFERER");
2303 2241 : if (pszReferer != nullptr)
2304 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_REFERER, pszReferer);
2305 :
2306 : /* Set User-Agent */
2307 2241 : const char *pszUserAgent = CSLFetchNameValue(papszOptions, "USERAGENT");
2308 2241 : if (pszUserAgent == nullptr)
2309 2197 : pszUserAgent = CPLGetConfigOption("GDAL_HTTP_USERAGENT",
2310 : gosDefaultUserAgent.c_str());
2311 2241 : if (pszUserAgent != nullptr && !EQUAL(pszUserAgent, ""))
2312 : {
2313 2241 : unchecked_curl_easy_setopt(http_handle, CURLOPT_USERAGENT,
2314 : pszUserAgent);
2315 : }
2316 :
2317 : /* NOSIGNAL should be set to true for timeout to work in multithread
2318 : * environments on Unix, requires libcurl 7.10 or more recent.
2319 : * (this force avoiding the use of signal handlers)
2320 : */
2321 2241 : unchecked_curl_easy_setopt(http_handle, CURLOPT_NOSIGNAL, 1);
2322 :
2323 : const char *pszFormFilePath =
2324 2241 : CSLFetchNameValue(papszOptions, "FORM_FILE_PATH");
2325 : const char *pszParametersCount =
2326 2241 : CSLFetchNameValue(papszOptions, "FORM_ITEM_COUNT");
2327 2241 : if (pszFormFilePath == nullptr && pszParametersCount == nullptr)
2328 : {
2329 : /* Set POST mode */
2330 2239 : const char *pszPost = CSLFetchNameValue(papszOptions, "POSTFIELDS");
2331 2239 : if (pszPost != nullptr)
2332 : {
2333 157 : CPLDebug("HTTP", "These POSTFIELDS were sent:%.4000s", pszPost);
2334 157 : unchecked_curl_easy_setopt(http_handle, CURLOPT_POST, 1);
2335 157 : unchecked_curl_easy_setopt(http_handle, CURLOPT_POSTFIELDS,
2336 : pszPost);
2337 : }
2338 : }
2339 :
2340 : const char *pszCustomRequest =
2341 2241 : CSLFetchNameValue(papszOptions, "CUSTOMREQUEST");
2342 2241 : if (pszCustomRequest != nullptr)
2343 : {
2344 137 : unchecked_curl_easy_setopt(http_handle, CURLOPT_CUSTOMREQUEST,
2345 : pszCustomRequest);
2346 : }
2347 :
2348 2241 : const char *pszCookie = CSLFetchNameValue(papszOptions, "COOKIE");
2349 2241 : if (pszCookie == nullptr)
2350 2241 : pszCookie = CPLGetConfigOption("GDAL_HTTP_COOKIE", nullptr);
2351 2241 : if (pszCookie != nullptr)
2352 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_COOKIE, pszCookie);
2353 :
2354 2241 : const char *pszCookieFile = CSLFetchNameValue(papszOptions, "COOKIEFILE");
2355 2241 : if (pszCookieFile == nullptr)
2356 2241 : pszCookieFile = CPLGetConfigOption("GDAL_HTTP_COOKIEFILE", nullptr);
2357 2241 : if (pszCookieFile != nullptr)
2358 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_COOKIEFILE,
2359 : pszCookieFile);
2360 :
2361 2241 : const char *pszCookieJar = CSLFetchNameValue(papszOptions, "COOKIEJAR");
2362 2241 : if (pszCookieJar == nullptr)
2363 2241 : pszCookieJar = CPLGetConfigOption("GDAL_HTTP_COOKIEJAR", nullptr);
2364 2241 : if (pszCookieJar != nullptr)
2365 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_COOKIEJAR,
2366 : pszCookieJar);
2367 :
2368 : // TCP keep-alive
2369 : const char *pszTCPKeepAlive =
2370 2241 : CSLFetchNameValue(papszOptions, "TCP_KEEPALIVE");
2371 2241 : if (pszTCPKeepAlive == nullptr)
2372 2241 : pszTCPKeepAlive = CPLGetConfigOption("GDAL_HTTP_TCP_KEEPALIVE", "YES");
2373 2241 : if (pszTCPKeepAlive != nullptr && CPLTestBool(pszTCPKeepAlive))
2374 : {
2375 : // Set keep-alive interval.
2376 2241 : int nKeepAliveInterval = 60;
2377 : const char *pszKeepAliveInterval =
2378 2241 : CSLFetchNameValue(papszOptions, "TCP_KEEPINTVL");
2379 2241 : if (pszKeepAliveInterval == nullptr)
2380 : pszKeepAliveInterval =
2381 2241 : CPLGetConfigOption("GDAL_HTTP_TCP_KEEPINTVL", nullptr);
2382 2241 : if (pszKeepAliveInterval != nullptr)
2383 1 : nKeepAliveInterval = atoi(pszKeepAliveInterval);
2384 :
2385 : // Set keep-alive idle wait time.
2386 2241 : int nKeepAliveIdle = 60;
2387 : const char *pszKeepAliveIdle =
2388 2241 : CSLFetchNameValue(papszOptions, "TCP_KEEPIDLE");
2389 2241 : if (pszKeepAliveIdle == nullptr)
2390 : pszKeepAliveIdle =
2391 2241 : CPLGetConfigOption("GDAL_HTTP_TCP_KEEPIDLE", nullptr);
2392 2241 : if (pszKeepAliveIdle != nullptr)
2393 1 : nKeepAliveIdle = atoi(pszKeepAliveIdle);
2394 :
2395 2241 : unchecked_curl_easy_setopt(http_handle, CURLOPT_TCP_KEEPALIVE, 1L);
2396 2241 : unchecked_curl_easy_setopt(http_handle, CURLOPT_TCP_KEEPINTVL,
2397 : nKeepAliveInterval);
2398 2241 : unchecked_curl_easy_setopt(http_handle, CURLOPT_TCP_KEEPIDLE,
2399 : nKeepAliveIdle);
2400 : }
2401 :
2402 2241 : struct curl_slist *headers = nullptr;
2403 2241 : const char *pszHeaderFile = CSLFetchNameValue(papszOptions, "HEADER_FILE");
2404 2241 : if (pszHeaderFile == nullptr)
2405 2239 : pszHeaderFile = CPLGetConfigOption("GDAL_HTTP_HEADER_FILE", nullptr);
2406 2241 : if (pszHeaderFile != nullptr)
2407 : {
2408 2 : VSILFILE *fp = nullptr;
2409 : // Do not allow /vsicurl/ access from /vsicurl because of
2410 : // GetCurlHandleFor() e.g. "/vsicurl/,HEADER_FILE=/vsicurl/,url= " would
2411 : // cause use of memory after free
2412 2 : if (!STARTS_WITH(pszHeaderFile, "/vsi") ||
2413 2 : STARTS_WITH(pszHeaderFile, "/vsimem/"))
2414 : {
2415 2 : fp = VSIFOpenL(pszHeaderFile, "rb");
2416 : }
2417 2 : if (fp == nullptr)
2418 : {
2419 0 : CPLError(CE_Failure, CPLE_FileIO, "Cannot read %s", pszHeaderFile);
2420 : }
2421 : else
2422 : {
2423 2 : const char *pszLine = nullptr;
2424 4 : while ((pszLine = CPLReadLineL(fp)) != nullptr)
2425 : {
2426 2 : headers = curl_slist_append(headers, pszLine);
2427 : }
2428 2 : VSIFCloseL(fp);
2429 : }
2430 : }
2431 :
2432 2241 : const char *pszHeaders = CSLFetchNameValue(papszOptions, "HEADERS");
2433 2241 : if (pszHeaders == nullptr)
2434 1648 : pszHeaders = CPLGetConfigOption("GDAL_HTTP_HEADERS", nullptr);
2435 2241 : if (pszHeaders)
2436 : {
2437 593 : bool bHeadersDone = false;
2438 : // Compatibility hack for "HEADERS=Accept: text/plain, application/json"
2439 593 : if (strstr(pszHeaders, "\r\n") == nullptr)
2440 : {
2441 538 : const char *pszComma = strchr(pszHeaders, ',');
2442 538 : if (pszComma != nullptr && strchr(pszComma, ':') == nullptr)
2443 : {
2444 234 : headers = curl_slist_append(headers, pszHeaders);
2445 234 : bHeadersDone = true;
2446 : }
2447 : }
2448 593 : if (!bHeadersDone)
2449 : {
2450 : // We accept both raw headers with \r\n as a separator, or as
2451 : // a comma separated list of foo: bar values.
2452 : const CPLStringList aosTokens(
2453 359 : strstr(pszHeaders, "\r\n")
2454 55 : ? CSLTokenizeString2(pszHeaders, "\r\n", 0)
2455 773 : : CSLTokenizeString2(pszHeaders, ",", CSLT_HONOURSTRINGS));
2456 777 : for (int i = 0; i < aosTokens.size(); ++i)
2457 : {
2458 418 : headers = curl_slist_append(headers, aosTokens[i]);
2459 : }
2460 : }
2461 : }
2462 :
2463 2241 : return headers;
2464 : }
2465 :
2466 : /************************************************************************/
2467 : /* CPLHTTPIgnoreSigPipe() */
2468 : /************************************************************************/
2469 :
2470 : /* If using OpenSSL with Curl, openssl can cause SIGPIPE to be triggered */
2471 : /* As we set CURLOPT_NOSIGNAL = 1, we must manually handle this situation */
2472 :
2473 2400 : void *CPLHTTPIgnoreSigPipe()
2474 : {
2475 : #if defined(SIGPIPE) && defined(HAVE_SIGACTION)
2476 : struct sigaction old_pipe_act;
2477 : struct sigaction action;
2478 : /* Get previous handler */
2479 2400 : memset(&old_pipe_act, 0, sizeof(struct sigaction));
2480 2400 : sigaction(SIGPIPE, nullptr, &old_pipe_act);
2481 :
2482 : /* Install new handler */
2483 2400 : action = old_pipe_act;
2484 2400 : action.sa_handler = SIG_IGN;
2485 2400 : sigaction(SIGPIPE, &action, nullptr);
2486 :
2487 2400 : void *ret = CPLMalloc(sizeof(old_pipe_act));
2488 2400 : memcpy(ret, &old_pipe_act, sizeof(old_pipe_act));
2489 2400 : return ret;
2490 : #else
2491 : return nullptr;
2492 : #endif
2493 : }
2494 :
2495 : /************************************************************************/
2496 : /* CPLHTTPRestoreSigPipeHandler() */
2497 : /************************************************************************/
2498 :
2499 2400 : void CPLHTTPRestoreSigPipeHandler(void *old_handler)
2500 : {
2501 : #if defined(SIGPIPE) && defined(HAVE_SIGACTION)
2502 2400 : sigaction(SIGPIPE, static_cast<struct sigaction *>(old_handler), nullptr);
2503 2400 : CPLFree(old_handler);
2504 : #else
2505 : (void)old_handler;
2506 : #endif
2507 2400 : }
2508 :
2509 : #endif // def HAVE_CURL
2510 :
2511 : /************************************************************************/
2512 : /* CPLHTTPEnabled() */
2513 : /************************************************************************/
2514 :
2515 : /**
2516 : * \brief Return if CPLHTTP services can be useful
2517 : *
2518 : * Those services depend on GDAL being build with libcurl support.
2519 : *
2520 : * @return TRUE if libcurl support is enabled
2521 : */
2522 8 : int CPLHTTPEnabled()
2523 :
2524 : {
2525 : #ifdef HAVE_CURL
2526 8 : return TRUE;
2527 : #else
2528 : return FALSE;
2529 : #endif
2530 : }
2531 :
2532 : /************************************************************************/
2533 : /* CPLHTTPCleanup() */
2534 : /************************************************************************/
2535 :
2536 : /**
2537 : * \brief Cleanup function to call at application termination
2538 : */
2539 852 : void CPLHTTPCleanup()
2540 :
2541 : {
2542 : #ifdef HAVE_CURL
2543 852 : if (!hSessionMapMutex)
2544 851 : return;
2545 :
2546 : {
2547 2 : CPLMutexHolder oHolder(&hSessionMapMutex);
2548 1 : if (poSessionMap)
2549 : {
2550 0 : for (auto &kv : *poSessionMap)
2551 : {
2552 0 : curl_easy_cleanup(kv.second);
2553 : }
2554 0 : delete poSessionMap;
2555 0 : poSessionMap = nullptr;
2556 : }
2557 1 : if (poSessionMultiMap)
2558 : {
2559 0 : for (auto &kv : *poSessionMultiMap)
2560 : {
2561 0 : VSICURLMultiCleanup(kv.second);
2562 : }
2563 0 : delete poSessionMultiMap;
2564 0 : poSessionMultiMap = nullptr;
2565 : }
2566 : }
2567 :
2568 : // Not quite a safe sequence.
2569 1 : CPLDestroyMutex(hSessionMapMutex);
2570 1 : hSessionMapMutex = nullptr;
2571 :
2572 : #if defined(_WIN32) && defined(HAVE_OPENSSL_CRYPTO)
2573 : // This cleanup must be absolutely done before CPLOpenSSLCleanup()
2574 : // for some unknown reason, but otherwise X509_free() in
2575 : // CPLWindowsCertificateListCleanup() will crash.
2576 : CPLWindowsCertificateListCleanup();
2577 : #endif
2578 :
2579 : #if defined(HAVE_OPENSSL_CRYPTO) && OPENSSL_VERSION_NUMBER < 0x10100000
2580 : CPLOpenSSLCleanup();
2581 : #endif
2582 :
2583 : #endif
2584 : }
2585 :
2586 : /************************************************************************/
2587 : /* CPLHTTPDestroyResult() */
2588 : /************************************************************************/
2589 :
2590 : /**
2591 : * \brief Clean the memory associated with the return value of CPLHTTPFetch()
2592 : *
2593 : * @param psResult pointer to the return value of CPLHTTPFetch()
2594 : */
2595 2153 : void CPLHTTPDestroyResult(CPLHTTPResult *psResult)
2596 :
2597 : {
2598 2153 : if (psResult)
2599 : {
2600 1968 : CPLFree(psResult->pabyData);
2601 1968 : CPLFree(psResult->pszErrBuf);
2602 1968 : CPLFree(psResult->pszContentType);
2603 1968 : CSLDestroy(psResult->papszHeaders);
2604 :
2605 2025 : for (int i = 0; i < psResult->nMimePartCount; i++)
2606 : {
2607 57 : CSLDestroy(psResult->pasMimePart[i].papszHeaders);
2608 : }
2609 1968 : CPLFree(psResult->pasMimePart);
2610 :
2611 1968 : CPLFree(psResult);
2612 : }
2613 2153 : }
2614 :
2615 : /************************************************************************/
2616 : /* CPLHTTPParseMultipartMime() */
2617 : /************************************************************************/
2618 :
2619 : /**
2620 : * \brief Parses a MIME multipart message.
2621 : *
2622 : * This function will iterate over each part and put it in a separate
2623 : * element of the pasMimePart array of the provided psResult structure.
2624 : *
2625 : * @param psResult pointer to the return value of CPLHTTPFetch()
2626 : * @return TRUE if the message contains MIME multipart message.
2627 : */
2628 45 : int CPLHTTPParseMultipartMime(CPLHTTPResult *psResult)
2629 :
2630 : {
2631 : /* -------------------------------------------------------------------- */
2632 : /* Is it already done? */
2633 : /* -------------------------------------------------------------------- */
2634 45 : if (psResult->nMimePartCount > 0)
2635 5 : return TRUE;
2636 :
2637 : /* -------------------------------------------------------------------- */
2638 : /* Find the boundary setting in the content type. */
2639 : /* -------------------------------------------------------------------- */
2640 40 : const char *pszBound = nullptr;
2641 :
2642 40 : if (psResult->pszContentType != nullptr)
2643 38 : pszBound = strstr(psResult->pszContentType, "boundary=");
2644 :
2645 40 : if (pszBound == nullptr)
2646 : {
2647 2 : CPLError(CE_Failure, CPLE_AppDefined,
2648 : "Unable to parse multi-part mime, no boundary setting.");
2649 2 : return FALSE;
2650 : }
2651 :
2652 76 : CPLString osBoundary;
2653 : char **papszTokens =
2654 38 : CSLTokenizeStringComplex(pszBound + 9, "\n ;", TRUE, FALSE);
2655 :
2656 38 : if (CSLCount(papszTokens) == 0 || strlen(papszTokens[0]) == 0)
2657 : {
2658 1 : CPLError(CE_Failure, CPLE_AppDefined,
2659 : "Unable to parse multi-part mime, boundary not parsable.");
2660 1 : CSLDestroy(papszTokens);
2661 1 : return FALSE;
2662 : }
2663 :
2664 37 : osBoundary = "--";
2665 37 : osBoundary += papszTokens[0];
2666 37 : CSLDestroy(papszTokens);
2667 :
2668 : /* -------------------------------------------------------------------- */
2669 : /* Find the start of the first chunk. */
2670 : /* -------------------------------------------------------------------- */
2671 37 : char *pszNext = psResult->pabyData
2672 37 : ? strstr(reinterpret_cast<char *>(psResult->pabyData),
2673 : osBoundary.c_str())
2674 37 : : nullptr;
2675 :
2676 37 : if (pszNext == nullptr)
2677 : {
2678 1 : CPLError(CE_Failure, CPLE_AppDefined, "No parts found.");
2679 1 : return FALSE;
2680 : }
2681 :
2682 36 : pszNext += osBoundary.size();
2683 146 : while (*pszNext != '\n' && *pszNext != '\r' && *pszNext != '\0')
2684 110 : pszNext++;
2685 36 : if (*pszNext == '\r')
2686 30 : pszNext++;
2687 36 : if (*pszNext == '\n')
2688 36 : pszNext++;
2689 :
2690 : /* -------------------------------------------------------------------- */
2691 : /* Loop over parts... */
2692 : /* -------------------------------------------------------------------- */
2693 : while (true)
2694 : {
2695 57 : psResult->nMimePartCount++;
2696 57 : psResult->pasMimePart = static_cast<CPLMimePart *>(
2697 114 : CPLRealloc(psResult->pasMimePart,
2698 57 : sizeof(CPLMimePart) * psResult->nMimePartCount));
2699 :
2700 57 : CPLMimePart *psPart =
2701 57 : psResult->pasMimePart + psResult->nMimePartCount - 1;
2702 :
2703 57 : memset(psPart, 0, sizeof(CPLMimePart));
2704 :
2705 : /* --------------------------------------------------------------------
2706 : */
2707 : /* Collect headers. */
2708 : /* --------------------------------------------------------------------
2709 : */
2710 145 : while (*pszNext != '\n' && *pszNext != '\r' && *pszNext != '\0')
2711 : {
2712 90 : if (!STARTS_WITH(pszNext, "Content-"))
2713 : {
2714 1 : break;
2715 : }
2716 89 : char *pszEOL = strstr(pszNext, "\n");
2717 :
2718 89 : if (pszEOL == nullptr)
2719 : {
2720 1 : CPLError(CE_Failure, CPLE_AppDefined,
2721 : "Error while parsing multipart content (at line %d)",
2722 : __LINE__);
2723 1 : return FALSE;
2724 : }
2725 :
2726 88 : *pszEOL = '\0';
2727 88 : bool bRestoreAntislashR = false;
2728 88 : if (pszEOL - pszNext > 1 && pszEOL[-1] == '\r')
2729 : {
2730 80 : bRestoreAntislashR = true;
2731 80 : pszEOL[-1] = '\0';
2732 : }
2733 88 : char *pszKey = nullptr;
2734 88 : const char *pszValue = CPLParseNameValue(pszNext, &pszKey);
2735 88 : if (pszKey && pszValue)
2736 : {
2737 88 : psPart->papszHeaders =
2738 88 : CSLSetNameValue(psPart->papszHeaders, pszKey, pszValue);
2739 : }
2740 88 : CPLFree(pszKey);
2741 88 : if (bRestoreAntislashR)
2742 80 : pszEOL[-1] = '\r';
2743 88 : *pszEOL = '\n';
2744 :
2745 88 : pszNext = pszEOL + 1;
2746 : }
2747 :
2748 56 : if (*pszNext == '\r')
2749 54 : pszNext++;
2750 56 : if (*pszNext == '\n')
2751 54 : pszNext++;
2752 :
2753 : /* --------------------------------------------------------------------
2754 : */
2755 : /* Work out the data block size. */
2756 : /* --------------------------------------------------------------------
2757 : */
2758 56 : psPart->pabyData = reinterpret_cast<GByte *>(pszNext);
2759 :
2760 56 : int nBytesAvail = psResult->nDataLen -
2761 56 : static_cast<int>(pszNext - reinterpret_cast<char *>(
2762 56 : psResult->pabyData));
2763 :
2764 6579040 : while (nBytesAvail > 0 &&
2765 3289520 : (*pszNext != '-' ||
2766 109 : strncmp(pszNext, osBoundary, osBoundary.size()) != 0))
2767 : {
2768 3289470 : pszNext++;
2769 3289470 : nBytesAvail--;
2770 : }
2771 :
2772 56 : if (nBytesAvail == 0)
2773 : {
2774 2 : CPLError(CE_Failure, CPLE_AppDefined,
2775 : "Error while parsing multipart content (at line %d)",
2776 : __LINE__);
2777 2 : return FALSE;
2778 : }
2779 :
2780 54 : psPart->nDataLen = static_cast<int>(
2781 54 : pszNext - reinterpret_cast<char *>(psPart->pabyData));
2782 : // Normally the part should end with "\r\n--boundary_marker"
2783 54 : if (psPart->nDataLen >= 2 && pszNext[-2] == '\r' && pszNext[-1] == '\n')
2784 : {
2785 46 : psPart->nDataLen -= 2;
2786 : }
2787 :
2788 54 : pszNext += osBoundary.size();
2789 :
2790 54 : if (STARTS_WITH(pszNext, "--"))
2791 : {
2792 31 : break;
2793 : }
2794 :
2795 23 : if (*pszNext == '\r')
2796 19 : pszNext++;
2797 23 : if (*pszNext == '\n')
2798 21 : pszNext++;
2799 : else
2800 : {
2801 2 : CPLError(CE_Failure, CPLE_AppDefined,
2802 : "Error while parsing multipart content (at line %d)",
2803 : __LINE__);
2804 2 : return FALSE;
2805 : }
2806 21 : }
2807 :
2808 31 : return TRUE;
2809 : }
2810 :
2811 : #if defined(__GNUC__)
2812 : #pragma GCC diagnostic pop
2813 : #endif
|
