Line data Source code
1 : /******************************************************************************
2 : *
3 : * Project: libcurl based HTTP client
4 : * Purpose: libcurl based HTTP client
5 : * Author: Frank Warmerdam, warmerdam@pobox.com
6 : *
7 : ******************************************************************************
8 : * Copyright (c) 2006, Frank Warmerdam
9 : * Copyright (c) 2008-2013, Even Rouault <even dot rouault at spatialys.com>
10 : *
11 : * SPDX-License-Identifier: MIT
12 : ****************************************************************************/
13 :
14 : #include "cpl_port.h"
15 : #include "cpl_http.h"
16 :
17 : #include <cstddef>
18 : #include <cstring>
19 :
20 : #include <algorithm>
21 : #include <array>
22 : #include <map>
23 : #include <mutex>
24 : #include <string>
25 : #include <vector>
26 :
27 : #include "cpl_http.h"
28 : #include "cpl_error.h"
29 : #include "cpl_multiproc.h"
30 : #include "cpl_vsi_virtual.h"
31 : #include "cpl_vsil_curl_class.h"
32 :
33 : // gcc or clang complains about C-style cast in #define like
34 : // CURL_ZERO_TERMINATED
35 : #if defined(__GNUC__)
36 : #pragma GCC diagnostic push
37 : #pragma GCC diagnostic ignored "-Wold-style-cast"
38 : #ifdef HAVE_WFLAG_CAST_FUNCTION_TYPE
39 : #pragma GCC diagnostic ignored "-Wcast-function-type"
40 : #endif
41 : #endif
42 :
43 : #ifdef HAVE_CURL
44 :
45 : #include "cpl_curl_priv.h"
46 :
47 : #ifdef HAVE_OPENSSL_CRYPTO
48 : #include <openssl/err.h>
49 : #include <openssl/ssl.h>
50 : #include <openssl/x509v3.h>
51 :
52 : #if defined(_WIN32)
53 : #include <wincrypt.h>
54 : #endif
55 :
56 : #endif
57 :
58 : #ifdef HAVE_SIGACTION
59 : #include <signal.h>
60 : #endif
61 :
62 : #define unchecked_curl_easy_setopt(handle, opt, param) \
63 : CPL_IGNORE_RET_VAL(curl_easy_setopt(handle, opt, param))
64 :
65 : #endif // HAVE_CURL
66 :
67 : // list of named persistent http sessions
68 :
69 : #ifdef HAVE_CURL
70 : static std::map<CPLString, CURL *> *poSessionMap = nullptr;
71 : static std::map<CPLString, CURLM *> *poSessionMultiMap = nullptr;
72 : static CPLMutex *hSessionMapMutex = nullptr;
73 : static bool bHasCheckVersion = false;
74 : static bool bSupportGZip = false;
75 : static bool bSupportHTTP2 = false;
76 : #if defined(_WIN32) && defined(HAVE_OPENSSL_CRYPTO)
77 : static std::vector<X509 *> *poWindowsCertificateList = nullptr;
78 :
79 : #if (OPENSSL_VERSION_NUMBER < 0x10100000L)
80 : #define EVP_PKEY_get0_RSA(x) (x->pkey.rsa)
81 : #define EVP_PKEY_get0_DSA(x) (x->pkey.dsa)
82 : #define X509_get_extension_flags(x) (x->ex_flags)
83 : #define X509_get_key_usage(x) (x->ex_kusage)
84 : #define X509_get_extended_key_usage(x) (x->ex_xkusage)
85 : #endif
86 :
87 : #endif // defined(_WIN32) && defined(HAVE_OPENSSL_CRYPTO)
88 :
89 : #if defined(HAVE_OPENSSL_CRYPTO) && OPENSSL_VERSION_NUMBER < 0x10100000
90 :
91 : // Ported from https://curl.haxx.se/libcurl/c/opensslthreadlock.html
92 : static CPLMutex **pahSSLMutex = nullptr;
93 :
94 : static void CPLOpenSSLLockingFunction(int mode, int n, const char * /*file*/,
95 : int /*line*/)
96 : {
97 : if (mode & CRYPTO_LOCK)
98 : {
99 : CPLAcquireMutex(pahSSLMutex[n], 3600.0);
100 : }
101 : else
102 : {
103 : CPLReleaseMutex(pahSSLMutex[n]);
104 : }
105 : }
106 :
107 : static unsigned long CPLOpenSSLIdCallback(void)
108 : {
109 : return static_cast<unsigned long>(CPLGetPID());
110 : }
111 :
112 : static void CPLOpenSSLInit()
113 : {
114 : if (strstr(curl_version(), "OpenSSL") &&
115 : CPLTestBool(CPLGetConfigOption("CPL_OPENSSL_INIT_ENABLED", "YES")) &&
116 : CRYPTO_get_id_callback() == nullptr)
117 : {
118 : pahSSLMutex = static_cast<CPLMutex **>(
119 : CPLMalloc(CRYPTO_num_locks() * sizeof(CPLMutex *)));
120 : for (int i = 0; i < CRYPTO_num_locks(); i++)
121 : {
122 : pahSSLMutex[i] = CPLCreateMutex();
123 : CPLReleaseMutex(pahSSLMutex[i]);
124 : }
125 : CRYPTO_set_id_callback(CPLOpenSSLIdCallback);
126 : CRYPTO_set_locking_callback(CPLOpenSSLLockingFunction);
127 : }
128 : }
129 :
130 : static void CPLOpenSSLCleanup()
131 : {
132 : if (pahSSLMutex)
133 : {
134 : for (int i = 0; i < CRYPTO_num_locks(); i++)
135 : {
136 : CPLDestroyMutex(pahSSLMutex[i]);
137 : }
138 : CPLFree(pahSSLMutex);
139 : pahSSLMutex = nullptr;
140 : CRYPTO_set_id_callback(nullptr);
141 : CRYPTO_set_locking_callback(nullptr);
142 : }
143 : }
144 :
145 : #endif
146 :
147 : #if defined(_WIN32) && defined(HAVE_OPENSSL_CRYPTO)
148 :
149 : /************************************************************************/
150 : /* CPLWindowsCertificateListCleanup() */
151 : /************************************************************************/
152 :
153 : static void CPLWindowsCertificateListCleanup()
154 : {
155 : if (poWindowsCertificateList)
156 : {
157 : for (auto &&pX509 : *poWindowsCertificateList)
158 : {
159 : X509_free(pX509);
160 : }
161 : delete poWindowsCertificateList;
162 : poWindowsCertificateList = nullptr;
163 : }
164 : }
165 :
166 : /************************************************************************/
167 : /* LoadCAPICertificates() */
168 : /************************************************************************/
169 :
170 : static CPLErr LoadCAPICertificates(const char *pszName,
171 : std::vector<X509 *> *poCertificateList)
172 : {
173 : CPLAssert(pszName);
174 : CPLAssert(poCertificateList);
175 :
176 : HCERTSTORE pCertStore = CertOpenSystemStore(
177 : reinterpret_cast<HCRYPTPROV_LEGACY>(nullptr), pszName);
178 : if (pCertStore == nullptr)
179 : {
180 : CPLError(CE_Failure, CPLE_AppDefined,
181 : "CPLLoadCAPICertificates(): Unable open system "
182 : "certificate store %s.",
183 : pszName);
184 : return CE_Failure;
185 : }
186 :
187 : PCCERT_CONTEXT pCertificate =
188 : CertEnumCertificatesInStore(pCertStore, nullptr);
189 : while (pCertificate != nullptr)
190 : {
191 : X509 *pX509 = d2i_X509(
192 : nullptr,
193 : const_cast<unsigned char const **>(&pCertificate->pbCertEncoded),
194 : pCertificate->cbCertEncoded);
195 : if (pX509 == nullptr)
196 : {
197 : CPLError(CE_Warning, CPLE_AppDefined,
198 : "CPLLoadCAPICertificates(): CertEnumCertificatesInStore() "
199 : "returned a null certificate, skipping.");
200 : }
201 : else
202 : {
203 : #ifdef DEBUG_VERBOSE
204 : char szSubject[256] = {0};
205 : CPLString osSubject;
206 : X509_NAME *pName = X509_get_subject_name(pX509);
207 : if (pName)
208 : {
209 : X509_NAME_oneline(pName, szSubject, sizeof(szSubject));
210 : osSubject = szSubject;
211 : }
212 : if (!osSubject.empty())
213 : CPLDebug("HTTP", "SSL Certificate: %s", osSubject.c_str());
214 : #endif
215 : poCertificateList->push_back(pX509);
216 : }
217 : pCertificate = CertEnumCertificatesInStore(pCertStore, pCertificate);
218 : }
219 : CertCloseStore(pCertStore, 0);
220 : return CE_None;
221 : }
222 :
223 : /************************************************************************/
224 : /* CPL_ssl_ctx_callback() */
225 : /************************************************************************/
226 :
227 : // Load certificates from Windows Crypto API store.
228 : static CURLcode CPL_ssl_ctx_callback(CURL *, void *pSSL, void *)
229 : {
230 : SSL_CTX *pSSL_CTX = static_cast<SSL_CTX *>(pSSL);
231 : if (pSSL_CTX == nullptr)
232 : {
233 : CPLError(CE_Failure, CPLE_AppDefined,
234 : "CPL_ssl_ctx_callback(): OpenSSL context pointer is NULL.");
235 : return CURLE_ABORTED_BY_CALLBACK;
236 : }
237 :
238 : static std::mutex goMutex;
239 : {
240 : std::lock_guard<std::mutex> oLock(goMutex);
241 : if (poWindowsCertificateList == nullptr)
242 : {
243 : poWindowsCertificateList = new std::vector<X509 *>();
244 : if (!poWindowsCertificateList)
245 : {
246 : CPLError(CE_Failure, CPLE_AppDefined,
247 : "CPL_ssl_ctx_callback(): Unable to allocate "
248 : "structure to hold certificates.");
249 : return CURLE_FAILED_INIT;
250 : }
251 :
252 : const std::array<const char *, 3> aszStores{
253 : {"CA", "AuthRoot", "ROOT"}};
254 : for (auto &&pszStore : aszStores)
255 : {
256 : if (LoadCAPICertificates(pszStore, poWindowsCertificateList) ==
257 : CE_Failure)
258 : {
259 : CPLError(
260 : CE_Failure, CPLE_AppDefined,
261 : "CPL_ssl_ctx_callback(): Unable to load certificates "
262 : "from '%s' store.",
263 : pszStore);
264 : return CURLE_FAILED_INIT;
265 : }
266 : }
267 :
268 : CPLDebug("HTTP", "Loading %d certificates from Windows store.",
269 : static_cast<int>(poWindowsCertificateList->size()));
270 : }
271 : }
272 :
273 : X509_STORE *pX509Store = SSL_CTX_get_cert_store(pSSL_CTX);
274 : for (X509 *x509 : *poWindowsCertificateList)
275 : X509_STORE_add_cert(pX509Store, x509);
276 :
277 : return CURLE_OK;
278 : }
279 :
280 : #endif // defined(_WIN32) && defined (HAVE_OPENSSL_CRYPTO)
281 :
282 : /************************************************************************/
283 : /* CheckCurlFeatures() */
284 : /************************************************************************/
285 :
286 2601 : static void CheckCurlFeatures()
287 : {
288 5202 : CPLMutexHolder oHolder(&hSessionMapMutex);
289 2601 : if (!bHasCheckVersion)
290 : {
291 7 : const char *pszVersion = curl_version();
292 7 : CPLDebug("HTTP", "%s", pszVersion);
293 7 : bSupportGZip = strstr(pszVersion, "zlib/") != nullptr;
294 7 : bSupportHTTP2 = strstr(curl_version(), "nghttp2/") != nullptr;
295 7 : bHasCheckVersion = true;
296 :
297 7 : curl_version_info_data *data = curl_version_info(CURLVERSION_NOW);
298 7 : if (data->version_num < LIBCURL_VERSION_NUM)
299 : {
300 0 : CPLError(CE_Warning, CPLE_AppDefined,
301 : "GDAL was built against curl %d.%d.%d, but is "
302 : "running against %s. Runtime failure is likely !",
303 : LIBCURL_VERSION_MAJOR, LIBCURL_VERSION_MINOR,
304 : LIBCURL_VERSION_PATCH, data->version);
305 : }
306 7 : else if (data->version_num > LIBCURL_VERSION_NUM)
307 : {
308 0 : CPLDebug("HTTP",
309 : "GDAL was built against curl %d.%d.%d, but is "
310 : "running against %s.",
311 : LIBCURL_VERSION_MAJOR, LIBCURL_VERSION_MINOR,
312 : LIBCURL_VERSION_PATCH, data->version);
313 : }
314 :
315 : #if defined(HAVE_OPENSSL_CRYPTO) && OPENSSL_VERSION_NUMBER < 0x10100000
316 : CPLOpenSSLInit();
317 : #endif
318 : }
319 2601 : }
320 :
321 : /************************************************************************/
322 : /* CPLWriteFct() */
323 : /* */
324 : /* Append incoming text to our collection buffer, reallocating */
325 : /* it larger as needed. */
326 : /************************************************************************/
327 :
328 : class CPLHTTPResultWithLimit
329 : {
330 : public:
331 : CPLHTTPResult *psResult = nullptr;
332 : int nMaxFileSize = 0;
333 : };
334 :
335 12727 : static size_t CPLWriteFct(void *buffer, size_t size, size_t nmemb,
336 : void *reqInfo)
337 :
338 : {
339 12727 : CPLHTTPResultWithLimit *psResultWithLimit =
340 : static_cast<CPLHTTPResultWithLimit *>(reqInfo);
341 12727 : CPLHTTPResult *psResult = psResultWithLimit->psResult;
342 :
343 12727 : int nBytesToWrite = static_cast<int>(nmemb) * static_cast<int>(size);
344 12727 : int nNewSize = psResult->nDataLen + nBytesToWrite + 1;
345 12727 : if (nNewSize > psResult->nDataAlloc)
346 : {
347 1640 : psResult->nDataAlloc = static_cast<int>(nNewSize * 1.25 + 100);
348 : GByte *pabyNewData = static_cast<GByte *>(
349 1640 : VSIRealloc(psResult->pabyData, psResult->nDataAlloc));
350 1640 : if (pabyNewData == nullptr)
351 : {
352 0 : VSIFree(psResult->pabyData);
353 0 : psResult->pabyData = nullptr;
354 0 : psResult->pszErrBuf = CPLStrdup(CPLString().Printf(
355 : "Out of memory allocating %d bytes for HTTP data buffer.",
356 0 : psResult->nDataAlloc));
357 0 : psResult->nDataAlloc = psResult->nDataLen = 0;
358 :
359 0 : return 0;
360 : }
361 1640 : psResult->pabyData = pabyNewData;
362 : }
363 :
364 12727 : memcpy(psResult->pabyData + psResult->nDataLen, buffer, nBytesToWrite);
365 :
366 12727 : psResult->nDataLen += nBytesToWrite;
367 12727 : psResult->pabyData[psResult->nDataLen] = 0;
368 :
369 12727 : if (psResultWithLimit->nMaxFileSize > 0 &&
370 8 : psResult->nDataLen > psResultWithLimit->nMaxFileSize)
371 : {
372 1 : CPLError(CE_Failure, CPLE_AppDefined, "Maximum file size reached");
373 1 : return 0;
374 : }
375 :
376 12726 : return nmemb;
377 : }
378 :
379 : /************************************************************************/
380 : /* CPLHdrWriteFct() */
381 : /************************************************************************/
382 7594 : static size_t CPLHdrWriteFct(void *buffer, size_t size, size_t nmemb,
383 : void *reqInfo)
384 : {
385 7594 : CPLHTTPResult *psResult = static_cast<CPLHTTPResult *>(reqInfo);
386 : // Copy the buffer to a char* and initialize with zeros (zero
387 : // terminate as well).
388 7594 : size_t nBytes = size * nmemb;
389 7594 : char *pszHdr = static_cast<char *>(CPLCalloc(1, nBytes + 1));
390 7594 : memcpy(pszHdr, buffer, nBytes);
391 7594 : size_t nIdx = nBytes - 1;
392 : // Remove end of line characters
393 21611 : while (nIdx > 0 && (pszHdr[nIdx] == '\r' || pszHdr[nIdx] == '\n'))
394 : {
395 14017 : pszHdr[nIdx] = 0;
396 14017 : nIdx--;
397 : }
398 7594 : char *pszKey = nullptr;
399 7594 : const char *pszValue = CPLParseNameValue(pszHdr, &pszKey);
400 7594 : if (pszKey && pszValue)
401 : {
402 5264 : psResult->papszHeaders =
403 5264 : CSLAddNameValue(psResult->papszHeaders, pszKey, pszValue);
404 : }
405 7594 : CPLFree(pszHdr);
406 7594 : CPLFree(pszKey);
407 7594 : return nmemb;
408 : }
409 :
410 : /************************************************************************/
411 : /* CPLHTTPReadFunction() */
412 : /************************************************************************/
413 0 : static size_t CPLHTTPReadFunction(char *buffer, size_t size, size_t nitems,
414 : void *arg)
415 : {
416 0 : return VSIFReadL(buffer, size, nitems, static_cast<VSILFILE *>(arg));
417 : }
418 :
419 : /************************************************************************/
420 : /* CPLHTTPSeekFunction() */
421 : /************************************************************************/
422 0 : static int CPLHTTPSeekFunction(void *arg, curl_off_t offset, int origin)
423 : {
424 0 : if (VSIFSeekL(static_cast<VSILFILE *>(arg), offset, origin) == 0)
425 0 : return CURL_SEEKFUNC_OK;
426 : else
427 0 : return CURL_SEEKFUNC_FAIL;
428 : }
429 :
430 : /************************************************************************/
431 : /* CPLHTTPFreeFunction() */
432 : /************************************************************************/
433 0 : static void CPLHTTPFreeFunction(void *arg)
434 : {
435 0 : VSIFCloseL(static_cast<VSILFILE *>(arg));
436 0 : }
437 :
438 : typedef struct
439 : {
440 : GDALProgressFunc pfnProgress;
441 : void *pProgressArg;
442 : } CurlProcessData, *CurlProcessDataL;
443 :
444 370 : static int NewProcessFunction(void *p, curl_off_t dltotal, curl_off_t dlnow,
445 : curl_off_t ultotal, curl_off_t ulnow)
446 : {
447 370 : CurlProcessDataL pData = static_cast<CurlProcessDataL>(p);
448 370 : if (nullptr != pData && pData->pfnProgress)
449 : {
450 370 : if (dltotal > 0)
451 : {
452 0 : const double dfDone = double(dlnow) / dltotal;
453 0 : return pData->pfnProgress(dfDone, "Downloading ...",
454 : pData->pProgressArg) == TRUE
455 0 : ? 0
456 0 : : 1;
457 : }
458 370 : else if (ultotal > 0)
459 : {
460 0 : const double dfDone = double(ulnow) / ultotal;
461 0 : return pData->pfnProgress(dfDone, "Uploading ...",
462 : pData->pProgressArg) == TRUE
463 0 : ? 0
464 0 : : 1;
465 : }
466 : }
467 370 : return 0;
468 : }
469 :
470 : #endif /* def HAVE_CURL */
471 :
472 : /************************************************************************/
473 : /* CPLHTTPSetDefaultUserAgent() */
474 : /************************************************************************/
475 :
476 : static std::string gosDefaultUserAgent;
477 :
478 : /**
479 : * \brief Set the default user agent.
480 : *
481 : * GDAL core will by default call this method with "GDAL/x.y.z" where x.y.z
482 : * is the GDAL version number (during driver initialization). Applications may
483 : * override it.
484 : *
485 : * @param pszUserAgent String (or nullptr to cancel the default user agent)
486 : *
487 : * @since GDAL 3.7
488 : */
489 1743 : void CPLHTTPSetDefaultUserAgent(const char *pszUserAgent)
490 : {
491 1743 : gosDefaultUserAgent = pszUserAgent ? pszUserAgent : "";
492 1743 : }
493 :
494 : /************************************************************************/
495 : /* CPLHTTPGetOptionsFromEnv() */
496 : /************************************************************************/
497 :
498 : typedef struct
499 : {
500 : const char *pszEnvVar;
501 : const char *pszOptionName;
502 : } TupleEnvVarOptionName;
503 :
504 : constexpr TupleEnvVarOptionName asAssocEnvVarOptionName[] = {
505 : {"GDAL_HTTP_VERSION", "HTTP_VERSION"},
506 : {"GDAL_HTTP_CONNECTTIMEOUT", "CONNECTTIMEOUT"},
507 : {"GDAL_HTTP_TIMEOUT", "TIMEOUT"},
508 : {"GDAL_HTTP_LOW_SPEED_TIME", "LOW_SPEED_TIME"},
509 : {"GDAL_HTTP_LOW_SPEED_LIMIT", "LOW_SPEED_LIMIT"},
510 : {"GDAL_HTTP_USERPWD", "USERPWD"},
511 : {"GDAL_HTTP_PROXY", "PROXY"},
512 : {"GDAL_HTTPS_PROXY", "HTTPS_PROXY"},
513 : {"GDAL_HTTP_PROXYUSERPWD", "PROXYUSERPWD"},
514 : {"GDAL_PROXY_AUTH", "PROXYAUTH"},
515 : {"GDAL_HTTP_NETRC", "NETRC"},
516 : {"GDAL_HTTP_NETRC_FILE", "NETRC_FILE"},
517 : {"GDAL_HTTP_MAX_RETRY", "MAX_RETRY"},
518 : {"GDAL_HTTP_RETRY_DELAY", "RETRY_DELAY"},
519 : {"GDAL_HTTP_RETRY_CODES", "RETRY_CODES"},
520 : {"GDAL_CURL_CA_BUNDLE", "CAINFO"},
521 : {"CURL_CA_BUNDLE", "CAINFO"},
522 : {"SSL_CERT_FILE", "CAINFO"},
523 : {"GDAL_HTTP_CAPATH", "CAPATH"},
524 : {"GDAL_HTTP_SSL_VERIFYSTATUS", "SSL_VERIFYSTATUS"},
525 : {"GDAL_HTTP_USE_CAPI_STORE", "USE_CAPI_STORE"},
526 : {"GDAL_HTTP_HEADERS", "HEADERS"},
527 : {"GDAL_HTTP_HEADER_FILE", "HEADER_FILE"},
528 : {"GDAL_HTTP_AUTH", "HTTPAUTH"},
529 : {"GDAL_GSSAPI_DELEGATION", "GSSAPI_DELEGATION"},
530 : {"GDAL_HTTP_BEARER", "HTTP_BEARER"},
531 : {"GDAL_HTTP_COOKIE", "COOKIE"},
532 : {"GDAL_HTTP_COOKIEFILE", "COOKIEFILE"},
533 : {"GDAL_HTTP_COOKIEJAR", "COOKIEJAR"},
534 : {"GDAL_HTTP_MAX_RETRY", "MAX_RETRY"},
535 : {"GDAL_HTTP_RETRY_DELAY", "RETRY_DELAY"},
536 : {"GDAL_HTTP_TCP_KEEPALIVE", "TCP_KEEPALIVE"},
537 : {"GDAL_HTTP_TCP_KEEPIDLE", "TCP_KEEPIDLE"},
538 : {"GDAL_HTTP_TCP_KEEPINTVL", "TCP_KEEPINTVL"},
539 : {"GDAL_HTTP_PATH_VERBATIM", "PATH_VERBATIM"},
540 : };
541 :
542 1588 : char **CPLHTTPGetOptionsFromEnv(const char *pszFilename)
543 : {
544 3176 : CPLStringList aosOptions;
545 3176 : std::string osNonStreamingFilename;
546 1588 : if (pszFilename && STARTS_WITH(pszFilename, "/vsi"))
547 : {
548 : VSIFilesystemHandler *poFSHandler =
549 1586 : VSIFileManager::GetHandler(pszFilename);
550 : osNonStreamingFilename =
551 1586 : poFSHandler->GetNonStreamingFilename(pszFilename);
552 1586 : if (osNonStreamingFilename == pszFilename)
553 : {
554 1521 : osNonStreamingFilename.clear();
555 : }
556 : else
557 : {
558 : // CPLDebug("HTTP", "Non-streaming filename for %s: %s", pszFilename, osNonStreamingFilename.c_str());
559 : }
560 : }
561 57168 : for (const auto &sTuple : asAssocEnvVarOptionName)
562 : {
563 55580 : const char *pszVal = nullptr;
564 55580 : if (pszFilename)
565 : {
566 55580 : pszVal = VSIGetPathSpecificOption(pszFilename, sTuple.pszEnvVar,
567 : nullptr);
568 55580 : if (!pszVal && !osNonStreamingFilename.empty())
569 : {
570 2263 : pszVal = VSIGetPathSpecificOption(
571 2263 : osNonStreamingFilename.c_str(), sTuple.pszEnvVar, nullptr);
572 : }
573 : }
574 55580 : if (!pszVal)
575 : {
576 55420 : pszVal = CPLGetConfigOption(sTuple.pszEnvVar, nullptr);
577 : }
578 55580 : if (pszVal)
579 : {
580 160 : aosOptions.AddNameValue(sTuple.pszOptionName, pszVal);
581 : }
582 : }
583 3176 : return aosOptions.StealList();
584 : }
585 :
586 : /************************************************************************/
587 : /* CPLHTTPGetNewRetryDelay() */
588 : /************************************************************************/
589 :
590 : /** Return the new retry delay.
591 : *
592 : * This takes into account the HTTP response code, the previous delay, the
593 : * HTTP payload error message, the Curl error message and a potential list of
594 : * retriable HTTP codes.
595 : *
596 : * @param response_code HTTP response code (e.g. 400)
597 : * @param dfOldDelay Previous delay (nominally in second)
598 : * @param pszErrBuf HTTP response body of the failed request (may be NULL)
599 : * @param pszCurlError Curl error as returned by CURLOPT_ERRORBUFFER (may be NULL)
600 : * @param pszRetriableCodes nullptr to limit to the default hard-coded scenarios,
601 : * "ALL" to ask to retry for all non-200 codes, or a comma-separated list of
602 : * HTTP codes (e.g. "400,500") that are accepted for retry.
603 : * @return the new delay, or 0 if no retry should be attempted.
604 : */
605 231 : static double CPLHTTPGetNewRetryDelay(int response_code, double dfOldDelay,
606 : const char *pszErrBuf,
607 : const char *pszCurlError,
608 : const char *pszRetriableCodes)
609 : {
610 231 : bool bRetry = false;
611 231 : if (pszRetriableCodes && pszRetriableCodes[0])
612 : {
613 5 : bRetry = EQUAL(pszRetriableCodes, "ALL") ||
614 2 : strstr(pszRetriableCodes, CPLSPrintf("%d", response_code));
615 : }
616 228 : else if (response_code == 429 || response_code == 500 ||
617 218 : (response_code >= 502 && response_code <= 504) ||
618 : // S3 sends some client timeout errors as 400 Client Error
619 6 : (response_code == 400 && pszErrBuf &&
620 206 : strstr(pszErrBuf, "RequestTimeout")) ||
621 206 : (pszCurlError &&
622 206 : (strstr(pszCurlError, "Connection timed out") ||
623 206 : strstr(pszCurlError, "Operation timed out") ||
624 206 : strstr(pszCurlError, "Connection reset by peer") ||
625 206 : strstr(pszCurlError, "Connection was reset") ||
626 202 : strstr(pszCurlError, "SSL connection timeout"))))
627 : {
628 26 : bRetry = true;
629 : }
630 231 : if (bRetry)
631 : {
632 : // 'Operation timed out': seen during some long running operation 'hang'
633 : // no error but no response from server and we are in the cURL loop
634 : // infinitely.
635 :
636 : // 'Connection was reset': was found with Azure: server resets
637 : // connection during TLS handshake (10054 error code). It seems like
638 : // the server process crashed or something forced TCP reset;
639 : // the request succeeds on retry.
640 :
641 : // Use an exponential backoff factor of 2 plus some random jitter
642 : // We don't care about cryptographic quality randomness, hence:
643 : #ifndef __COVERITY__
644 28 : return dfOldDelay * (2 + rand() * 0.5 / RAND_MAX);
645 : #else
646 : return dfOldDelay * 2;
647 : #endif
648 : }
649 : else
650 : {
651 203 : return 0;
652 : }
653 : }
654 :
655 : /*! @cond Doxygen_Suppress */
656 :
657 : /************************************************************************/
658 : /* CPLHTTPRetryParameters() */
659 : /************************************************************************/
660 :
661 : /** Constructs a CPLHTTPRetryParameters instance from configuration
662 : * options or path-specific options.
663 : *
664 : * @param aosHTTPOptions HTTP options returned by CPLHTTPGetOptionsFromEnv()
665 : */
666 1299 : CPLHTTPRetryParameters::CPLHTTPRetryParameters(
667 1299 : const CPLStringList &aosHTTPOptions)
668 1299 : : nMaxRetry(atoi(aosHTTPOptions.FetchNameValueDef(
669 : "MAX_RETRY", CPLSPrintf("%d", CPL_HTTP_MAX_RETRY)))),
670 1299 : dfInitialDelay(CPLAtof(aosHTTPOptions.FetchNameValueDef(
671 : "RETRY_DELAY", CPLSPrintf("%f", CPL_HTTP_RETRY_DELAY)))),
672 2598 : osRetryCodes(aosHTTPOptions.FetchNameValueDef("RETRY_CODES", ""))
673 : {
674 1299 : }
675 :
676 : /************************************************************************/
677 : /* CPLHTTPRetryContext() */
678 : /************************************************************************/
679 :
680 : /** Constructor */
681 1090 : CPLHTTPRetryContext::CPLHTTPRetryContext(const CPLHTTPRetryParameters &oParams)
682 1090 : : m_oParameters(oParams), m_dfNextDelay(oParams.dfInitialDelay)
683 : {
684 1090 : }
685 :
686 : /************************************************************************/
687 : /* CPLHTTPRetryContext::CanRetry() */
688 : /************************************************************************/
689 :
690 : /** Returns whether we can attempt a new retry, based on the retry counter,
691 : * and increment that counter.
692 : */
693 0 : bool CPLHTTPRetryContext::CanRetry()
694 : {
695 0 : if (m_nRetryCount >= m_oParameters.nMaxRetry)
696 0 : return false;
697 0 : m_nRetryCount++;
698 0 : return true;
699 : }
700 :
701 : /** Returns whether we can attempt a new retry, based on the retry counter,
702 : * the response code, payload and curl error buffers.
703 : *
704 : * If successful, the retry counter is incremented, and GetCurrentDelay()
705 : * returns the delay to apply with CPLSleep().
706 : */
707 245 : bool CPLHTTPRetryContext::CanRetry(int response_code, const char *pszErrBuf,
708 : const char *pszCurlError)
709 : {
710 245 : if (m_nRetryCount >= m_oParameters.nMaxRetry)
711 224 : return false;
712 21 : m_dfCurDelay = m_dfNextDelay;
713 21 : m_dfNextDelay = CPLHTTPGetNewRetryDelay(response_code, m_dfNextDelay,
714 : pszErrBuf, pszCurlError,
715 : m_oParameters.osRetryCodes.c_str());
716 21 : if (m_dfNextDelay == 0.0)
717 2 : return false;
718 19 : m_nRetryCount++;
719 19 : return true;
720 : }
721 :
722 : /************************************************************************/
723 : /* CPLHTTPRetryContext::GetCurrentDelay() */
724 : /************************************************************************/
725 :
726 : /** Returns the delay to apply. Only valid after a successful call to CanRetry() */
727 38 : double CPLHTTPRetryContext::GetCurrentDelay() const
728 : {
729 38 : if (m_nRetryCount == 0)
730 0 : CPLDebug("CPL",
731 : "GetCurrentDelay() should only be called after CanRetry()");
732 38 : return m_dfCurDelay;
733 : }
734 :
735 : /*! @endcond Doxygen_Suppress */
736 :
737 : #ifdef HAVE_CURL
738 :
739 : /************************************************************************/
740 : /* CPLHTTPEmitFetchDebug() */
741 : /************************************************************************/
742 :
743 1175 : static void CPLHTTPEmitFetchDebug(const char *pszURL,
744 : const char *pszExtraDebug = "")
745 : {
746 1175 : const char *pszArobase = strchr(pszURL, '@');
747 1175 : const char *pszSlash = strchr(pszURL, '/');
748 1175 : const char *pszColon = (pszSlash) ? strchr(pszSlash, ':') : nullptr;
749 1175 : if (pszArobase != nullptr && pszColon != nullptr &&
750 0 : pszArobase - pszColon > 0)
751 : {
752 : /* http://user:password@www.example.com */
753 0 : char *pszSanitizedURL = CPLStrdup(pszURL);
754 0 : pszSanitizedURL[pszColon - pszURL] = 0;
755 0 : CPLDebug("HTTP", "Fetch(%s:#password#%s%s)", pszSanitizedURL,
756 : pszArobase, pszExtraDebug);
757 0 : CPLFree(pszSanitizedURL);
758 : }
759 : else
760 : {
761 1175 : CPLDebug("HTTP", "Fetch(%s%s)", pszURL, pszExtraDebug);
762 : }
763 1175 : }
764 :
765 : #endif
766 :
767 : #ifdef HAVE_CURL
768 :
769 : /************************************************************************/
770 : /* class CPLHTTPPostFields */
771 : /************************************************************************/
772 :
773 : class CPLHTTPPostFields
774 : {
775 : public:
776 1175 : CPLHTTPPostFields() = default;
777 : CPLHTTPPostFields &operator=(const CPLHTTPPostFields &) = delete;
778 : CPLHTTPPostFields(const CPLHTTPPostFields &) = delete;
779 :
780 1175 : CPLErr Fill(CURL *http_handle, CSLConstList papszOptions)
781 : {
782 : // Fill POST form if present
783 : const char *pszFormFilePath =
784 1175 : CSLFetchNameValue(papszOptions, "FORM_FILE_PATH");
785 : const char *pszParametersCount =
786 1175 : CSLFetchNameValue(papszOptions, "FORM_ITEM_COUNT");
787 :
788 1175 : if (pszFormFilePath != nullptr || pszParametersCount != nullptr)
789 : {
790 2 : mime = curl_mime_init(http_handle);
791 2 : curl_mimepart *mimepart = curl_mime_addpart(mime);
792 2 : if (pszFormFilePath != nullptr)
793 : {
794 : const char *pszFormFileName =
795 1 : CSLFetchNameValue(papszOptions, "FORM_FILE_NAME");
796 1 : const char *pszFilename = CPLGetFilename(pszFormFilePath);
797 1 : if (pszFormFileName == nullptr)
798 : {
799 1 : pszFormFileName = pszFilename;
800 : }
801 :
802 : VSIStatBufL sStat;
803 1 : if (VSIStatL(pszFormFilePath, &sStat) == 0)
804 : {
805 0 : VSILFILE *mime_fp = VSIFOpenL(pszFormFilePath, "rb");
806 0 : if (mime_fp != nullptr)
807 : {
808 0 : curl_mime_name(mimepart, pszFormFileName);
809 0 : CPL_IGNORE_RET_VAL(
810 0 : curl_mime_filename(mimepart, pszFilename));
811 0 : curl_mime_data_cb(
812 : mimepart, sStat.st_size, CPLHTTPReadFunction,
813 : CPLHTTPSeekFunction, CPLHTTPFreeFunction, mime_fp);
814 : }
815 : else
816 : {
817 : osErrMsg = CPLSPrintf("Failed to open file %s",
818 0 : pszFormFilePath);
819 1 : return CE_Failure;
820 : }
821 :
822 0 : CPLDebug("HTTP", "Send file: %s, COPYNAME: %s",
823 : pszFormFilePath, pszFormFileName);
824 : }
825 : else
826 : {
827 : osErrMsg =
828 1 : CPLSPrintf("File '%s' not found", pszFormFilePath);
829 1 : return CE_Failure;
830 : }
831 : }
832 :
833 1 : int nParametersCount = 0;
834 1 : if (pszParametersCount != nullptr)
835 : {
836 1 : nParametersCount = atoi(pszParametersCount);
837 : }
838 :
839 2 : for (int i = 0; i < nParametersCount; ++i)
840 : {
841 2 : const char *pszKey = CSLFetchNameValue(
842 : papszOptions, CPLSPrintf("FORM_KEY_%d", i));
843 2 : const char *pszValue = CSLFetchNameValue(
844 : papszOptions, CPLSPrintf("FORM_VALUE_%d", i));
845 :
846 2 : if (nullptr == pszKey)
847 : {
848 : osErrMsg = CPLSPrintf("Key #%d is not exists. Maybe wrong "
849 : "count of form items",
850 1 : i);
851 1 : return CE_Failure;
852 : }
853 :
854 1 : if (nullptr == pszValue)
855 : {
856 : osErrMsg = CPLSPrintf("Value #%d is not exists. Maybe "
857 : "wrong count of form items",
858 0 : i);
859 0 : return CE_Failure;
860 : }
861 :
862 1 : mimepart = curl_mime_addpart(mime);
863 1 : curl_mime_name(mimepart, pszKey);
864 1 : CPL_IGNORE_RET_VAL(
865 1 : curl_mime_data(mimepart, pszValue, CURL_ZERO_TERMINATED));
866 :
867 1 : CPLDebug("HTTP", "COPYNAME: %s, COPYCONTENTS: %s", pszKey,
868 : pszValue);
869 : }
870 :
871 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_MIMEPOST, mime);
872 : }
873 1173 : return CE_None;
874 : }
875 :
876 1175 : ~CPLHTTPPostFields()
877 1175 : {
878 1175 : if (mime != nullptr)
879 : {
880 2 : curl_mime_free(mime);
881 : }
882 1175 : }
883 :
884 2 : std::string GetErrorMessage() const
885 : {
886 2 : return osErrMsg;
887 : }
888 :
889 : private:
890 : curl_mime *mime = nullptr;
891 : std::string osErrMsg{};
892 : };
893 :
894 : /************************************************************************/
895 : /* CPLHTTPFetchCleanup() */
896 : /************************************************************************/
897 :
898 1175 : static void CPLHTTPFetchCleanup(CURL *http_handle, struct curl_slist *headers,
899 : const char *pszPersistent,
900 : CSLConstList papszOptions)
901 : {
902 1175 : if (CSLFetchNameValue(papszOptions, "POSTFIELDS"))
903 159 : unchecked_curl_easy_setopt(http_handle, CURLOPT_POST, 0);
904 1175 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTPHEADER, nullptr);
905 :
906 1175 : if (!pszPersistent)
907 673 : curl_easy_cleanup(http_handle);
908 :
909 1175 : curl_slist_free_all(headers);
910 1175 : }
911 : #endif // HAVE_CURL
912 :
913 : struct CPLHTTPFetchContext
914 : {
915 : std::vector<std::pair<CPLHTTPFetchCallbackFunc, void *>> stack{};
916 : };
917 :
918 : /************************************************************************/
919 : /* GetHTTPFetchContext() */
920 : /************************************************************************/
921 :
922 1368 : static CPLHTTPFetchContext *GetHTTPFetchContext(bool bAlloc)
923 : {
924 1368 : int bError = FALSE;
925 : CPLHTTPFetchContext *psCtx = static_cast<CPLHTTPFetchContext *>(
926 1368 : CPLGetTLSEx(CTLS_HTTPFETCHCALLBACK, &bError));
927 1368 : if (bError)
928 0 : return nullptr;
929 :
930 1368 : if (psCtx == nullptr && bAlloc)
931 : {
932 1 : const auto FreeFunc = [](void *pData)
933 1 : { delete static_cast<CPLHTTPFetchContext *>(pData); };
934 1 : psCtx = new CPLHTTPFetchContext();
935 1 : CPLSetTLSWithFreeFuncEx(CTLS_HTTPFETCHCALLBACK, psCtx, FreeFunc,
936 : &bError);
937 1 : if (bError)
938 : {
939 0 : delete psCtx;
940 0 : psCtx = nullptr;
941 : }
942 : }
943 1368 : return psCtx;
944 : }
945 :
946 : /************************************************************************/
947 : /* CPLHTTPSetFetchCallback() */
948 : /************************************************************************/
949 :
950 : static CPLHTTPFetchCallbackFunc gpsHTTPFetchCallbackFunc = nullptr;
951 : static void *gpHTTPFetchCallbackUserData = nullptr;
952 :
953 : /** Installs an alternate callback to the default implementation of
954 : * CPLHTTPFetchEx().
955 : *
956 : * This callback will be used by all threads, unless contextual callbacks are
957 : * installed with CPLHTTPPushFetchCallback().
958 : *
959 : * It is the responsibility of the caller to make sure this function is not
960 : * called concurrently, or during CPLHTTPFetchEx() execution.
961 : *
962 : * @param pFunc Callback function to be called with CPLHTTPFetchEx() is called
963 : * (or NULL to restore default handler)
964 : * @param pUserData Last argument to provide to the pFunc callback.
965 : *
966 : * @since GDAL 3.2
967 : */
968 2 : void CPLHTTPSetFetchCallback(CPLHTTPFetchCallbackFunc pFunc, void *pUserData)
969 : {
970 2 : gpsHTTPFetchCallbackFunc = pFunc;
971 2 : gpHTTPFetchCallbackUserData = pUserData;
972 2 : }
973 :
974 : /************************************************************************/
975 : /* CPLHTTPPushFetchCallback() */
976 : /************************************************************************/
977 :
978 : /** Installs an alternate callback to the default implementation of
979 : * CPLHTTPFetchEx().
980 : *
981 : * This callback will only be used in the thread where this function has been
982 : * called. It must be un-installed by CPLHTTPPopFetchCallback(), which must also
983 : * be called from the same thread.
984 : *
985 : * @param pFunc Callback function to be called with CPLHTTPFetchEx() is called.
986 : * @param pUserData Last argument to provide to the pFunc callback.
987 : * @return TRUE in case of success.
988 : *
989 : * @since GDAL 3.2
990 : */
991 1 : int CPLHTTPPushFetchCallback(CPLHTTPFetchCallbackFunc pFunc, void *pUserData)
992 : {
993 1 : auto psCtx = GetHTTPFetchContext(true);
994 1 : if (psCtx == nullptr)
995 0 : return false;
996 : psCtx->stack.emplace_back(
997 1 : std::pair<CPLHTTPFetchCallbackFunc, void *>(pFunc, pUserData));
998 1 : return true;
999 : }
1000 :
1001 : /************************************************************************/
1002 : /* CPLHTTPPopFetchCallback() */
1003 : /************************************************************************/
1004 :
1005 : /** Uninstalls a callback set by CPLHTTPPushFetchCallback().
1006 : *
1007 : * @see CPLHTTPPushFetchCallback()
1008 : * @return TRUE in case of success.
1009 : * @since GDAL 3.2
1010 : */
1011 2 : int CPLHTTPPopFetchCallback(void)
1012 : {
1013 2 : auto psCtx = GetHTTPFetchContext(false);
1014 2 : if (psCtx == nullptr || psCtx->stack.empty())
1015 : {
1016 1 : CPLError(
1017 : CE_Failure, CPLE_AppDefined,
1018 : "CPLHTTPPushFetchCallback / CPLHTTPPopFetchCallback not balanced");
1019 1 : return false;
1020 : }
1021 : else
1022 : {
1023 1 : psCtx->stack.pop_back();
1024 1 : return true;
1025 : }
1026 : }
1027 :
1028 : /************************************************************************/
1029 : /* CPLHTTPFetch() */
1030 : /************************************************************************/
1031 :
1032 : // NOTE: when adding an option below, add it in asAssocEnvVarOptionName[]
1033 :
1034 : // clang-format off
1035 : /**
1036 : * \brief Fetch a document from an url and return in a string.
1037 : *
1038 : * Different options may be passed through the papszOptions function parameter,
1039 : * or for most of them through a configuration option:
1040 : * <ul>
1041 : * <li>CONNECTTIMEOUT=val, where
1042 : * val is in seconds (possibly with decimals). This is the maximum delay for the
1043 : * connection to be established before being aborted (GDAL >= 2.2).
1044 : * Corresponding configuration option: GDAL_HTTP_CONNECTTIMEOUT.
1045 : * </li>
1046 : * <li>TIMEOUT=val, where val is in seconds. This is the maximum delay for the
1047 : * whole request to complete before being aborted.
1048 : * Corresponding configuration option: GDAL_HTTP_TIMEOUT.
1049 : * </li>
1050 : * <li>LOW_SPEED_TIME=val,
1051 : * where val is in seconds. This is the maximum time where the transfer speed
1052 : * should be below the LOW_SPEED_LIMIT (if not specified 1b/s), before the
1053 : * transfer to be considered too slow and aborted. (GDAL >= 2.1).
1054 : * Corresponding configuration option: GDAL_HTTP_LOW_SPEED_TIME.
1055 : * </li>
1056 : * <li>LOW_SPEED_LIMIT=val, where val is in bytes/second. See LOW_SPEED_TIME.
1057 : * Has only effect if LOW_SPEED_TIME is specified too. (GDAL >= 2.1).
1058 : * Corresponding configuration option: GDAL_HTTP_LOW_SPEED_LIMIT.
1059 : * </li>
1060 : * <li>HEADERS=val, where val is an extra header to use when getting a web page.
1061 : * For example "Accept: application/x-ogcwkt"
1062 : * Corresponding configuration option: GDAL_HTTP_HEADERS.
1063 : * Starting with GDAL 3.6, the GDAL_HTTP_HEADERS configuration option can also
1064 : * be used to specify a comma separated list of key: value pairs. This is an
1065 : * alternative to the GDAL_HTTP_HEADER_FILE mechanism. If a comma or a
1066 : * double-quote character is needed in the value, then the key: value pair must
1067 : * be enclosed in double-quote characters. In that situation, backslash and
1068 : * double quote character must be backslash-escaped. e.g GDAL_HTTP_HEADERS=Foo:
1069 : * Bar,"Baz: escaped backslash \\, escaped double-quote \", end of
1070 : * value",Another: Header
1071 : * </li>
1072 : * <li>HEADER_FILE=filename: filename of a text file with "key: value" headers.
1073 : * The content of the file is not cached, and thus it is read again before
1074 : * issuing each HTTP request. (GDAL >= 2.2)
1075 : * Corresponding configuration option: GDAL_HTTP_HEADER_FILE.
1076 : * </li>
1077 : * <li>HTTPAUTH=[BASIC/NTLM/NEGOTIATE/ANY/ANYSAFE/BEARER] to specify an
1078 : * authentication scheme to use.
1079 : * Corresponding configuration option: GDAL_HTTP_AUTH.
1080 : * </li>
1081 : * <li>USERPWD=userid:password to specify a user and password for
1082 : * authentication.
1083 : * Corresponding configuration option: GDAL_HTTP_USERPWD.
1084 : * </li>
1085 : * <li>GSSAPI_DELEGATION=[NONE/POLICY/ALWAYS] set allowed
1086 : * GSS-API delegation. Relevant only with HTTPAUTH=NEGOTIATE (GDAL >= 3.3).
1087 : * Corresponding configuration option: GDAL_GSSAPI_DELEGATION (note: no "HTTP_" in the name)
1088 : * </li>
1089 : * <li>HTTP_BEARER=val set OAuth 2.0 Bearer Access Token.
1090 : * Relevant only with HTTPAUTH=BEARER (GDAL >= 3.9).
1091 : * Corresponding configuration option: GDAL_HTTP_BEARER
1092 : * </li>
1093 : * <li>POSTFIELDS=val, where val is a nul-terminated string to be passed to the
1094 : * server with a POST request.
1095 : * No Corresponding configuration option.
1096 : * </li>
1097 : * <li>PROXY=val, to make requests go through a
1098 : * proxy server, where val is of the form proxy.server.com:port_number. This
1099 : * option affects both HTTP and HTTPS URLs.
1100 : * Corresponding configuration option: GDAL_HTTP_PROXY.
1101 : * </li>
1102 : * <li>HTTPS_PROXY=val (GDAL
1103 : * >= 2.4), the same meaning as PROXY, but this option is taken into account
1104 : * only for HTTPS URLs.
1105 : * Corresponding configuration option: GDAL_HTTPS_PROXY.
1106 : * </li>
1107 : * <li>PROXYUSERPWD=val, where val is of the form username:password.
1108 : * Corresponding configuration option: GDAL_HTTP_PROXYUSERPWD
1109 : * </li>
1110 : * <li>PROXYAUTH=[BASIC/NTLM/DIGEST/NEGOTIATE/ANY/ANYSAFE] to
1111 : * specify an proxy authentication scheme to use..
1112 : * Corresponding configuration option: GDAL_PROXYAUTH (note: no "HTTP_" in the name)
1113 : * </li>
1114 : * <li>NETRC=[YES/NO] to
1115 : * enable or disable use of $HOME/.netrc (or NETRC_FILE), default YES.
1116 : * Corresponding configuration option: GDAL_HTTP_NETRC.
1117 : * </li>
1118 : * <li>NETRC_FILE=file name to read .netrc info from (GDAL >= 3.7).
1119 : * Corresponding configuration option: GDAL_HTTP_NETRC_FILE.
1120 : * </li>
1121 : * <li>CUSTOMREQUEST=val, where val is GET, PUT, POST, DELETE, etc...
1122 : * No corresponding configuration option.
1123 : * </li>
1124 : * <li>FORM_FILE_NAME=val, where val is upload file name. If this
1125 : * option and FORM_FILE_PATH present, request type will set to POST.
1126 : * No corresponding configuration option.
1127 : * </li>
1128 : * <li>FORM_FILE_PATH=val, where val is upload file path.
1129 : * No corresponding configuration option.
1130 : * </li>
1131 : * <li>FORM_KEY_0=val...FORM_KEY_N, where val is name of form item.
1132 : * No corresponding configuration option.
1133 : * </li>
1134 : * <li>FORM_VALUE_0=val...FORM_VALUE_N, where val is value of the form
1135 : * item.
1136 : * No corresponding configuration option.
1137 : * </li>
1138 : * <li>FORM_ITEM_COUNT=val, where val is count of form items.
1139 : * No corresponding configuration option.
1140 : * </li>
1141 : * <li>COOKIE=val, where val is formatted as COOKIE1=VALUE1; COOKIE2=VALUE2;...
1142 : * Corresponding configuration option: GDAL_HTTP_COOKIE.</li>
1143 : * <li>COOKIEFILE=val, where val is file name to read cookies from
1144 : * (GDAL >= 2.4).
1145 : * Corresponding configuration option: GDAL_HTTP_COOKIEFILE.</li>
1146 : * <li>COOKIEJAR=val, where val is file name to store cookies to (GDAL >= 2.4).
1147 : * Corresponding configuration option: GDAL_HTTP_COOKIEJAR.</li>
1148 : * <li>MAX_RETRY=val, where val is the maximum number of
1149 : * retry attempts, when a retry is allowed (cf RETRY_CODES option).
1150 : * Default is 0, meaning no retry.
1151 : * Corresponding configuration option: GDAL_HTTP_MAX_RETRY.
1152 : * </li>
1153 : * <li>RETRY_DELAY=val, where val is the number of seconds
1154 : * between retry attempts. Default is 30.
1155 : * Corresponding configuration option: GDAL_HTTP_RETRY_DELAY.
1156 : * <li>RETRY_CODES=val, where val is "ALL" or a comma-separated list of HTTP
1157 : * codes that are considered for retry. By default, 429, 500, 502, 503 or 504
1158 : * HTTP errors are considered, as well as other situations with a particular
1159 : * HTTP or Curl error message. (GDAL >= 3.10).
1160 : * Corresponding configuration option: GDAL_HTTP_RETRY_CODES.
1161 : * </li>
1162 : * <li>MAX_FILE_SIZE=val, where val is a number of bytes (GDAL >= 2.2)
1163 : * No corresponding configuration option.
1164 : * </li>
1165 : * <li>CAINFO=/path/to/bundle.crt. This is path to Certificate Authority (CA)
1166 : * bundle file. By default, it will be looked for in a system location. If
1167 : * the CAINFO option is not defined, GDAL will also look in the
1168 : * CURL_CA_BUNDLE and SSL_CERT_FILE environment variables respectively
1169 : * and use the first one found as the CAINFO value (GDAL >= 2.1.3). The
1170 : * GDAL_CURL_CA_BUNDLE environment variable may also be used to set the
1171 : * CAINFO value in GDAL >= 3.2.</li>
1172 : * <li>HTTP_VERSION=1.0/1.1/2/2TLS (GDAL >= 2.3)/2PRIOR_KNOWLEDGE (GDAL >= 3.10).
1173 : * Specify HTTP version to use.
1174 : * Will default to 1.1 generally (except on some controlled environments,
1175 : * like Google Compute Engine VMs, where 2TLS will be the default).
1176 : * Support for HTTP/2 requires curl 7.33 or later, built against nghttp2.
1177 : * "2TLS" means that HTTP/2 will be attempted for HTTPS connections only.
1178 : * Whereas "2" means that HTTP/2 will be attempted for HTTP or HTTPS.
1179 : * "2PRIOR_KNOWLEDGE" means that the server will be assumed to support
1180 : * HTTP/2.
1181 : * Corresponding configuration option: GDAL_HTTP_VERSION.
1182 : * </li>
1183 : * <li>SSL_VERIFYSTATUS=YES/NO (GDAL >= 2.3, and curl >= 7.41): determines
1184 : * whether the status of the server cert using the "Certificate Status Request"
1185 : * TLS extension (aka. OCSP stapling) should be checked. If this option is
1186 : * enabled but the server does not support the TLS extension, the verification
1187 : * will fail. Default to NO.
1188 : * Corresponding configuration option: GDAL_HTTP_SSL_VERIFYSTATUS.
1189 : * </li>
1190 : * <li>USE_CAPI_STORE=YES/NO (GDAL >= 2.3,
1191 : * Windows only): whether CA certificates from the Windows certificate store.
1192 : * Defaults to NO.
1193 : * Corresponding configuration option: GDAL_HTTP_USE_CAPI_STORE.
1194 : * </li>
1195 : * <li>TCP_KEEPALIVE=YES/NO (GDAL >= 3.6): whether to
1196 : * enable TCP keep-alive. Defaults to NO.
1197 : * Corresponding configuration option: GDAL_HTTP_TCP_KEEPALIVE.
1198 : * </li>
1199 : * <li>TCP_KEEPIDLE=integer, in
1200 : * seconds (GDAL >= 3.6): keep-alive idle time. Defaults to 60. Only taken into
1201 : * account if TCP_KEEPALIVE=YES.
1202 : * Corresponding configuration option: GDAL_HTTP_TCP_KEEPIDLE.
1203 : * </li>
1204 : * <li>TCP_KEEPINTVL=integer, in seconds
1205 : * (GDAL >= 3.6): interval time between keep-alive probes. Defaults to 60. Only
1206 : * taken into account if TCP_KEEPALIVE=YES.
1207 : * Corresponding configuration option: GDAL_HTTP_TCP_KEEPINTVL.
1208 : * </li>
1209 : * <li>USERAGENT=string: value of User-Agent header. Starting with GDAL 3.7,
1210 : * GDAL core sets it by default (during driver initialization) to GDAL/x.y.z
1211 : * where x.y.z is the GDAL version number. Applications may override it with the
1212 : * CPLHTTPSetDefaultUserAgent() function.
1213 : * Corresponding configuration option: GDAL_HTTP_USERAGENT.
1214 : * </li>
1215 : * <li>SSLCERT=filename (GDAL >= 3.7): Filename of the the SSL client certificate.
1216 : * Cf https://curl.se/libcurl/c/CURLOPT_SSLCERT.html.
1217 : * Corresponding configuration option: GDAL_HTTP_SSLCERT.
1218 : * </li>
1219 : * <li>SSLCERTTYPE=string (GDAL >= 3.7): Format of the SSL certificate: "PEM"
1220 : * or "DER". Cf https://curl.se/libcurl/c/CURLOPT_SSLCERTTYPE.html.
1221 : * Corresponding configuration option: GDAL_HTTP_SSLCERTTYPE.
1222 : * </li>
1223 : * <li>SSLKEY=filename (GDAL >= 3.7): Private key file for TLS and SSL client
1224 : * certificate. Cf https://curl.se/libcurl/c/CURLOPT_SSLKEY.html.
1225 : * Corresponding configuration option: GDAL_HTTP_SSLKEY.
1226 : * </li>
1227 : * <li>KEYPASSWD=string (GDAL >= 3.7): Passphrase to private key.
1228 : * Cf https://curl.se/libcurl/c/CURLOPT_KEYPASSWD.html.
1229 : * Corresponding configuration option: GDAL_HTTP_KEYPASSWD.
1230 : * <li>PATH_VERBATIM=[YES/NO] Can be set to YES so that sequences of "/../" or "/./"
1231 : * that may exist in the URL's path part are kept as it. Otherwise, by default,
1232 : * they are squashed, according to RFC 3986 section 5.2.4
1233 : * Cf https://curl.se/libcurl/c/CURLOPT_PATH_AS_IS.html
1234 : * Corresponding configuration option: GDAL_HTTP_PATH_VERBATIM.
1235 : * </li>
1236 : * </ul>
1237 : *
1238 : * If an option is specified through papszOptions and as a configuration option,
1239 : * the former takes precedence over the later.
1240 : *
1241 : * Starting with GDAL 3.7, the above configuration options can also be specified
1242 : * as path-specific options with VSISetPathSpecificOption().
1243 : *
1244 : * @param pszURL valid URL recognized by underlying download library (libcurl)
1245 : * @param papszOptions option list as a NULL-terminated array of strings. May be
1246 : * NULL.
1247 : *
1248 : * @return a CPLHTTPResult* structure that must be freed by
1249 : * CPLHTTPDestroyResult(), or NULL if libcurl support is disabled
1250 : */
1251 : // clang-format on
1252 2131 : CPLHTTPResult *CPLHTTPFetch(const char *pszURL, CSLConstList papszOptions)
1253 : {
1254 2131 : return CPLHTTPFetchEx(pszURL, papszOptions, nullptr, nullptr, nullptr,
1255 2131 : nullptr);
1256 : }
1257 :
1258 : /**
1259 : * Fetch a document from an url and return in a string.
1260 : * @param pszURL Url to fetch document from web.
1261 : * @param papszOptions Option list as a NULL-terminated array of strings.
1262 : * Available keys see in CPLHTTPFetch.
1263 : * @param pfnProgress Callback for reporting algorithm progress matching the
1264 : * GDALProgressFunc() semantics. May be NULL.
1265 : * @param pProgressArg Callback argument passed to pfnProgress.
1266 : * @param pfnWrite Write function pointer matching the CPLHTTPWriteFunc()
1267 : * semantics. May be NULL.
1268 : * @param pWriteArg Argument which will pass to a write function.
1269 : * @return A CPLHTTPResult* structure that must be freed by
1270 : * CPLHTTPDestroyResult(), or NULL if libcurl support is disabled.
1271 : */
1272 2161 : CPLHTTPResult *CPLHTTPFetchEx(const char *pszURL, CSLConstList papszOptions,
1273 : GDALProgressFunc pfnProgress, void *pProgressArg,
1274 : CPLHTTPFetchWriteFunc pfnWrite, void *pWriteArg)
1275 :
1276 : {
1277 2982 : if (STARTS_WITH(pszURL, "/vsimem/") &&
1278 : // Disabled by default for potential security issues.
1279 821 : CPLTestBool(CPLGetConfigOption("CPL_CURL_ENABLE_VSIMEM", "FALSE")))
1280 : {
1281 796 : CPLString osURL(pszURL);
1282 : const char *pszCustomRequest =
1283 796 : CSLFetchNameValue(papszOptions, "CUSTOMREQUEST");
1284 796 : if (pszCustomRequest != nullptr)
1285 : {
1286 5 : osURL += "&CUSTOMREQUEST=";
1287 5 : osURL += pszCustomRequest;
1288 : }
1289 796 : const char *pszUserPwd = CSLFetchNameValue(papszOptions, "USERPWD");
1290 796 : if (pszUserPwd != nullptr)
1291 : {
1292 1 : osURL += "&USERPWD=";
1293 1 : osURL += pszUserPwd;
1294 : }
1295 796 : const char *pszPost = CSLFetchNameValue(papszOptions, "POSTFIELDS");
1296 796 : if (pszPost != nullptr) // Hack: We append post content to filename.
1297 : {
1298 281 : osURL += "&POSTFIELDS=";
1299 281 : osURL += pszPost;
1300 : }
1301 796 : const char *pszHeaders = CSLFetchNameValue(papszOptions, "HEADERS");
1302 928 : if (pszHeaders != nullptr &&
1303 132 : CPLTestBool(
1304 : CPLGetConfigOption("CPL_CURL_VSIMEM_PRINT_HEADERS", "FALSE")))
1305 : {
1306 5 : osURL += "&HEADERS=";
1307 5 : osURL += pszHeaders;
1308 : }
1309 796 : vsi_l_offset nLength = 0;
1310 : CPLHTTPResult *psResult =
1311 796 : static_cast<CPLHTTPResult *>(CPLCalloc(1, sizeof(CPLHTTPResult)));
1312 796 : GByte *pabyData = VSIGetMemFileBuffer(osURL, &nLength, FALSE);
1313 796 : if (pabyData == nullptr)
1314 : {
1315 132 : CPLDebug("HTTP", "Cannot find %s", osURL.c_str());
1316 132 : psResult->nStatus = 1;
1317 132 : psResult->pszErrBuf =
1318 132 : CPLStrdup(CPLSPrintf("HTTP error code : %d", 404));
1319 132 : CPLError(CE_Failure, CPLE_AppDefined, "%s", psResult->pszErrBuf);
1320 : }
1321 664 : else if (nLength != 0)
1322 : {
1323 621 : psResult->nDataLen = static_cast<int>(nLength);
1324 621 : psResult->pabyData = static_cast<GByte *>(
1325 621 : CPLMalloc(static_cast<size_t>(nLength) + 1));
1326 621 : memcpy(psResult->pabyData, pabyData, static_cast<size_t>(nLength));
1327 621 : psResult->pabyData[static_cast<size_t>(nLength)] = 0;
1328 : }
1329 :
1330 796 : if (psResult->pabyData != nullptr &&
1331 621 : STARTS_WITH(reinterpret_cast<char *>(psResult->pabyData),
1332 : "Content-Type: "))
1333 : {
1334 7 : const char *pszContentType =
1335 7 : reinterpret_cast<char *>(psResult->pabyData) +
1336 : strlen("Content-type: ");
1337 7 : const char *pszEOL = strchr(pszContentType, '\r');
1338 7 : if (pszEOL)
1339 7 : pszEOL = strchr(pszContentType, '\n');
1340 7 : if (pszEOL)
1341 : {
1342 7 : size_t nContentLength = pszEOL - pszContentType;
1343 7 : psResult->pszContentType =
1344 7 : static_cast<char *>(CPLMalloc(nContentLength + 1));
1345 7 : memcpy(psResult->pszContentType, pszContentType,
1346 : nContentLength);
1347 7 : psResult->pszContentType[nContentLength] = 0;
1348 : }
1349 : }
1350 :
1351 796 : return psResult;
1352 : }
1353 :
1354 : // Try to user alternate network layer if set.
1355 1365 : auto pCtx = GetHTTPFetchContext(false);
1356 1365 : if (pCtx)
1357 : {
1358 3 : for (size_t i = pCtx->stack.size(); i > 0;)
1359 : {
1360 1 : --i;
1361 1 : const auto &cbk = pCtx->stack[i];
1362 1 : auto cbkFunc = cbk.first;
1363 1 : auto pUserData = cbk.second;
1364 1 : auto res = cbkFunc(pszURL, papszOptions, pfnProgress, pProgressArg,
1365 : pfnWrite, pWriteArg, pUserData);
1366 1 : if (res)
1367 : {
1368 1 : if (CSLFetchNameValue(papszOptions, "CLOSE_PERSISTENT"))
1369 : {
1370 0 : CPLHTTPDestroyResult(res);
1371 0 : res = nullptr;
1372 : }
1373 1 : return res;
1374 : }
1375 : }
1376 : }
1377 :
1378 1364 : if (gpsHTTPFetchCallbackFunc)
1379 : {
1380 1 : auto res = gpsHTTPFetchCallbackFunc(pszURL, papszOptions, pfnProgress,
1381 : pProgressArg, pfnWrite, pWriteArg,
1382 : gpHTTPFetchCallbackUserData);
1383 1 : if (res)
1384 : {
1385 1 : if (CSLFetchNameValue(papszOptions, "CLOSE_PERSISTENT"))
1386 : {
1387 0 : CPLHTTPDestroyResult(res);
1388 0 : res = nullptr;
1389 : }
1390 1 : return res;
1391 : }
1392 : }
1393 :
1394 : #ifndef HAVE_CURL
1395 : CPLError(CE_Failure, CPLE_NotSupported,
1396 : "GDAL/OGR not compiled with libcurl support, "
1397 : "remote requests not supported.");
1398 : return nullptr;
1399 : #else
1400 :
1401 : /* -------------------------------------------------------------------- */
1402 : /* Are we using a persistent named session? If so, search for */
1403 : /* or create it. */
1404 : /* */
1405 : /* Currently this code does not attempt to protect against */
1406 : /* multiple threads asking for the same named session. If that */
1407 : /* occurs it will be in use in multiple threads at once, which */
1408 : /* will lead to potential crashes in libcurl. */
1409 : /* -------------------------------------------------------------------- */
1410 1363 : CURL *http_handle = nullptr;
1411 :
1412 1363 : const char *pszPersistent = CSLFetchNameValue(papszOptions, "PERSISTENT");
1413 : const char *pszClosePersistent =
1414 1363 : CSLFetchNameValue(papszOptions, "CLOSE_PERSISTENT");
1415 1363 : if (pszPersistent)
1416 : {
1417 1004 : CPLString osSessionName = pszPersistent;
1418 502 : CPLMutexHolder oHolder(&hSessionMapMutex);
1419 :
1420 502 : if (poSessionMap == nullptr)
1421 136 : poSessionMap = new std::map<CPLString, CURL *>;
1422 502 : if (poSessionMap->count(osSessionName) == 0)
1423 : {
1424 158 : (*poSessionMap)[osSessionName] = curl_easy_init();
1425 158 : CPLDebug("HTTP", "Establish persistent session named '%s'.",
1426 : osSessionName.c_str());
1427 : }
1428 :
1429 502 : http_handle = (*poSessionMap)[osSessionName];
1430 : }
1431 : /* -------------------------------------------------------------------- */
1432 : /* Are we requested to close a persistent named session? */
1433 : /* -------------------------------------------------------------------- */
1434 861 : else if (pszClosePersistent)
1435 : {
1436 376 : CPLString osSessionName = pszClosePersistent;
1437 188 : CPLMutexHolder oHolder(&hSessionMapMutex);
1438 :
1439 188 : if (poSessionMap)
1440 : {
1441 : std::map<CPLString, CURL *>::iterator oIter =
1442 158 : poSessionMap->find(osSessionName);
1443 158 : if (oIter != poSessionMap->end())
1444 : {
1445 158 : curl_easy_cleanup(oIter->second);
1446 158 : poSessionMap->erase(oIter);
1447 158 : if (poSessionMap->empty())
1448 : {
1449 136 : delete poSessionMap;
1450 136 : poSessionMap = nullptr;
1451 : }
1452 158 : CPLDebug("HTTP", "Ended persistent session named '%s'.",
1453 : osSessionName.c_str());
1454 : }
1455 : else
1456 : {
1457 0 : CPLDebug("HTTP",
1458 : "Could not find persistent session named '%s'.",
1459 : osSessionName.c_str());
1460 : }
1461 : }
1462 :
1463 188 : return nullptr;
1464 : }
1465 : else
1466 673 : http_handle = curl_easy_init();
1467 :
1468 : /* -------------------------------------------------------------------- */
1469 : /* Setup the request. */
1470 : /* -------------------------------------------------------------------- */
1471 1175 : char szCurlErrBuf[CURL_ERROR_SIZE + 1] = {};
1472 :
1473 1175 : CPLHTTPEmitFetchDebug(pszURL);
1474 :
1475 : CPLHTTPResult *psResult =
1476 1175 : static_cast<CPLHTTPResult *>(CPLCalloc(1, sizeof(CPLHTTPResult)));
1477 :
1478 : struct curl_slist *headers = reinterpret_cast<struct curl_slist *>(
1479 1175 : CPLHTTPSetOptions(http_handle, pszURL, papszOptions));
1480 1175 : if (headers != nullptr)
1481 581 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTPHEADER, headers);
1482 :
1483 : // Are we making a head request.
1484 1175 : const char *pszNoBody = nullptr;
1485 1175 : if ((pszNoBody = CSLFetchNameValue(papszOptions, "NO_BODY")) != nullptr)
1486 : {
1487 0 : if (CPLTestBool(pszNoBody))
1488 : {
1489 0 : CPLDebug("HTTP", "HEAD Request: %s", pszURL);
1490 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_NOBODY, 1L);
1491 : }
1492 : }
1493 :
1494 : // Capture response headers.
1495 1175 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HEADERDATA, psResult);
1496 1175 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HEADERFUNCTION,
1497 : CPLHdrWriteFct);
1498 :
1499 1175 : CPLHTTPResultWithLimit sResultWithLimit;
1500 1175 : if (nullptr == pfnWrite)
1501 : {
1502 1174 : pfnWrite = CPLWriteFct;
1503 :
1504 1174 : sResultWithLimit.psResult = psResult;
1505 1174 : sResultWithLimit.nMaxFileSize = 0;
1506 : const char *pszMaxFileSize =
1507 1174 : CSLFetchNameValue(papszOptions, "MAX_FILE_SIZE");
1508 1174 : if (pszMaxFileSize != nullptr)
1509 : {
1510 10 : sResultWithLimit.nMaxFileSize = atoi(pszMaxFileSize);
1511 : // Only useful if size is returned by server before actual download.
1512 10 : unchecked_curl_easy_setopt(http_handle, CURLOPT_MAXFILESIZE,
1513 : sResultWithLimit.nMaxFileSize);
1514 : }
1515 1174 : pWriteArg = &sResultWithLimit;
1516 : }
1517 :
1518 1175 : unchecked_curl_easy_setopt(http_handle, CURLOPT_WRITEDATA, pWriteArg);
1519 1175 : unchecked_curl_easy_setopt(http_handle, CURLOPT_WRITEFUNCTION, pfnWrite);
1520 :
1521 1175 : CurlProcessData stProcessData = {pfnProgress, pProgressArg};
1522 1175 : if (nullptr != pfnProgress)
1523 : {
1524 24 : unchecked_curl_easy_setopt(http_handle, CURLOPT_XFERINFOFUNCTION,
1525 : NewProcessFunction);
1526 24 : unchecked_curl_easy_setopt(http_handle, CURLOPT_XFERINFODATA,
1527 : &stProcessData);
1528 24 : unchecked_curl_easy_setopt(http_handle, CURLOPT_NOPROGRESS, 0L);
1529 : }
1530 :
1531 1175 : szCurlErrBuf[0] = '\0';
1532 :
1533 1175 : unchecked_curl_easy_setopt(http_handle, CURLOPT_ERRORBUFFER, szCurlErrBuf);
1534 :
1535 1175 : bool bGZipRequested = false;
1536 1175 : if (bSupportGZip && CPLTestBool(CPLGetConfigOption("CPL_CURL_GZIP", "YES")))
1537 : {
1538 1175 : bGZipRequested = true;
1539 1175 : unchecked_curl_easy_setopt(http_handle, CURLOPT_ENCODING, "gzip");
1540 : }
1541 :
1542 2350 : CPLHTTPPostFields oPostFields;
1543 1175 : if (oPostFields.Fill(http_handle, papszOptions) != CE_None)
1544 : {
1545 2 : psResult->nStatus = 34; // CURLE_HTTP_POST_ERROR
1546 2 : psResult->pszErrBuf = CPLStrdup(oPostFields.GetErrorMessage().c_str());
1547 2 : CPLError(CE_Failure, CPLE_AppDefined, "%s", psResult->pszErrBuf);
1548 2 : CPLHTTPFetchCleanup(http_handle, headers, pszPersistent, papszOptions);
1549 2 : return psResult;
1550 : }
1551 :
1552 : /* -------------------------------------------------------------------- */
1553 : /* Depending on status code, retry this HTTP call until max */
1554 : /* retry has been reached */
1555 : /* -------------------------------------------------------------------- */
1556 1173 : const char *pszRetryDelay = CSLFetchNameValue(papszOptions, "RETRY_DELAY");
1557 1173 : if (pszRetryDelay == nullptr)
1558 1173 : pszRetryDelay = CPLGetConfigOption(
1559 : "GDAL_HTTP_RETRY_DELAY", CPLSPrintf("%f", CPL_HTTP_RETRY_DELAY));
1560 1173 : const char *pszMaxRetries = CSLFetchNameValue(papszOptions, "MAX_RETRY");
1561 1173 : if (pszMaxRetries == nullptr)
1562 1109 : pszMaxRetries = CPLGetConfigOption(
1563 : "GDAL_HTTP_MAX_RETRY", CPLSPrintf("%d", CPL_HTTP_MAX_RETRY));
1564 : // coverity[tainted_data]
1565 1173 : double dfRetryDelaySecs = CPLAtof(pszRetryDelay);
1566 1173 : int nMaxRetries = atoi(pszMaxRetries);
1567 1173 : const char *pszRetryCodes = CSLFetchNameValue(papszOptions, "RETRY_CODES");
1568 1173 : if (!pszRetryCodes)
1569 1173 : pszRetryCodes = CPLGetConfigOption("GDAL_HTTP_RETRY_CODES", nullptr);
1570 1173 : int nRetryCount = 0;
1571 :
1572 : while (true)
1573 : {
1574 : /* --------------------------------------------------------------------
1575 : */
1576 : /* Execute the request, waiting for results. */
1577 : /* --------------------------------------------------------------------
1578 : */
1579 1173 : void *old_handler = CPLHTTPIgnoreSigPipe();
1580 1173 : psResult->nStatus = static_cast<int>(curl_easy_perform(http_handle));
1581 1173 : CPLHTTPRestoreSigPipeHandler(old_handler);
1582 :
1583 : /* --------------------------------------------------------------------
1584 : */
1585 : /* Fetch content-type if possible. */
1586 : /* --------------------------------------------------------------------
1587 : */
1588 1173 : psResult->pszContentType = nullptr;
1589 1173 : curl_easy_getinfo(http_handle, CURLINFO_CONTENT_TYPE,
1590 : &(psResult->pszContentType));
1591 1173 : if (psResult->pszContentType != nullptr)
1592 532 : psResult->pszContentType = CPLStrdup(psResult->pszContentType);
1593 :
1594 1173 : long response_code = 0;
1595 1173 : curl_easy_getinfo(http_handle, CURLINFO_RESPONSE_CODE, &response_code);
1596 1173 : if (response_code != 200)
1597 : {
1598 420 : const double dfNewRetryDelay = CPLHTTPGetNewRetryDelay(
1599 : static_cast<int>(response_code), dfRetryDelaySecs,
1600 210 : reinterpret_cast<const char *>(psResult->pabyData),
1601 : szCurlErrBuf, pszRetryCodes);
1602 210 : if (dfNewRetryDelay > 0 && nRetryCount < nMaxRetries)
1603 : {
1604 0 : CPLError(CE_Warning, CPLE_AppDefined,
1605 : "HTTP error code: %d - %s. "
1606 : "Retrying again in %.1f secs",
1607 : static_cast<int>(response_code), pszURL,
1608 : dfRetryDelaySecs);
1609 0 : CPLSleep(dfRetryDelaySecs);
1610 0 : dfRetryDelaySecs = dfNewRetryDelay;
1611 0 : nRetryCount++;
1612 :
1613 0 : CPLFree(psResult->pszContentType);
1614 0 : psResult->pszContentType = nullptr;
1615 0 : CSLDestroy(psResult->papszHeaders);
1616 0 : psResult->papszHeaders = nullptr;
1617 0 : CPLFree(psResult->pabyData);
1618 0 : psResult->pabyData = nullptr;
1619 0 : psResult->nDataLen = 0;
1620 0 : psResult->nDataAlloc = 0;
1621 :
1622 0 : continue;
1623 : }
1624 : }
1625 :
1626 : /* --------------------------------------------------------------------
1627 : */
1628 : /* Have we encountered some sort of error? */
1629 : /* --------------------------------------------------------------------
1630 : */
1631 1173 : if (strlen(szCurlErrBuf) > 0)
1632 : {
1633 8 : bool bSkipError = false;
1634 : const char *pszContentLength =
1635 8 : CSLFetchNameValue(psResult->papszHeaders, "Content-Length");
1636 : // Some servers such as
1637 : // http://115.113.193.14/cgi-bin/world/qgis_mapserv.fcgi?VERSION=1.1.1&SERVICE=WMS&REQUEST=GetCapabilities
1638 : // invalidly return Content-Length as the uncompressed size, with
1639 : // makes curl to wait for more data and time-out finally. If we got
1640 : // the expected data size, then we don't emit an error but turn off
1641 : // GZip requests.
1642 8 : if (bGZipRequested &&
1643 8 : strstr(szCurlErrBuf, "transfer closed with") &&
1644 0 : strstr(szCurlErrBuf, "bytes remaining to read"))
1645 : {
1646 0 : if (pszContentLength && psResult->nDataLen != 0 &&
1647 0 : atoi(pszContentLength) == psResult->nDataLen)
1648 : {
1649 : const char *pszCurlGZIPOption =
1650 0 : CPLGetConfigOption("CPL_CURL_GZIP", nullptr);
1651 0 : if (pszCurlGZIPOption == nullptr)
1652 : {
1653 0 : CPLSetConfigOption("CPL_CURL_GZIP", "NO");
1654 0 : CPLDebug("HTTP",
1655 : "Disabling CPL_CURL_GZIP, "
1656 : "because %s doesn't support it properly",
1657 : pszURL);
1658 : }
1659 0 : psResult->nStatus = 0;
1660 0 : bSkipError = true;
1661 0 : }
1662 : }
1663 :
1664 : // Ignore SSL errors about non-properly terminated connection,
1665 : // often due to HTTP proxies
1666 8 : else if (pszContentLength == nullptr &&
1667 : // Cf https://github.com/curl/curl/pull/3148
1668 8 : (strstr(szCurlErrBuf,
1669 : "GnuTLS recv error (-110): The TLS connection was "
1670 8 : "non-properly terminated") != nullptr ||
1671 : // Cf https://github.com/curl/curl/issues/9024
1672 8 : strstr(szCurlErrBuf,
1673 : "SSL_read: error:0A000126:SSL "
1674 : "routines::unexpected eof while reading") !=
1675 : nullptr))
1676 : {
1677 0 : psResult->nStatus = 0;
1678 0 : bSkipError = true;
1679 : }
1680 8 : else if (CPLTestBool(
1681 : CPLGetConfigOption("CPL_CURL_IGNORE_ERROR", "NO")))
1682 : {
1683 0 : psResult->nStatus = 0;
1684 0 : bSkipError = true;
1685 : }
1686 :
1687 8 : if (!bSkipError)
1688 : {
1689 8 : psResult->pszErrBuf = CPLStrdup(szCurlErrBuf);
1690 8 : if (psResult->nDataLen > 0)
1691 : {
1692 1 : CPLError(CE_Failure, CPLE_AppDefined,
1693 : "%s. You may set the CPL_CURL_IGNORE_ERROR "
1694 : "configuration option to YES to try to ignore it.",
1695 : szCurlErrBuf);
1696 : }
1697 : else
1698 : {
1699 7 : CPLError(CE_Failure, CPLE_AppDefined, "%s", szCurlErrBuf);
1700 : }
1701 : }
1702 : }
1703 : else
1704 : {
1705 1165 : if (response_code >= 400 && response_code < 600)
1706 : {
1707 191 : psResult->pszErrBuf = CPLStrdup(CPLSPrintf(
1708 : "HTTP error code : %d", static_cast<int>(response_code)));
1709 191 : CPLError(CE_Failure, CPLE_AppDefined, "%s",
1710 : psResult->pszErrBuf);
1711 : }
1712 : }
1713 1173 : break;
1714 0 : }
1715 :
1716 1173 : CPLHTTPFetchCleanup(http_handle, headers, pszPersistent, papszOptions);
1717 :
1718 1173 : return psResult;
1719 : #endif /* def HAVE_CURL */
1720 : }
1721 :
1722 : #ifdef HAVE_CURL
1723 : /************************************************************************/
1724 : /* CPLMultiPerformWait() */
1725 : /************************************************************************/
1726 :
1727 3310 : bool CPLMultiPerformWait(void *hCurlMultiHandleIn, int & /*repeats*/)
1728 : {
1729 3310 : CURLM *hCurlMultiHandle = static_cast<CURLM *>(hCurlMultiHandleIn);
1730 :
1731 : // Wait for events on the sockets
1732 :
1733 : // Using curl_multi_poll() is preferred to avoid hitting the 1024 file
1734 : // descriptor limit
1735 :
1736 3310 : int numfds = 0;
1737 3310 : if (curl_multi_poll(hCurlMultiHandle, nullptr, 0, 1000, &numfds) !=
1738 : CURLM_OK)
1739 : {
1740 0 : CPLError(CE_Failure, CPLE_AppDefined, "curl_multi_poll() failed");
1741 0 : return false;
1742 : }
1743 3309 : return true;
1744 : }
1745 :
1746 : class CPLHTTPErrorBuffer
1747 : {
1748 : public:
1749 : char szBuffer[CURL_ERROR_SIZE + 1];
1750 :
1751 0 : CPLHTTPErrorBuffer()
1752 0 : {
1753 0 : szBuffer[0] = '\0';
1754 0 : }
1755 : };
1756 :
1757 : #endif // HAVE_CURL
1758 :
1759 : /************************************************************************/
1760 : /* CPLHTTPMultiFetch() */
1761 : /************************************************************************/
1762 :
1763 : /**
1764 : * \brief Fetch several documents at once
1765 : *
1766 : * @param papszURL array of valid URLs recognized by underlying download library
1767 : * (libcurl)
1768 : * @param nURLCount number of URLs of papszURL
1769 : * @param nMaxSimultaneous maximum number of downloads to issue simultaneously.
1770 : * Any negative or zer value means unlimited.
1771 : * @param papszOptions option list as a NULL-terminated array of strings. May be
1772 : * NULL. Refer to CPLHTTPFetch() for valid options.
1773 : * @return an array of CPLHTTPResult* structures that must be freed by
1774 : * CPLHTTPDestroyMultiResult() or NULL if libcurl support is disabled
1775 : *
1776 : * @since GDAL 2.3
1777 : */
1778 0 : CPLHTTPResult **CPLHTTPMultiFetch(const char *const *papszURL, int nURLCount,
1779 : int nMaxSimultaneous,
1780 : CSLConstList papszOptions)
1781 : {
1782 : #ifndef HAVE_CURL
1783 : (void)papszURL;
1784 : (void)nURLCount;
1785 : (void)nMaxSimultaneous;
1786 : (void)papszOptions;
1787 :
1788 : CPLError(CE_Failure, CPLE_NotSupported,
1789 : "GDAL/OGR not compiled with libcurl support, "
1790 : "remote requests not supported.");
1791 : return nullptr;
1792 : #else /* def HAVE_CURL */
1793 :
1794 : /* -------------------------------------------------------------------- */
1795 : /* Are we using a persistent named session? If so, search for */
1796 : /* or create it. */
1797 : /* */
1798 : /* Currently this code does not attempt to protect against */
1799 : /* multiple threads asking for the same named session. If that */
1800 : /* occurs it will be in use in multiple threads at once, which */
1801 : /* will lead to potential crashes in libcurl. */
1802 : /* -------------------------------------------------------------------- */
1803 0 : CURLM *hCurlMultiHandle = nullptr;
1804 :
1805 0 : const char *pszPersistent = CSLFetchNameValue(papszOptions, "PERSISTENT");
1806 : const char *pszClosePersistent =
1807 0 : CSLFetchNameValue(papszOptions, "CLOSE_PERSISTENT");
1808 0 : if (pszPersistent)
1809 : {
1810 0 : CPLString osSessionName = pszPersistent;
1811 0 : CPLMutexHolder oHolder(&hSessionMapMutex);
1812 :
1813 0 : if (poSessionMultiMap == nullptr)
1814 0 : poSessionMultiMap = new std::map<CPLString, CURLM *>;
1815 0 : if (poSessionMultiMap->count(osSessionName) == 0)
1816 : {
1817 0 : (*poSessionMultiMap)[osSessionName] = curl_multi_init();
1818 0 : CPLDebug("HTTP", "Establish persistent session named '%s'.",
1819 : osSessionName.c_str());
1820 : }
1821 :
1822 0 : hCurlMultiHandle = (*poSessionMultiMap)[osSessionName];
1823 : }
1824 : /* -------------------------------------------------------------------- */
1825 : /* Are we requested to close a persistent named session? */
1826 : /* -------------------------------------------------------------------- */
1827 0 : else if (pszClosePersistent)
1828 : {
1829 0 : CPLString osSessionName = pszClosePersistent;
1830 0 : CPLMutexHolder oHolder(&hSessionMapMutex);
1831 :
1832 0 : if (poSessionMultiMap)
1833 : {
1834 0 : auto oIter = poSessionMultiMap->find(osSessionName);
1835 0 : if (oIter != poSessionMultiMap->end())
1836 : {
1837 0 : VSICURLMultiCleanup(oIter->second);
1838 0 : poSessionMultiMap->erase(oIter);
1839 0 : if (poSessionMultiMap->empty())
1840 : {
1841 0 : delete poSessionMultiMap;
1842 0 : poSessionMultiMap = nullptr;
1843 : }
1844 0 : CPLDebug("HTTP", "Ended persistent session named '%s'.",
1845 : osSessionName.c_str());
1846 : }
1847 : else
1848 : {
1849 0 : CPLDebug("HTTP",
1850 : "Could not find persistent session named '%s'.",
1851 : osSessionName.c_str());
1852 : }
1853 : }
1854 :
1855 0 : return nullptr;
1856 : }
1857 : else
1858 : {
1859 0 : hCurlMultiHandle = curl_multi_init();
1860 : }
1861 :
1862 : CPLHTTPResult **papsResults = static_cast<CPLHTTPResult **>(
1863 0 : CPLCalloc(nURLCount, sizeof(CPLHTTPResult *)));
1864 0 : std::vector<CURL *> asHandles;
1865 0 : std::vector<CPLHTTPResultWithLimit> asResults;
1866 0 : asResults.resize(nURLCount);
1867 0 : std::vector<struct curl_slist *> aHeaders;
1868 0 : aHeaders.resize(nURLCount);
1869 0 : std::vector<CPLHTTPErrorBuffer> asErrorBuffers;
1870 0 : asErrorBuffers.resize(nURLCount);
1871 :
1872 0 : for (int i = 0; i < nURLCount; i++)
1873 : {
1874 0 : papsResults[i] =
1875 0 : static_cast<CPLHTTPResult *>(CPLCalloc(1, sizeof(CPLHTTPResult)));
1876 :
1877 0 : const char *pszURL = papszURL[i];
1878 0 : CURL *http_handle = curl_easy_init();
1879 :
1880 0 : aHeaders[i] = reinterpret_cast<struct curl_slist *>(
1881 0 : CPLHTTPSetOptions(http_handle, pszURL, papszOptions));
1882 :
1883 : // Set Headers.
1884 0 : const char *pszHeaders = CSLFetchNameValue(papszOptions, "HEADERS");
1885 0 : if (pszHeaders != nullptr)
1886 : {
1887 : char **papszTokensHeaders =
1888 0 : CSLTokenizeString2(pszHeaders, "\r\n", 0);
1889 0 : for (int j = 0; papszTokensHeaders[j] != nullptr; ++j)
1890 0 : aHeaders[i] =
1891 0 : curl_slist_append(aHeaders[i], papszTokensHeaders[j]);
1892 0 : CSLDestroy(papszTokensHeaders);
1893 : }
1894 :
1895 0 : if (aHeaders[i] != nullptr)
1896 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTPHEADER,
1897 : aHeaders[i]);
1898 :
1899 : // Capture response headers.
1900 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HEADERDATA,
1901 : papsResults[i]);
1902 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HEADERFUNCTION,
1903 : CPLHdrWriteFct);
1904 :
1905 0 : asResults[i].psResult = papsResults[i];
1906 : const char *pszMaxFileSize =
1907 0 : CSLFetchNameValue(papszOptions, "MAX_FILE_SIZE");
1908 0 : if (pszMaxFileSize != nullptr)
1909 : {
1910 0 : asResults[i].nMaxFileSize = atoi(pszMaxFileSize);
1911 : // Only useful if size is returned by server before actual download.
1912 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_MAXFILESIZE,
1913 : asResults[i].nMaxFileSize);
1914 : }
1915 :
1916 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_WRITEDATA,
1917 : &asResults[i]);
1918 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_WRITEFUNCTION,
1919 : CPLWriteFct);
1920 :
1921 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_ERRORBUFFER,
1922 : asErrorBuffers[i].szBuffer);
1923 :
1924 0 : if (bSupportGZip &&
1925 0 : CPLTestBool(CPLGetConfigOption("CPL_CURL_GZIP", "YES")))
1926 : {
1927 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_ENCODING, "gzip");
1928 : }
1929 :
1930 0 : asHandles.push_back(http_handle);
1931 : }
1932 :
1933 0 : int iCurRequest = 0;
1934 0 : for (;
1935 0 : iCurRequest <
1936 0 : std::min(nURLCount, nMaxSimultaneous > 0 ? nMaxSimultaneous : INT_MAX);
1937 : iCurRequest++)
1938 : {
1939 0 : CPLHTTPEmitFetchDebug(papszURL[iCurRequest],
1940 : CPLSPrintf(" %d/%d", iCurRequest + 1, nURLCount));
1941 0 : curl_multi_add_handle(hCurlMultiHandle, asHandles[iCurRequest]);
1942 : }
1943 :
1944 0 : int repeats = 0;
1945 0 : void *old_handler = CPLHTTPIgnoreSigPipe();
1946 : while (true)
1947 : {
1948 0 : int still_running = 0;
1949 0 : while (curl_multi_perform(hCurlMultiHandle, &still_running) ==
1950 : CURLM_CALL_MULTI_PERFORM)
1951 : {
1952 : // loop
1953 : }
1954 0 : if (!still_running && iCurRequest == nURLCount)
1955 : {
1956 0 : break;
1957 : }
1958 :
1959 0 : bool bRequestsAdded = false;
1960 : CURLMsg *msg;
1961 0 : do
1962 : {
1963 0 : int msgq = 0;
1964 0 : msg = curl_multi_info_read(hCurlMultiHandle, &msgq);
1965 0 : if (msg && (msg->msg == CURLMSG_DONE))
1966 : {
1967 0 : if (iCurRequest < nURLCount)
1968 : {
1969 0 : CPLHTTPEmitFetchDebug(
1970 0 : papszURL[iCurRequest],
1971 : CPLSPrintf(" %d/%d", iCurRequest + 1, nURLCount));
1972 0 : curl_multi_add_handle(hCurlMultiHandle,
1973 0 : asHandles[iCurRequest]);
1974 0 : iCurRequest++;
1975 0 : bRequestsAdded = true;
1976 : }
1977 : }
1978 0 : } while (msg);
1979 :
1980 0 : if (!bRequestsAdded)
1981 0 : CPLMultiPerformWait(hCurlMultiHandle, repeats);
1982 0 : }
1983 0 : CPLHTTPRestoreSigPipeHandler(old_handler);
1984 :
1985 0 : for (int i = 0; i < nURLCount; i++)
1986 : {
1987 0 : if (asErrorBuffers[i].szBuffer[0] != '\0')
1988 0 : papsResults[i]->pszErrBuf = CPLStrdup(asErrorBuffers[i].szBuffer);
1989 : else
1990 : {
1991 0 : long response_code = 0;
1992 0 : curl_easy_getinfo(asHandles[i], CURLINFO_RESPONSE_CODE,
1993 : &response_code);
1994 :
1995 0 : if (response_code >= 400 && response_code < 600)
1996 : {
1997 0 : papsResults[i]->pszErrBuf = CPLStrdup(CPLSPrintf(
1998 : "HTTP error code : %d", static_cast<int>(response_code)));
1999 : }
2000 : }
2001 :
2002 0 : curl_easy_getinfo(asHandles[i], CURLINFO_CONTENT_TYPE,
2003 : &(papsResults[i]->pszContentType));
2004 0 : if (papsResults[i]->pszContentType != nullptr)
2005 0 : papsResults[i]->pszContentType =
2006 0 : CPLStrdup(papsResults[i]->pszContentType);
2007 :
2008 0 : curl_multi_remove_handle(hCurlMultiHandle, asHandles[i]);
2009 0 : curl_easy_cleanup(asHandles[i]);
2010 : }
2011 :
2012 0 : if (!pszPersistent)
2013 0 : VSICURLMultiCleanup(hCurlMultiHandle);
2014 :
2015 0 : for (size_t i = 0; i < aHeaders.size(); i++)
2016 0 : curl_slist_free_all(aHeaders[i]);
2017 :
2018 0 : return papsResults;
2019 : #endif /* def HAVE_CURL */
2020 : }
2021 :
2022 : /************************************************************************/
2023 : /* CPLHTTPDestroyMultiResult() */
2024 : /************************************************************************/
2025 : /**
2026 : * \brief Clean the memory associated with the return value of
2027 : * CPLHTTPMultiFetch()
2028 : *
2029 : * @param papsResults pointer to the return value of CPLHTTPMultiFetch()
2030 : * @param nCount value of the nURLCount parameter passed to CPLHTTPMultiFetch()
2031 : * @since GDAL 2.3
2032 : */
2033 0 : void CPLHTTPDestroyMultiResult(CPLHTTPResult **papsResults, int nCount)
2034 : {
2035 0 : if (papsResults)
2036 : {
2037 0 : for (int i = 0; i < nCount; i++)
2038 : {
2039 0 : CPLHTTPDestroyResult(papsResults[i]);
2040 : }
2041 0 : CPLFree(papsResults);
2042 : }
2043 0 : }
2044 :
2045 : #ifdef HAVE_CURL
2046 :
2047 : #ifdef _WIN32
2048 :
2049 : #include <windows.h>
2050 :
2051 : /************************************************************************/
2052 : /* CPLFindWin32CurlCaBundleCrt() */
2053 : /************************************************************************/
2054 :
2055 : static const char *CPLFindWin32CurlCaBundleCrt()
2056 : {
2057 : DWORD nResLen;
2058 : const DWORD nBufSize = MAX_PATH + 1;
2059 : char *pszFilePart = nullptr;
2060 :
2061 : char *pszPath = static_cast<char *>(CPLCalloc(1, nBufSize));
2062 :
2063 : nResLen = SearchPathA(nullptr, "curl-ca-bundle.crt", nullptr, nBufSize,
2064 : pszPath, &pszFilePart);
2065 : if (nResLen > 0)
2066 : {
2067 : const char *pszRet = CPLSPrintf("%s", pszPath);
2068 : CPLFree(pszPath);
2069 : return pszRet;
2070 : }
2071 : CPLFree(pszPath);
2072 : return nullptr;
2073 : }
2074 :
2075 : #endif // WIN32
2076 :
2077 : /************************************************************************/
2078 : /* CPLHTTPCurlDebugFunction() */
2079 : /************************************************************************/
2080 :
2081 16 : static int CPLHTTPCurlDebugFunction(CURL *handle, curl_infotype type,
2082 : char *data, size_t size, void *userp)
2083 : {
2084 : (void)handle;
2085 : (void)userp;
2086 :
2087 16 : const char *pszDebugKey = nullptr;
2088 16 : if (type == CURLINFO_TEXT)
2089 : {
2090 10 : pszDebugKey = "CURL_INFO_TEXT";
2091 : }
2092 6 : else if (type == CURLINFO_HEADER_OUT)
2093 : {
2094 1 : pszDebugKey = "CURL_INFO_HEADER_OUT";
2095 : }
2096 5 : else if (type == CURLINFO_HEADER_IN)
2097 : {
2098 5 : pszDebugKey = "CURL_INFO_HEADER_IN";
2099 : }
2100 0 : else if (type == CURLINFO_DATA_IN &&
2101 0 : CPLTestBool(CPLGetConfigOption("CPL_CURL_VERBOSE_DATA_IN", "NO")))
2102 : {
2103 0 : pszDebugKey = "CURL_INFO_DATA_IN";
2104 : }
2105 :
2106 16 : if (pszDebugKey)
2107 : {
2108 32 : std::string osMsg(data, size);
2109 16 : if (!osMsg.empty() && osMsg.back() == '\n')
2110 16 : osMsg.pop_back();
2111 16 : CPLDebug(pszDebugKey, "%s", osMsg.c_str());
2112 : }
2113 16 : return 0;
2114 : }
2115 :
2116 : /************************************************************************/
2117 : /* CPLHTTPSetOptions() */
2118 : /************************************************************************/
2119 :
2120 : // Note: papszOptions must be kept alive until curl_easy/multi_perform()
2121 : // has completed, and we must be careful not to set short lived strings
2122 : // with unchecked_curl_easy_setopt(), as long as we need to support curl < 7.17
2123 : // see https://curl.haxx.se/libcurl/c/unchecked_curl_easy_setopt.html
2124 : // caution: if we remove that assumption, we'll needto use
2125 : // CURLOPT_COPYPOSTFIELDS
2126 :
2127 2601 : void *CPLHTTPSetOptions(void *pcurl, const char *pszURL,
2128 : const char *const *papszOptions)
2129 : {
2130 2601 : CheckCurlFeatures();
2131 :
2132 2601 : CURL *http_handle = reinterpret_cast<CURL *>(pcurl);
2133 :
2134 2601 : unchecked_curl_easy_setopt(http_handle, CURLOPT_URL, pszURL);
2135 :
2136 2601 : if (CPLTestBool(CPLGetConfigOption("CPL_CURL_VERBOSE", "NO")))
2137 : {
2138 1 : unchecked_curl_easy_setopt(http_handle, CURLOPT_VERBOSE, 1);
2139 :
2140 1 : if (CPLIsDebugEnabled())
2141 : {
2142 1 : unchecked_curl_easy_setopt(http_handle, CURLOPT_DEBUGFUNCTION,
2143 : CPLHTTPCurlDebugFunction);
2144 : }
2145 : }
2146 :
2147 2601 : const char *pszPathAsIt = CSLFetchNameValue(papszOptions, "PATH_VERBATIM");
2148 2601 : if (pszPathAsIt == nullptr)
2149 2600 : pszPathAsIt = CPLGetConfigOption("GDAL_HTTP_PATH_VERBATIM", nullptr);
2150 2601 : if (pszPathAsIt && CPLTestBool(pszPathAsIt))
2151 : {
2152 1 : unchecked_curl_easy_setopt(http_handle, CURLOPT_PATH_AS_IS, 1L);
2153 : }
2154 :
2155 : const char *pszHttpVersion =
2156 2601 : CSLFetchNameValue(papszOptions, "HTTP_VERSION");
2157 2601 : if (pszHttpVersion == nullptr)
2158 2601 : pszHttpVersion = CPLGetConfigOption("GDAL_HTTP_VERSION", nullptr);
2159 2601 : if (pszHttpVersion && strcmp(pszHttpVersion, "1.0") == 0)
2160 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTP_VERSION,
2161 : CURL_HTTP_VERSION_1_0);
2162 2601 : else if (pszHttpVersion && strcmp(pszHttpVersion, "1.1") == 0)
2163 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTP_VERSION,
2164 : CURL_HTTP_VERSION_1_1);
2165 2601 : else if (pszHttpVersion && (strcmp(pszHttpVersion, "2") == 0 ||
2166 0 : strcmp(pszHttpVersion, "2.0") == 0))
2167 : {
2168 0 : if (bSupportHTTP2)
2169 : {
2170 : // Try HTTP/2 both for HTTP and HTTPS. With fallback to HTTP/1.1
2171 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTP_VERSION,
2172 : CURL_HTTP_VERSION_2_0);
2173 : }
2174 : }
2175 2601 : else if (pszHttpVersion && strcmp(pszHttpVersion, "2PRIOR_KNOWLEDGE") == 0)
2176 : {
2177 0 : if (bSupportHTTP2)
2178 : {
2179 : // Assume HTTP/2 is supported by the server. The cURL docs indicate
2180 : // that it makes no difference for HTTPS, but it does seem to work
2181 : // in practice.
2182 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTP_VERSION,
2183 : CURL_HTTP_VERSION_2_PRIOR_KNOWLEDGE);
2184 : }
2185 : }
2186 2601 : else if (pszHttpVersion == nullptr || strcmp(pszHttpVersion, "2TLS") == 0)
2187 : {
2188 2601 : if (bSupportHTTP2)
2189 : {
2190 : // Only enable this mode if explicitly required, or if the
2191 : // machine is a GCE instance. On other networks, requesting a
2192 : // file in HTTP/2 is found to be significantly slower than HTTP/1.1
2193 : // for unknown reasons.
2194 2601 : if (pszHttpVersion != nullptr || CPLIsMachineForSureGCEInstance())
2195 : {
2196 : static bool bDebugEmitted = false;
2197 0 : if (!bDebugEmitted)
2198 : {
2199 0 : CPLDebug("HTTP", "Using HTTP/2 for HTTPS when possible");
2200 0 : bDebugEmitted = true;
2201 : }
2202 :
2203 : // CURL_HTTP_VERSION_2TLS means for HTTPS connection, try to
2204 : // negotiate HTTP/2 with the server (and fallback to HTTP/1.1
2205 : // otherwise), and for HTTP connection do HTTP/1
2206 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTP_VERSION,
2207 : CURL_HTTP_VERSION_2TLS);
2208 : }
2209 2601 : }
2210 : }
2211 : else
2212 : {
2213 0 : CPLError(CE_Warning, CPLE_NotSupported, "HTTP_VERSION=%s not supported",
2214 : pszHttpVersion);
2215 : }
2216 :
2217 : // Default value is 1 since curl 7.50.2. But worth applying it on
2218 : // previous versions as well.
2219 : const char *pszTCPNoDelay =
2220 2601 : CSLFetchNameValueDef(papszOptions, "TCP_NODELAY", "1");
2221 2601 : unchecked_curl_easy_setopt(http_handle, CURLOPT_TCP_NODELAY,
2222 : atoi(pszTCPNoDelay));
2223 :
2224 : /* Support control over HTTPAUTH */
2225 2601 : const char *pszHttpAuth = CSLFetchNameValue(papszOptions, "HTTPAUTH");
2226 2601 : if (pszHttpAuth == nullptr)
2227 2591 : pszHttpAuth = CPLGetConfigOption("GDAL_HTTP_AUTH", nullptr);
2228 2601 : if (pszHttpAuth == nullptr)
2229 : {
2230 : /* do nothing */;
2231 : }
2232 15 : else if (EQUAL(pszHttpAuth, "BASIC"))
2233 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTPAUTH,
2234 : CURLAUTH_BASIC);
2235 15 : else if (EQUAL(pszHttpAuth, "NTLM"))
2236 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTPAUTH,
2237 : CURLAUTH_NTLM);
2238 15 : else if (EQUAL(pszHttpAuth, "ANY"))
2239 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTPAUTH, CURLAUTH_ANY);
2240 15 : else if (EQUAL(pszHttpAuth, "ANYSAFE"))
2241 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTPAUTH,
2242 : CURLAUTH_ANYSAFE);
2243 15 : else if (EQUAL(pszHttpAuth, "BEARER"))
2244 : {
2245 15 : const char *pszAuthorizationHeaderAllowed = CSLFetchNameValueDef(
2246 : papszOptions, "AUTHORIZATION_HEADER_ALLOWED", "YES");
2247 : const bool bAuthorizationHeaderAllowed =
2248 15 : CPLTestBool(pszAuthorizationHeaderAllowed);
2249 15 : if (bAuthorizationHeaderAllowed)
2250 : {
2251 : const char *pszBearer =
2252 12 : CSLFetchNameValue(papszOptions, "HTTP_BEARER");
2253 12 : if (pszBearer == nullptr)
2254 5 : pszBearer = CPLGetConfigOption("GDAL_HTTP_BEARER", nullptr);
2255 12 : if (pszBearer != nullptr)
2256 12 : unchecked_curl_easy_setopt(http_handle, CURLOPT_XOAUTH2_BEARER,
2257 : pszBearer);
2258 12 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTPAUTH,
2259 : CURLAUTH_BEARER);
2260 : }
2261 : }
2262 0 : else if (EQUAL(pszHttpAuth, "NEGOTIATE"))
2263 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTPAUTH,
2264 : CURLAUTH_NEGOTIATE);
2265 : else
2266 : {
2267 0 : CPLError(CE_Warning, CPLE_AppDefined,
2268 : "Unsupported HTTPAUTH value '%s', ignored.", pszHttpAuth);
2269 : }
2270 :
2271 : const char *pszGssDelegation =
2272 2601 : CSLFetchNameValue(papszOptions, "GSSAPI_DELEGATION");
2273 2601 : if (pszGssDelegation == nullptr)
2274 : pszGssDelegation =
2275 2601 : CPLGetConfigOption("GDAL_GSSAPI_DELEGATION", nullptr);
2276 2601 : if (pszGssDelegation == nullptr)
2277 : {
2278 : }
2279 0 : else if (EQUAL(pszGssDelegation, "NONE"))
2280 : {
2281 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_GSSAPI_DELEGATION,
2282 : CURLGSSAPI_DELEGATION_NONE);
2283 : }
2284 0 : else if (EQUAL(pszGssDelegation, "POLICY"))
2285 : {
2286 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_GSSAPI_DELEGATION,
2287 : CURLGSSAPI_DELEGATION_POLICY_FLAG);
2288 : }
2289 0 : else if (EQUAL(pszGssDelegation, "ALWAYS"))
2290 : {
2291 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_GSSAPI_DELEGATION,
2292 : CURLGSSAPI_DELEGATION_FLAG);
2293 : }
2294 : else
2295 : {
2296 0 : CPLError(CE_Warning, CPLE_AppDefined,
2297 : "Unsupported GSSAPI_DELEGATION value '%s', ignored.",
2298 : pszGssDelegation);
2299 : }
2300 :
2301 : // Support use of .netrc - default enabled.
2302 2601 : const char *pszHttpNetrc = CSLFetchNameValue(papszOptions, "NETRC");
2303 2601 : if (pszHttpNetrc == nullptr)
2304 2601 : pszHttpNetrc = CPLGetConfigOption("GDAL_HTTP_NETRC", "YES");
2305 2601 : if (pszHttpNetrc == nullptr || CPLTestBool(pszHttpNetrc))
2306 2601 : unchecked_curl_easy_setopt(http_handle, CURLOPT_NETRC, 1L);
2307 :
2308 : // Custom .netrc file location
2309 : const char *pszHttpNetrcFile =
2310 2601 : CSLFetchNameValue(papszOptions, "NETRC_FILE");
2311 2601 : if (pszHttpNetrcFile == nullptr)
2312 2601 : pszHttpNetrcFile = CPLGetConfigOption("GDAL_HTTP_NETRC_FILE", nullptr);
2313 2601 : if (pszHttpNetrcFile)
2314 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_NETRC_FILE,
2315 : pszHttpNetrcFile);
2316 :
2317 : // Support setting userid:password.
2318 2601 : const char *pszUserPwd = CSLFetchNameValue(papszOptions, "USERPWD");
2319 2601 : if (pszUserPwd == nullptr)
2320 2563 : pszUserPwd = CPLGetConfigOption("GDAL_HTTP_USERPWD", nullptr);
2321 2601 : if (pszUserPwd != nullptr)
2322 38 : unchecked_curl_easy_setopt(http_handle, CURLOPT_USERPWD, pszUserPwd);
2323 :
2324 : // Set Proxy parameters.
2325 2601 : const char *pszProxy = CSLFetchNameValue(papszOptions, "PROXY");
2326 2601 : if (pszProxy == nullptr)
2327 2601 : pszProxy = CPLGetConfigOption("GDAL_HTTP_PROXY", nullptr);
2328 2601 : if (pszProxy)
2329 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_PROXY, pszProxy);
2330 :
2331 2601 : const char *pszHttpsProxy = CSLFetchNameValue(papszOptions, "HTTPS_PROXY");
2332 2601 : if (pszHttpsProxy == nullptr)
2333 2601 : pszHttpsProxy = CPLGetConfigOption("GDAL_HTTPS_PROXY", nullptr);
2334 2601 : if (pszHttpsProxy && (STARTS_WITH(pszURL, "https")))
2335 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_PROXY, pszHttpsProxy);
2336 :
2337 : const char *pszProxyUserPwd =
2338 2601 : CSLFetchNameValue(papszOptions, "PROXYUSERPWD");
2339 2601 : if (pszProxyUserPwd == nullptr)
2340 2601 : pszProxyUserPwd = CPLGetConfigOption("GDAL_HTTP_PROXYUSERPWD", nullptr);
2341 2601 : if (pszProxyUserPwd)
2342 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_PROXYUSERPWD,
2343 : pszProxyUserPwd);
2344 :
2345 : // Support control over PROXYAUTH.
2346 2601 : const char *pszProxyAuth = CSLFetchNameValue(papszOptions, "PROXYAUTH");
2347 2601 : if (pszProxyAuth == nullptr)
2348 2601 : pszProxyAuth = CPLGetConfigOption("GDAL_PROXY_AUTH", nullptr);
2349 2601 : if (pszProxyAuth == nullptr)
2350 : {
2351 : // Do nothing.
2352 : }
2353 0 : else if (EQUAL(pszProxyAuth, "BASIC"))
2354 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_PROXYAUTH,
2355 : CURLAUTH_BASIC);
2356 0 : else if (EQUAL(pszProxyAuth, "NTLM"))
2357 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_PROXYAUTH,
2358 : CURLAUTH_NTLM);
2359 0 : else if (EQUAL(pszProxyAuth, "DIGEST"))
2360 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_PROXYAUTH,
2361 : CURLAUTH_DIGEST);
2362 0 : else if (EQUAL(pszProxyAuth, "ANY"))
2363 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_PROXYAUTH,
2364 : CURLAUTH_ANY);
2365 0 : else if (EQUAL(pszProxyAuth, "ANYSAFE"))
2366 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_PROXYAUTH,
2367 : CURLAUTH_ANYSAFE);
2368 0 : else if (EQUAL(pszProxyAuth, "NEGOTIATE"))
2369 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_PROXYAUTH,
2370 : CURLAUTH_NEGOTIATE);
2371 : else
2372 : {
2373 0 : CPLError(CE_Warning, CPLE_AppDefined,
2374 : "Unsupported PROXYAUTH value '%s', ignored.", pszProxyAuth);
2375 : }
2376 :
2377 2601 : unchecked_curl_easy_setopt(http_handle, CURLOPT_SUPPRESS_CONNECT_HEADERS,
2378 : 1L);
2379 :
2380 2601 : unchecked_curl_easy_setopt(http_handle, CURLOPT_FOLLOWLOCATION, 1);
2381 2601 : const char *pszUnrestrictedAuth = CPLGetConfigOption(
2382 : "CPL_VSIL_CURL_AUTHORIZATION_HEADER_ALLOWED_IF_REDIRECT",
2383 : "IF_SAME_HOST");
2384 2619 : if (!EQUAL(pszUnrestrictedAuth, "IF_SAME_HOST") &&
2385 18 : CPLTestBool(pszUnrestrictedAuth))
2386 : {
2387 9 : unchecked_curl_easy_setopt(http_handle, CURLOPT_UNRESTRICTED_AUTH, 1);
2388 : }
2389 :
2390 2601 : unchecked_curl_easy_setopt(http_handle, CURLOPT_MAXREDIRS, 10);
2391 2601 : unchecked_curl_easy_setopt(http_handle, CURLOPT_POSTREDIR,
2392 : CURL_REDIR_POST_ALL);
2393 :
2394 : // Set connect timeout.
2395 : const char *pszConnectTimeout =
2396 2601 : CSLFetchNameValue(papszOptions, "CONNECTTIMEOUT");
2397 2601 : if (pszConnectTimeout == nullptr)
2398 : pszConnectTimeout =
2399 2601 : CPLGetConfigOption("GDAL_HTTP_CONNECTTIMEOUT", nullptr);
2400 2601 : if (pszConnectTimeout != nullptr)
2401 : {
2402 : // coverity[tainted_data]
2403 1 : unchecked_curl_easy_setopt(
2404 : http_handle, CURLOPT_CONNECTTIMEOUT_MS,
2405 : static_cast<int>(1000 * CPLAtof(pszConnectTimeout)));
2406 : }
2407 :
2408 : // Set timeout.
2409 2601 : const char *pszTimeout = CSLFetchNameValue(papszOptions, "TIMEOUT");
2410 2601 : if (pszTimeout == nullptr)
2411 2307 : pszTimeout = CPLGetConfigOption("GDAL_HTTP_TIMEOUT", nullptr);
2412 2601 : if (pszTimeout != nullptr)
2413 : {
2414 : // coverity[tainted_data]
2415 303 : unchecked_curl_easy_setopt(
2416 : http_handle, CURLOPT_TIMEOUT_MS,
2417 : static_cast<int>(1000 * CPLAtof(pszTimeout)));
2418 : }
2419 :
2420 : // Set low speed time and limit.
2421 : const char *pszLowSpeedTime =
2422 2601 : CSLFetchNameValue(papszOptions, "LOW_SPEED_TIME");
2423 2601 : if (pszLowSpeedTime == nullptr)
2424 : pszLowSpeedTime =
2425 2601 : CPLGetConfigOption("GDAL_HTTP_LOW_SPEED_TIME", nullptr);
2426 2601 : if (pszLowSpeedTime != nullptr)
2427 : {
2428 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_LOW_SPEED_TIME,
2429 : atoi(pszLowSpeedTime));
2430 :
2431 : const char *pszLowSpeedLimit =
2432 0 : CSLFetchNameValue(papszOptions, "LOW_SPEED_LIMIT");
2433 0 : if (pszLowSpeedLimit == nullptr)
2434 : pszLowSpeedLimit =
2435 0 : CPLGetConfigOption("GDAL_HTTP_LOW_SPEED_LIMIT", "1");
2436 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_LOW_SPEED_LIMIT,
2437 : atoi(pszLowSpeedLimit));
2438 : }
2439 :
2440 : /* Disable some SSL verification */
2441 2601 : const char *pszUnsafeSSL = CSLFetchNameValue(papszOptions, "UNSAFESSL");
2442 2601 : if (pszUnsafeSSL == nullptr)
2443 2578 : pszUnsafeSSL = CPLGetConfigOption("GDAL_HTTP_UNSAFESSL", nullptr);
2444 2601 : if (pszUnsafeSSL != nullptr && CPLTestBool(pszUnsafeSSL))
2445 : {
2446 23 : unchecked_curl_easy_setopt(http_handle, CURLOPT_SSL_VERIFYPEER, 0L);
2447 23 : unchecked_curl_easy_setopt(http_handle, CURLOPT_SSL_VERIFYHOST, 0L);
2448 : }
2449 :
2450 : const char *pszUseCAPIStore =
2451 2601 : CSLFetchNameValue(papszOptions, "USE_CAPI_STORE");
2452 2601 : if (pszUseCAPIStore == nullptr)
2453 2601 : pszUseCAPIStore = CPLGetConfigOption("GDAL_HTTP_USE_CAPI_STORE", "NO");
2454 2601 : if (CPLTestBool(pszUseCAPIStore))
2455 : {
2456 : #if defined(_WIN32) && defined(HAVE_OPENSSL_CRYPTO)
2457 : // Use certificates from Windows certificate store; requires
2458 : // crypt32.lib, OpenSSL crypto and ssl libraries.
2459 : unchecked_curl_easy_setopt(http_handle, CURLOPT_SSL_CTX_FUNCTION,
2460 : *CPL_ssl_ctx_callback);
2461 : #else // defined(_WIN32) && defined(HAVE_OPENSSL_CRYPTO)
2462 2 : CPLError(CE_Warning, CPLE_NotSupported,
2463 : "GDAL_HTTP_USE_CAPI_STORE requested, but libcurl too old, "
2464 : "non-Windows platform or OpenSSL missing.");
2465 : #endif // defined(_WIN32) && defined(HAVE_OPENSSL_CRYPTO)
2466 : }
2467 :
2468 : // Enable OCSP stapling if requested.
2469 : const char *pszSSLVerifyStatus =
2470 2601 : CSLFetchNameValue(papszOptions, "SSL_VERIFYSTATUS");
2471 2601 : if (pszSSLVerifyStatus == nullptr)
2472 : pszSSLVerifyStatus =
2473 2601 : CPLGetConfigOption("GDAL_HTTP_SSL_VERIFYSTATUS", "NO");
2474 2601 : if (CPLTestBool(pszSSLVerifyStatus))
2475 : {
2476 1 : unchecked_curl_easy_setopt(http_handle, CURLOPT_SSL_VERIFYSTATUS, 1L);
2477 : }
2478 :
2479 : // Custom path to SSL certificates.
2480 2601 : const char *pszCAInfo = CSLFetchNameValue(papszOptions, "CAINFO");
2481 2601 : if (pszCAInfo == nullptr)
2482 : // Name of GDAL environment variable for the CA Bundle path
2483 2601 : pszCAInfo = CPLGetConfigOption("GDAL_CURL_CA_BUNDLE", nullptr);
2484 2601 : if (pszCAInfo == nullptr)
2485 : // Name of environment variable used by the curl binary
2486 2601 : pszCAInfo = CPLGetConfigOption("CURL_CA_BUNDLE", nullptr);
2487 2601 : if (pszCAInfo == nullptr)
2488 : // Name of environment variable used by the curl binary (tested
2489 : // after CURL_CA_BUNDLE
2490 2601 : pszCAInfo = CPLGetConfigOption("SSL_CERT_FILE", nullptr);
2491 : #ifdef _WIN32
2492 : if (pszCAInfo == nullptr)
2493 : {
2494 : pszCAInfo = CPLFindWin32CurlCaBundleCrt();
2495 : }
2496 : #endif
2497 2601 : if (pszCAInfo != nullptr)
2498 : {
2499 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_CAINFO, pszCAInfo);
2500 : }
2501 :
2502 2601 : const char *pszCAPath = CSLFetchNameValue(papszOptions, "CAPATH");
2503 2601 : if (pszCAPath != nullptr)
2504 : {
2505 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_CAPATH, pszCAPath);
2506 : }
2507 :
2508 : // Support for SSL client certificates
2509 :
2510 : // Filename of the the client certificate
2511 2601 : const char *pszSSLCert = CSLFetchNameValue(papszOptions, "SSLCERT");
2512 2601 : if (!pszSSLCert)
2513 2601 : pszSSLCert = CPLGetConfigOption("GDAL_HTTP_SSLCERT", nullptr);
2514 2601 : if (pszSSLCert)
2515 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_SSLCERT, pszSSLCert);
2516 :
2517 : // private key file for TLS and SSL client cert
2518 2601 : const char *pszSSLKey = CSLFetchNameValue(papszOptions, "SSLKEY");
2519 2601 : if (!pszSSLKey)
2520 2601 : pszSSLKey = CPLGetConfigOption("GDAL_HTTP_SSLKEY", nullptr);
2521 2601 : if (pszSSLKey)
2522 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_SSLKEY, pszSSLKey);
2523 :
2524 : // type of client SSL certificate ("PEM", "DER", ...)
2525 2601 : const char *pszSSLCertType = CSLFetchNameValue(papszOptions, "SSLCERTTYPE");
2526 2601 : if (!pszSSLCertType)
2527 2601 : pszSSLCertType = CPLGetConfigOption("GDAL_HTTP_SSLCERTTYPE", nullptr);
2528 2601 : if (pszSSLCertType)
2529 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_SSLCERTTYPE,
2530 : pszSSLCertType);
2531 :
2532 : // passphrase to private key
2533 2601 : const char *pszKeyPasswd = CSLFetchNameValue(papszOptions, "KEYPASSWD");
2534 2601 : if (!pszKeyPasswd)
2535 2601 : pszKeyPasswd = CPLGetConfigOption("GDAL_HTTP_KEYPASSWD", nullptr);
2536 2601 : if (pszKeyPasswd)
2537 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_KEYPASSWD,
2538 : pszKeyPasswd);
2539 :
2540 : /* Set Referer */
2541 2601 : const char *pszReferer = CSLFetchNameValue(papszOptions, "REFERER");
2542 2601 : if (pszReferer != nullptr)
2543 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_REFERER, pszReferer);
2544 :
2545 : /* Set User-Agent */
2546 2601 : const char *pszUserAgent = CSLFetchNameValue(papszOptions, "USERAGENT");
2547 2601 : if (pszUserAgent == nullptr)
2548 2513 : pszUserAgent = CPLGetConfigOption("GDAL_HTTP_USERAGENT",
2549 : gosDefaultUserAgent.c_str());
2550 2601 : if (pszUserAgent != nullptr && !EQUAL(pszUserAgent, ""))
2551 : {
2552 2601 : unchecked_curl_easy_setopt(http_handle, CURLOPT_USERAGENT,
2553 : pszUserAgent);
2554 : }
2555 :
2556 : /* NOSIGNAL should be set to true for timeout to work in multithread
2557 : * environments on Unix, requires libcurl 7.10 or more recent.
2558 : * (this force avoiding the use of signal handlers)
2559 : */
2560 2601 : unchecked_curl_easy_setopt(http_handle, CURLOPT_NOSIGNAL, 1);
2561 :
2562 : const char *pszFormFilePath =
2563 2601 : CSLFetchNameValue(papszOptions, "FORM_FILE_PATH");
2564 : const char *pszParametersCount =
2565 2601 : CSLFetchNameValue(papszOptions, "FORM_ITEM_COUNT");
2566 2601 : if (pszFormFilePath == nullptr && pszParametersCount == nullptr)
2567 : {
2568 : /* Set POST mode */
2569 2599 : const char *pszPost = CSLFetchNameValue(papszOptions, "POSTFIELDS");
2570 2599 : if (pszPost != nullptr)
2571 : {
2572 159 : CPLDebug("HTTP", "These POSTFIELDS were sent:%.4000s", pszPost);
2573 159 : unchecked_curl_easy_setopt(http_handle, CURLOPT_POST, 1);
2574 159 : unchecked_curl_easy_setopt(http_handle, CURLOPT_POSTFIELDS,
2575 : pszPost);
2576 : }
2577 : }
2578 :
2579 : const char *pszCustomRequest =
2580 2601 : CSLFetchNameValue(papszOptions, "CUSTOMREQUEST");
2581 2601 : if (pszCustomRequest != nullptr)
2582 : {
2583 138 : unchecked_curl_easy_setopt(http_handle, CURLOPT_CUSTOMREQUEST,
2584 : pszCustomRequest);
2585 : }
2586 :
2587 2601 : const char *pszCookie = CSLFetchNameValue(papszOptions, "COOKIE");
2588 2601 : if (pszCookie == nullptr)
2589 2601 : pszCookie = CPLGetConfigOption("GDAL_HTTP_COOKIE", nullptr);
2590 2601 : if (pszCookie != nullptr)
2591 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_COOKIE, pszCookie);
2592 :
2593 2601 : const char *pszCookieFile = CSLFetchNameValue(papszOptions, "COOKIEFILE");
2594 2601 : if (pszCookieFile == nullptr)
2595 2601 : pszCookieFile = CPLGetConfigOption("GDAL_HTTP_COOKIEFILE", nullptr);
2596 2601 : if (pszCookieFile != nullptr)
2597 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_COOKIEFILE,
2598 : pszCookieFile);
2599 :
2600 2601 : const char *pszCookieJar = CSLFetchNameValue(papszOptions, "COOKIEJAR");
2601 2601 : if (pszCookieJar == nullptr)
2602 2601 : pszCookieJar = CPLGetConfigOption("GDAL_HTTP_COOKIEJAR", nullptr);
2603 2601 : if (pszCookieJar != nullptr)
2604 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_COOKIEJAR,
2605 : pszCookieJar);
2606 :
2607 : // TCP keep-alive
2608 : const char *pszTCPKeepAlive =
2609 2601 : CSLFetchNameValue(papszOptions, "TCP_KEEPALIVE");
2610 2601 : if (pszTCPKeepAlive == nullptr)
2611 2601 : pszTCPKeepAlive = CPLGetConfigOption("GDAL_HTTP_TCP_KEEPALIVE", "YES");
2612 2601 : if (pszTCPKeepAlive != nullptr && CPLTestBool(pszTCPKeepAlive))
2613 : {
2614 : // Set keep-alive interval.
2615 2601 : int nKeepAliveInterval = 60;
2616 : const char *pszKeepAliveInterval =
2617 2601 : CSLFetchNameValue(papszOptions, "TCP_KEEPINTVL");
2618 2601 : if (pszKeepAliveInterval == nullptr)
2619 : pszKeepAliveInterval =
2620 2601 : CPLGetConfigOption("GDAL_HTTP_TCP_KEEPINTVL", nullptr);
2621 2601 : if (pszKeepAliveInterval != nullptr)
2622 1 : nKeepAliveInterval = atoi(pszKeepAliveInterval);
2623 :
2624 : // Set keep-alive idle wait time.
2625 2601 : int nKeepAliveIdle = 60;
2626 : const char *pszKeepAliveIdle =
2627 2601 : CSLFetchNameValue(papszOptions, "TCP_KEEPIDLE");
2628 2601 : if (pszKeepAliveIdle == nullptr)
2629 : pszKeepAliveIdle =
2630 2601 : CPLGetConfigOption("GDAL_HTTP_TCP_KEEPIDLE", nullptr);
2631 2601 : if (pszKeepAliveIdle != nullptr)
2632 1 : nKeepAliveIdle = atoi(pszKeepAliveIdle);
2633 :
2634 2601 : unchecked_curl_easy_setopt(http_handle, CURLOPT_TCP_KEEPALIVE, 1L);
2635 2601 : unchecked_curl_easy_setopt(http_handle, CURLOPT_TCP_KEEPINTVL,
2636 : nKeepAliveInterval);
2637 2601 : unchecked_curl_easy_setopt(http_handle, CURLOPT_TCP_KEEPIDLE,
2638 : nKeepAliveIdle);
2639 : }
2640 :
2641 2601 : struct curl_slist *headers = nullptr;
2642 2601 : const char *pszAccept = CSLFetchNameValue(papszOptions, "ACCEPT");
2643 2601 : if (pszAccept)
2644 : {
2645 4 : headers =
2646 4 : curl_slist_append(headers, CPLSPrintf("Accept: %s", pszAccept));
2647 : }
2648 :
2649 1694 : const auto AddHeader = [&headers, pszAccept](const char *pszHeader)
2650 : {
2651 676 : if (STARTS_WITH_CI(pszHeader, "Accept:") && pszAccept)
2652 : {
2653 1 : const char *pszVal = pszHeader + strlen("Accept:");
2654 2 : while (*pszVal == ' ')
2655 1 : ++pszVal;
2656 1 : if (!EQUAL(pszVal, pszAccept))
2657 : {
2658 : // Cf https://github.com/OSGeo/gdal/issues/7691#issuecomment-2873711603
2659 1 : CPLDebug(
2660 : "HTTP",
2661 : "Ignoring '%s' since ACCEPT option = '%s' is specified",
2662 : pszHeader, pszAccept);
2663 1 : }
2664 : }
2665 : else
2666 : {
2667 675 : headers = curl_slist_append(headers, pszHeader);
2668 : }
2669 3277 : };
2670 :
2671 2601 : const char *pszHeaderFile = CSLFetchNameValue(papszOptions, "HEADER_FILE");
2672 2601 : if (pszHeaderFile == nullptr)
2673 2599 : pszHeaderFile = CPLGetConfigOption("GDAL_HTTP_HEADER_FILE", nullptr);
2674 2601 : if (pszHeaderFile != nullptr)
2675 : {
2676 2 : VSILFILE *fp = nullptr;
2677 : // Do not allow /vsicurl/ access from /vsicurl because of
2678 : // GetCurlHandleFor() e.g. "/vsicurl/,HEADER_FILE=/vsicurl/,url= " would
2679 : // cause use of memory after free
2680 2 : if (!STARTS_WITH(pszHeaderFile, "/vsi") ||
2681 2 : STARTS_WITH(pszHeaderFile, "/vsimem/"))
2682 : {
2683 2 : fp = VSIFOpenL(pszHeaderFile, "rb");
2684 : }
2685 2 : if (fp == nullptr)
2686 : {
2687 0 : CPLError(CE_Failure, CPLE_FileIO, "Cannot read %s", pszHeaderFile);
2688 : }
2689 : else
2690 : {
2691 2 : const char *pszLine = nullptr;
2692 4 : while ((pszLine = CPLReadLineL(fp)) != nullptr)
2693 : {
2694 2 : AddHeader(pszLine);
2695 : }
2696 2 : VSIFCloseL(fp);
2697 : }
2698 : }
2699 :
2700 2601 : const char *pszHeaders = CSLFetchNameValue(papszOptions, "HEADERS");
2701 2601 : if (pszHeaders == nullptr)
2702 1999 : pszHeaders = CPLGetConfigOption("GDAL_HTTP_HEADERS", nullptr);
2703 2601 : if (pszHeaders)
2704 : {
2705 611 : bool bHeadersDone = false;
2706 : // Compatibility hack for "HEADERS=Accept: text/plain, application/json"
2707 611 : if (strstr(pszHeaders, "\r\n") == nullptr)
2708 : {
2709 552 : const char *pszComma = strchr(pszHeaders, ',');
2710 552 : if (pszComma != nullptr && strchr(pszComma, ':') == nullptr)
2711 : {
2712 211 : AddHeader(pszHeaders);
2713 211 : bHeadersDone = true;
2714 : }
2715 : }
2716 611 : if (!bHeadersDone)
2717 : {
2718 400 : const char *pszAuthorizationHeaderAllowed = CSLFetchNameValueDef(
2719 : papszOptions, "AUTHORIZATION_HEADER_ALLOWED", "YES");
2720 : const bool bAuthorizationHeaderAllowed =
2721 400 : CPLTestBool(pszAuthorizationHeaderAllowed);
2722 :
2723 : // We accept both raw headers with \r\n as a separator, or as
2724 : // a comma separated list of foo: bar values.
2725 : const CPLStringList aosTokens(
2726 400 : strstr(pszHeaders, "\r\n")
2727 59 : ? CSLTokenizeString2(pszHeaders, "\r\n", 0)
2728 859 : : CSLTokenizeString2(pszHeaders, ",", CSLT_HONOURSTRINGS));
2729 867 : for (int i = 0; i < aosTokens.size(); ++i)
2730 : {
2731 471 : if (bAuthorizationHeaderAllowed ||
2732 4 : !STARTS_WITH_CI(aosTokens[i], "Authorization:"))
2733 : {
2734 463 : AddHeader(aosTokens[i]);
2735 : }
2736 : }
2737 : }
2738 : }
2739 :
2740 2601 : return headers;
2741 : }
2742 :
2743 : /************************************************************************/
2744 : /* CPLHTTPIgnoreSigPipe() */
2745 : /************************************************************************/
2746 :
2747 : /* If using OpenSSL with Curl, openssl can cause SIGPIPE to be triggered */
2748 : /* As we set CURLOPT_NOSIGNAL = 1, we must manually handle this situation */
2749 :
2750 2819 : void *CPLHTTPIgnoreSigPipe()
2751 : {
2752 : #if defined(SIGPIPE) && defined(HAVE_SIGACTION)
2753 : struct sigaction old_pipe_act;
2754 : struct sigaction action;
2755 : /* Get previous handler */
2756 2819 : memset(&old_pipe_act, 0, sizeof(struct sigaction));
2757 2819 : sigaction(SIGPIPE, nullptr, &old_pipe_act);
2758 :
2759 : /* Install new handler */
2760 2819 : action = old_pipe_act;
2761 2819 : action.sa_handler = SIG_IGN;
2762 2819 : sigaction(SIGPIPE, &action, nullptr);
2763 :
2764 2819 : void *ret = CPLMalloc(sizeof(old_pipe_act));
2765 2819 : memcpy(ret, &old_pipe_act, sizeof(old_pipe_act));
2766 2819 : return ret;
2767 : #else
2768 : return nullptr;
2769 : #endif
2770 : }
2771 :
2772 : /************************************************************************/
2773 : /* CPLHTTPRestoreSigPipeHandler() */
2774 : /************************************************************************/
2775 :
2776 2819 : void CPLHTTPRestoreSigPipeHandler(void *old_handler)
2777 : {
2778 : #if defined(SIGPIPE) && defined(HAVE_SIGACTION)
2779 2819 : sigaction(SIGPIPE, static_cast<struct sigaction *>(old_handler), nullptr);
2780 2819 : CPLFree(old_handler);
2781 : #else
2782 : (void)old_handler;
2783 : #endif
2784 2819 : }
2785 :
2786 : #endif // def HAVE_CURL
2787 :
2788 : /************************************************************************/
2789 : /* CPLHTTPEnabled() */
2790 : /************************************************************************/
2791 :
2792 : /**
2793 : * \brief Return if CPLHTTP services can be useful
2794 : *
2795 : * Those services depend on GDAL being build with libcurl support.
2796 : *
2797 : * @return TRUE if libcurl support is enabled
2798 : */
2799 9 : int CPLHTTPEnabled()
2800 :
2801 : {
2802 : #ifdef HAVE_CURL
2803 9 : return TRUE;
2804 : #else
2805 : return FALSE;
2806 : #endif
2807 : }
2808 :
2809 : /************************************************************************/
2810 : /* CPLHTTPCleanup() */
2811 : /************************************************************************/
2812 :
2813 : /**
2814 : * \brief Cleanup function to call at application termination
2815 : */
2816 1121 : void CPLHTTPCleanup()
2817 :
2818 : {
2819 : #ifdef HAVE_CURL
2820 1121 : if (!hSessionMapMutex)
2821 1120 : return;
2822 :
2823 : {
2824 2 : CPLMutexHolder oHolder(&hSessionMapMutex);
2825 1 : if (poSessionMap)
2826 : {
2827 0 : for (auto &kv : *poSessionMap)
2828 : {
2829 0 : curl_easy_cleanup(kv.second);
2830 : }
2831 0 : delete poSessionMap;
2832 0 : poSessionMap = nullptr;
2833 : }
2834 1 : if (poSessionMultiMap)
2835 : {
2836 0 : for (auto &kv : *poSessionMultiMap)
2837 : {
2838 0 : VSICURLMultiCleanup(kv.second);
2839 : }
2840 0 : delete poSessionMultiMap;
2841 0 : poSessionMultiMap = nullptr;
2842 : }
2843 : }
2844 :
2845 : // Not quite a safe sequence.
2846 1 : CPLDestroyMutex(hSessionMapMutex);
2847 1 : hSessionMapMutex = nullptr;
2848 :
2849 : #if defined(_WIN32) && defined(HAVE_OPENSSL_CRYPTO)
2850 : // This cleanup must be absolutely done before CPLOpenSSLCleanup()
2851 : // for some unknown reason, but otherwise X509_free() in
2852 : // CPLWindowsCertificateListCleanup() will crash.
2853 : CPLWindowsCertificateListCleanup();
2854 : #endif
2855 :
2856 : #if defined(HAVE_OPENSSL_CRYPTO) && OPENSSL_VERSION_NUMBER < 0x10100000
2857 : CPLOpenSSLCleanup();
2858 : #endif
2859 :
2860 : #endif
2861 : }
2862 :
2863 : /************************************************************************/
2864 : /* CPLHTTPDestroyResult() */
2865 : /************************************************************************/
2866 :
2867 : /**
2868 : * \brief Clean the memory associated with the return value of CPLHTTPFetch()
2869 : *
2870 : * @param psResult pointer to the return value of CPLHTTPFetch()
2871 : */
2872 2203 : void CPLHTTPDestroyResult(CPLHTTPResult *psResult)
2873 :
2874 : {
2875 2203 : if (psResult)
2876 : {
2877 2015 : CPLFree(psResult->pabyData);
2878 2015 : CPLFree(psResult->pszErrBuf);
2879 2015 : CPLFree(psResult->pszContentType);
2880 2015 : CSLDestroy(psResult->papszHeaders);
2881 :
2882 2072 : for (int i = 0; i < psResult->nMimePartCount; i++)
2883 : {
2884 57 : CSLDestroy(psResult->pasMimePart[i].papszHeaders);
2885 : }
2886 2015 : CPLFree(psResult->pasMimePart);
2887 :
2888 2015 : CPLFree(psResult);
2889 : }
2890 2203 : }
2891 :
2892 : /************************************************************************/
2893 : /* CPLHTTPParseMultipartMime() */
2894 : /************************************************************************/
2895 :
2896 : /**
2897 : * \brief Parses a MIME multipart message.
2898 : *
2899 : * This function will iterate over each part and put it in a separate
2900 : * element of the pasMimePart array of the provided psResult structure.
2901 : *
2902 : * @param psResult pointer to the return value of CPLHTTPFetch()
2903 : * @return TRUE if the message contains MIME multipart message.
2904 : */
2905 45 : int CPLHTTPParseMultipartMime(CPLHTTPResult *psResult)
2906 :
2907 : {
2908 : /* -------------------------------------------------------------------- */
2909 : /* Is it already done? */
2910 : /* -------------------------------------------------------------------- */
2911 45 : if (psResult->nMimePartCount > 0)
2912 5 : return TRUE;
2913 :
2914 : /* -------------------------------------------------------------------- */
2915 : /* Find the boundary setting in the content type. */
2916 : /* -------------------------------------------------------------------- */
2917 40 : const char *pszBound = nullptr;
2918 :
2919 40 : if (psResult->pszContentType != nullptr)
2920 38 : pszBound = strstr(psResult->pszContentType, "boundary=");
2921 :
2922 40 : if (pszBound == nullptr)
2923 : {
2924 2 : CPLError(CE_Failure, CPLE_AppDefined,
2925 : "Unable to parse multi-part mime, no boundary setting.");
2926 2 : return FALSE;
2927 : }
2928 :
2929 76 : CPLString osBoundary;
2930 : char **papszTokens =
2931 38 : CSLTokenizeStringComplex(pszBound + 9, "\n ;", TRUE, FALSE);
2932 :
2933 38 : if (CSLCount(papszTokens) == 0 || strlen(papszTokens[0]) == 0)
2934 : {
2935 1 : CPLError(CE_Failure, CPLE_AppDefined,
2936 : "Unable to parse multi-part mime, boundary not parsable.");
2937 1 : CSLDestroy(papszTokens);
2938 1 : return FALSE;
2939 : }
2940 :
2941 37 : osBoundary = "--";
2942 37 : osBoundary += papszTokens[0];
2943 37 : CSLDestroy(papszTokens);
2944 :
2945 : /* -------------------------------------------------------------------- */
2946 : /* Find the start of the first chunk. */
2947 : /* -------------------------------------------------------------------- */
2948 37 : char *pszNext = psResult->pabyData
2949 37 : ? strstr(reinterpret_cast<char *>(psResult->pabyData),
2950 : osBoundary.c_str())
2951 37 : : nullptr;
2952 :
2953 37 : if (pszNext == nullptr)
2954 : {
2955 1 : CPLError(CE_Failure, CPLE_AppDefined, "No parts found.");
2956 1 : return FALSE;
2957 : }
2958 :
2959 36 : pszNext += osBoundary.size();
2960 146 : while (*pszNext != '\n' && *pszNext != '\r' && *pszNext != '\0')
2961 110 : pszNext++;
2962 36 : if (*pszNext == '\r')
2963 30 : pszNext++;
2964 36 : if (*pszNext == '\n')
2965 36 : pszNext++;
2966 :
2967 : /* -------------------------------------------------------------------- */
2968 : /* Loop over parts... */
2969 : /* -------------------------------------------------------------------- */
2970 : while (true)
2971 : {
2972 57 : psResult->nMimePartCount++;
2973 57 : psResult->pasMimePart = static_cast<CPLMimePart *>(
2974 114 : CPLRealloc(psResult->pasMimePart,
2975 57 : sizeof(CPLMimePart) * psResult->nMimePartCount));
2976 :
2977 57 : CPLMimePart *psPart =
2978 57 : psResult->pasMimePart + psResult->nMimePartCount - 1;
2979 :
2980 57 : memset(psPart, 0, sizeof(CPLMimePart));
2981 :
2982 : /* --------------------------------------------------------------------
2983 : */
2984 : /* Collect headers. */
2985 : /* --------------------------------------------------------------------
2986 : */
2987 145 : while (*pszNext != '\n' && *pszNext != '\r' && *pszNext != '\0')
2988 : {
2989 90 : if (!STARTS_WITH(pszNext, "Content-"))
2990 : {
2991 1 : break;
2992 : }
2993 89 : char *pszEOL = strstr(pszNext, "\n");
2994 :
2995 89 : if (pszEOL == nullptr)
2996 : {
2997 1 : CPLError(CE_Failure, CPLE_AppDefined,
2998 : "Error while parsing multipart content (at line %d)",
2999 : __LINE__);
3000 1 : return FALSE;
3001 : }
3002 :
3003 88 : *pszEOL = '\0';
3004 88 : bool bRestoreAntislashR = false;
3005 88 : if (pszEOL - pszNext > 1 && pszEOL[-1] == '\r')
3006 : {
3007 80 : bRestoreAntislashR = true;
3008 80 : pszEOL[-1] = '\0';
3009 : }
3010 88 : char *pszKey = nullptr;
3011 88 : const char *pszValue = CPLParseNameValue(pszNext, &pszKey);
3012 88 : if (pszKey && pszValue)
3013 : {
3014 88 : psPart->papszHeaders =
3015 88 : CSLSetNameValue(psPart->papszHeaders, pszKey, pszValue);
3016 : }
3017 88 : CPLFree(pszKey);
3018 88 : if (bRestoreAntislashR)
3019 80 : pszEOL[-1] = '\r';
3020 88 : *pszEOL = '\n';
3021 :
3022 88 : pszNext = pszEOL + 1;
3023 : }
3024 :
3025 56 : if (*pszNext == '\r')
3026 54 : pszNext++;
3027 56 : if (*pszNext == '\n')
3028 54 : pszNext++;
3029 :
3030 : /* --------------------------------------------------------------------
3031 : */
3032 : /* Work out the data block size. */
3033 : /* --------------------------------------------------------------------
3034 : */
3035 56 : psPart->pabyData = reinterpret_cast<GByte *>(pszNext);
3036 :
3037 56 : int nBytesAvail = psResult->nDataLen -
3038 56 : static_cast<int>(pszNext - reinterpret_cast<char *>(
3039 56 : psResult->pabyData));
3040 :
3041 6579040 : while (nBytesAvail > 0 &&
3042 3289520 : (*pszNext != '-' ||
3043 109 : strncmp(pszNext, osBoundary, osBoundary.size()) != 0))
3044 : {
3045 3289470 : pszNext++;
3046 3289470 : nBytesAvail--;
3047 : }
3048 :
3049 56 : if (nBytesAvail == 0)
3050 : {
3051 2 : CPLError(CE_Failure, CPLE_AppDefined,
3052 : "Error while parsing multipart content (at line %d)",
3053 : __LINE__);
3054 2 : return FALSE;
3055 : }
3056 :
3057 54 : psPart->nDataLen = static_cast<int>(
3058 54 : pszNext - reinterpret_cast<char *>(psPart->pabyData));
3059 : // Normally the part should end with "\r\n--boundary_marker"
3060 54 : if (psPart->nDataLen >= 2 && pszNext[-2] == '\r' && pszNext[-1] == '\n')
3061 : {
3062 46 : psPart->nDataLen -= 2;
3063 : }
3064 :
3065 54 : pszNext += osBoundary.size();
3066 :
3067 54 : if (STARTS_WITH(pszNext, "--"))
3068 : {
3069 31 : break;
3070 : }
3071 :
3072 23 : if (*pszNext == '\r')
3073 19 : pszNext++;
3074 23 : if (*pszNext == '\n')
3075 21 : pszNext++;
3076 : else
3077 : {
3078 2 : CPLError(CE_Failure, CPLE_AppDefined,
3079 : "Error while parsing multipart content (at line %d)",
3080 : __LINE__);
3081 2 : return FALSE;
3082 : }
3083 21 : }
3084 :
3085 31 : return TRUE;
3086 : }
3087 :
3088 : #if defined(__GNUC__)
3089 : #pragma GCC diagnostic pop
3090 : #endif
|
