Line data    Source code

       1             : /******************************************************************************
       2             :  *
       3             :  * Project:  WMS Client Driver
       4             :  * Purpose:  Implementation of Dataset and RasterBand classes for WMS
       5             :  *           and other similar services.
       6             :  * Author:   Adam Nowacki, nowak@xpam.de
       7             :  *
       8             :  ******************************************************************************
       9             :  * Copyright (c) 2007, Adam Nowacki
      10             :  * Copyright (c) 2007-2013, Even Rouault <even dot rouault at spatialys.com>
      11             :  * Copyright (c) 2016, Lucian Plesea
      12             :  *
      13             :  * SPDX-License-Identifier: MIT
      14             :  ****************************************************************************/
      15             : 
      16             : #include "wmsdriver.h"
      17             : #include <algorithm>
      18             : 
      19          57 : static size_t WriteFunc(void *buffer, size_t count, size_t nmemb, void *req)
      20             : {
      21          57 :     WMSHTTPRequest *psRequest = reinterpret_cast<WMSHTTPRequest *>(req);
      22          57 :     size_t size = count * nmemb;
      23             : 
      24          57 :     if (size == 0)
      25           0 :         return 0;
      26             : 
      27          57 :     const size_t required_size = psRequest->nDataLen + size + 1;
      28          57 :     if (required_size > psRequest->nDataAlloc)
      29             :     {
      30          39 :         size_t new_size = required_size * 2;
      31          39 :         if (new_size < 512)
      32           0 :             new_size = 512;
      33          39 :         psRequest->nDataAlloc = new_size;
      34             :         GByte *pabyNewData = reinterpret_cast<GByte *>(
      35          39 :             VSIRealloc(psRequest->pabyData, new_size));
      36          39 :         if (pabyNewData == nullptr)
      37             :         {
      38           0 :             VSIFree(psRequest->pabyData);
      39           0 :             psRequest->pabyData = nullptr;
      40             :             psRequest->Error.Printf(
      41             :                 "Out of memory allocating %u bytes for HTTP data buffer.",
      42           0 :                 static_cast<unsigned int>(new_size));
      43           0 :             psRequest->nDataAlloc = 0;
      44           0 :             psRequest->nDataLen = 0;
      45           0 :             return 0;
      46             :         }
      47          39 :         psRequest->pabyData = pabyNewData;
      48             :     }
      49          57 :     memcpy(psRequest->pabyData + psRequest->nDataLen, buffer, size);
      50          57 :     psRequest->nDataLen += size;
      51          57 :     psRequest->pabyData[psRequest->nDataLen] = 0;
      52          57 :     return nmemb;
      53             : }
      54             : 
      55             : // Process curl errors
      56          34 : static void ProcessCurlErrors(CURLMsg *msg, WMSHTTPRequest *pasRequest,
      57             :                               int nRequestCount)
      58             : {
      59          34 :     CPLAssert(msg != nullptr);
      60          34 :     CPLAssert(msg->msg == CURLMSG_DONE);
      61             : 
      62             :     // in case of local file error: update status code
      63          34 :     if (msg->data.result == CURLE_FILE_COULDNT_READ_FILE)
      64             :     {
      65             :         // identify current request
      66           1 :         for (int current_req_i = 0; current_req_i < nRequestCount;
      67             :              ++current_req_i)
      68             :         {
      69           1 :             WMSHTTPRequest *const psRequest = &pasRequest[current_req_i];
      70           1 :             if (psRequest->m_curl_handle != msg->easy_handle)
      71           0 :                 continue;
      72             : 
      73             :             // sanity check for local files
      74           1 :             if (STARTS_WITH(psRequest->URL.c_str(), "file://"))
      75             :             {
      76           1 :                 psRequest->nStatus = 404;
      77           1 :                 break;
      78             :             }
      79             :         }
      80             :     }
      81          34 : }
      82             : 
      83             : // Builds a curl request
      84          44 : void WMSHTTPInitializeRequest(WMSHTTPRequest *psRequest)
      85             : {
      86          44 :     psRequest->nStatus = 0;
      87          44 :     psRequest->pabyData = nullptr;
      88          44 :     psRequest->nDataLen = 0;
      89          44 :     psRequest->nDataAlloc = 0;
      90             : 
      91          44 :     psRequest->m_curl_handle = curl_easy_init();
      92          44 :     if (psRequest->m_curl_handle == nullptr)
      93             :     {
      94           0 :         CPLError(CE_Failure, CPLE_AppDefined,
      95             :                  "CPLHTTPInitializeRequest(): Unable to create CURL handle.");
      96           0 :         return;
      97             :     }
      98             : 
      99          44 :     if (!psRequest->Range.empty())
     100             :     {
     101           0 :         CPL_IGNORE_RET_VAL(curl_easy_setopt(
     102             :             psRequest->m_curl_handle, CURLOPT_RANGE, psRequest->Range.c_str()));
     103             :     }
     104             : 
     105          44 :     CPL_IGNORE_RET_VAL(curl_easy_setopt(psRequest->m_curl_handle,
     106             :                                         CURLOPT_WRITEDATA, psRequest));
     107          44 :     CPL_IGNORE_RET_VAL(curl_easy_setopt(psRequest->m_curl_handle,
     108             :                                         CURLOPT_WRITEFUNCTION, WriteFunc));
     109             : 
     110          44 :     psRequest->m_curl_error.resize(CURL_ERROR_SIZE + 1);
     111          44 :     CPL_IGNORE_RET_VAL(curl_easy_setopt(psRequest->m_curl_handle,
     112             :                                         CURLOPT_ERRORBUFFER,
     113             :                                         &psRequest->m_curl_error[0]));
     114             : 
     115          44 :     psRequest->m_headers = static_cast<struct curl_slist *>(CPLHTTPSetOptions(
     116             :         psRequest->m_curl_handle, psRequest->URL.c_str(), psRequest->options));
     117          44 :     const char *pszAccept = CSLFetchNameValue(psRequest->options, "ACCEPT");
     118          44 :     if (pszAccept)
     119             :     {
     120           2 :         psRequest->m_headers = curl_slist_append(
     121             :             psRequest->m_headers, CPLSPrintf("Accept: %s", pszAccept));
     122             :     }
     123          44 :     if (psRequest->m_headers != nullptr)
     124             :     {
     125           2 :         CPL_IGNORE_RET_VAL(curl_easy_setopt(psRequest->m_curl_handle,
     126             :                                             CURLOPT_HTTPHEADER,
     127             :                                             psRequest->m_headers));
     128             :     }
     129             : }
     130             : 
     131          56 : WMSHTTPRequest::~WMSHTTPRequest()
     132             : {
     133          56 :     if (m_curl_handle != nullptr)
     134          44 :         curl_easy_cleanup(m_curl_handle);
     135          56 :     if (m_headers != nullptr)
     136           2 :         curl_slist_free_all(m_headers);
     137          56 :     if (pabyData != nullptr)
     138          38 :         CPLFree(pabyData);
     139          56 : }
     140             : 
     141             : //
     142             : // Like CPLHTTPFetch, but multiple requests in parallel
     143             : // By default it uses 5 connections
     144             : //
     145          36 : CPLErr WMSHTTPFetchMulti(WMSHTTPRequest *pasRequest, int nRequestCount)
     146             : {
     147          36 :     CPLErr ret = CE_None;
     148          36 :     CURLM *curl_multi = nullptr;
     149             :     int max_conn;
     150             :     int i, conn_i;
     151             : 
     152          36 :     CPLAssert(nRequestCount >= 0);
     153          36 :     if (nRequestCount == 0)
     154           5 :         return CE_None;
     155             : 
     156             :     const char *max_conn_opt =
     157          31 :         CSLFetchNameValue(const_cast<char **>(pasRequest->options), "MAXCONN");
     158          31 :     max_conn =
     159          31 :         (max_conn_opt == nullptr) ? 5 : MAX(1, MIN(atoi(max_conn_opt), 1000));
     160             : 
     161             :     // If the first url starts with vsimem, assume all do and defer to
     162             :     // CPLHTTPFetch
     163          41 :     if (STARTS_WITH(pasRequest[0].URL.c_str(), "/vsimem/") &&
     164             :         /* Disabled by default for potential security issues */
     165          10 :         CPLTestBool(CPLGetConfigOption("CPL_CURL_ENABLE_VSIMEM", "FALSE")))
     166             :     {
     167          20 :         for (i = 0; i < nRequestCount; i++)
     168             :         {
     169             :             CPLHTTPResult *psResult =
     170          10 :                 CPLHTTPFetch(pasRequest[i].URL.c_str(),
     171          10 :                              const_cast<char **>(pasRequest[i].options));
     172          10 :             pasRequest[i].pabyData = psResult->pabyData;
     173          10 :             pasRequest[i].nDataLen = psResult->nDataLen;
     174          10 :             pasRequest[i].Error =
     175          10 :                 psResult->pszErrBuf ? psResult->pszErrBuf : "";
     176             :             // Conventions are different between this module and cpl_http...
     177          10 :             if (psResult->pszErrBuf != nullptr &&
     178           1 :                 strcmp(psResult->pszErrBuf, "HTTP error code : 404") == 0)
     179           1 :                 pasRequest[i].nStatus = 404;
     180             :             else
     181           9 :                 pasRequest[i].nStatus = 200;
     182          10 :             pasRequest[i].ContentType =
     183          10 :                 psResult->pszContentType ? psResult->pszContentType : "";
     184             :             // took ownership of content, we're done with the rest
     185          10 :             psResult->pabyData = nullptr;
     186          10 :             psResult->nDataLen = 0;
     187          10 :             CPLHTTPDestroyResult(psResult);
     188             :         }
     189          10 :         return CE_None;
     190             :     }
     191             : 
     192          21 :     curl_multi = curl_multi_init();
     193          21 :     if (curl_multi == nullptr)
     194             :     {
     195           0 :         CPLError(CE_Fatal, CPLE_AppDefined,
     196             :                  "CPLHTTPFetchMulti(): Unable to create CURL multi-handle.");
     197             :     }
     198             : 
     199             :     // add at most max_conn requests
     200          21 :     int torun = std::min(nRequestCount, max_conn);
     201          51 :     for (conn_i = 0; conn_i < torun; ++conn_i)
     202             :     {
     203          30 :         WMSHTTPRequest *const psRequest = &pasRequest[conn_i];
     204          30 :         CPLDebug("HTTP", "Requesting [%d/%d] %s", conn_i + 1, nRequestCount,
     205          30 :                  pasRequest[conn_i].URL.c_str());
     206          30 :         curl_multi_add_handle(curl_multi, psRequest->m_curl_handle);
     207             :     }
     208             : 
     209          21 :     void *old_handler = CPLHTTPIgnoreSigPipe();
     210             :     int still_running;
     211         327 :     do
     212             :     {
     213             :         CURLMcode mc;
     214           0 :         do
     215             :         {
     216         348 :             mc = curl_multi_perform(curl_multi, &still_running);
     217         348 :         } while (CURLM_CALL_MULTI_PERFORM == mc);
     218             : 
     219             :         // Pick up messages, clean up the completed ones, add more
     220         348 :         int msgs_in_queue = 0;
     221           0 :         do
     222             :         {
     223         348 :             CURLMsg *m = curl_multi_info_read(curl_multi, &msgs_in_queue);
     224         348 :             if (m && (m->msg == CURLMSG_DONE))
     225             :             {
     226          34 :                 ProcessCurlErrors(m, pasRequest, nRequestCount);
     227             : 
     228          34 :                 curl_multi_remove_handle(curl_multi, m->easy_handle);
     229          34 :                 if (conn_i < nRequestCount)
     230             :                 {
     231           4 :                     auto psRequest = &pasRequest[conn_i];
     232           4 :                     CPLDebug("HTTP", "Requesting [%d/%d] %s", conn_i + 1,
     233           4 :                              nRequestCount, pasRequest[conn_i].URL.c_str());
     234           4 :                     curl_multi_add_handle(curl_multi, psRequest->m_curl_handle);
     235           4 :                     ++conn_i;
     236           4 :                     still_running = 1;  // Still have request pending
     237             :                 }
     238             :             }
     239         348 :         } while (msgs_in_queue);
     240             : 
     241         348 :         if (CURLM_OK == mc)
     242             :         {
     243             :             int numfds;
     244         348 :             curl_multi_wait(curl_multi, nullptr, 0, 100, &numfds);
     245             :         }
     246         348 :     } while (still_running || conn_i != nRequestCount);
     247             : 
     248             :     // process any message still in queue
     249             :     CURLMsg *msg;
     250             :     int msgs_in_queue;
     251           0 :     do
     252             :     {
     253          21 :         msg = curl_multi_info_read(curl_multi, &msgs_in_queue);
     254          21 :         if (msg != nullptr)
     255             :         {
     256           0 :             if (msg->msg == CURLMSG_DONE)
     257             :             {
     258           0 :                 ProcessCurlErrors(msg, pasRequest, nRequestCount);
     259             :             }
     260             :         }
     261          21 :     } while (msg != nullptr);
     262             : 
     263          21 :     CPLHTTPRestoreSigPipeHandler(old_handler);
     264             : 
     265          21 :     if (conn_i != nRequestCount)
     266             :     {  // something gone really really wrong
     267             :         // oddly built libcurl or perhaps absence of network interface
     268           0 :         CPLError(CE_Failure, CPLE_AppDefined,
     269             :                  "WMSHTTPFetchMulti(): conn_i != nRequestCount, this should "
     270             :                  "never happen ...");
     271           0 :         nRequestCount = conn_i;
     272           0 :         ret = CE_Failure;
     273             :     }
     274             : 
     275          55 :     for (i = 0; i < nRequestCount; ++i)
     276             :     {
     277          34 :         WMSHTTPRequest *const psRequest = &pasRequest[i];
     278             : 
     279             :         long response_code;
     280          34 :         curl_easy_getinfo(psRequest->m_curl_handle, CURLINFO_RESPONSE_CODE,
     281             :                           &response_code);
     282             :         // for local files, don't update the status code if one is already set
     283          35 :         if (!(psRequest->nStatus != 0 &&
     284           1 :               STARTS_WITH(psRequest->URL.c_str(), "file://")))
     285          33 :             psRequest->nStatus = static_cast<int>(response_code);
     286             : 
     287          34 :         char *content_type = nullptr;
     288          34 :         curl_easy_getinfo(psRequest->m_curl_handle, CURLINFO_CONTENT_TYPE,
     289             :                           &content_type);
     290          34 :         psRequest->ContentType = content_type ? content_type : "";
     291             : 
     292          34 :         if (psRequest->Error.empty())
     293          34 :             psRequest->Error = &psRequest->m_curl_error[0];
     294             : 
     295             :         /* In the case of a file:// URL, curl will return a status == 0, so if
     296             :          * there's no */
     297             :         /* error returned, patch the status code to be 200, as it would be for
     298             :          * http:// */
     299          34 :         if (psRequest->nStatus == 0 && psRequest->Error.empty() &&
     300           0 :             STARTS_WITH(psRequest->URL.c_str(), "file://"))
     301           0 :             psRequest->nStatus = 200;
     302             : 
     303             :         // If there is an error with no error message, use the content if it is
     304             :         // text
     305          66 :         if (psRequest->Error.empty() && psRequest->nStatus != 0 &&
     306          32 :             psRequest->nStatus != 200 &&
     307          66 :             strstr(psRequest->ContentType, "text") &&
     308           0 :             psRequest->pabyData != nullptr)
     309             :             psRequest->Error =
     310           0 :                 reinterpret_cast<const char *>(psRequest->pabyData);
     311             : 
     312          66 :         CPLDebug(
     313             :             "HTTP", "Request [%d] %s : status = %d, type = %s, error = %s", i,
     314             :             psRequest->URL.c_str(), psRequest->nStatus,
     315          34 :             !psRequest->ContentType.empty() ? psRequest->ContentType.c_str()
     316             :                                             : "(null)",
     317          34 :             !psRequest->Error.empty() ? psRequest->Error.c_str() : "(null)");
     318             : 
     319          34 :         curl_multi_remove_handle(curl_multi, pasRequest->m_curl_handle);
     320             :     }
     321             : 
     322          21 :     curl_multi_cleanup(curl_multi);
     323             : 
     324          21 :     return ret;
     325             : }