Line data Source code
1 : /**********************************************************************
2 : * Project: CPL - Common Portability Library
3 : * Purpose: Microsoft Azure Storage Blob routines
4 : * Author: Even Rouault <even.rouault at spatialys.com>
5 : *
6 : **********************************************************************
7 : * Copyright (c) 2017, Even Rouault <even.rouault at spatialys.com>
8 : *
9 : * Permission is hereby granted, free of charge, to any person obtaining a
10 : * copy of this software and associated documentation files (the "Software"),
11 : * to deal in the Software without restriction, including without limitation
12 : * the rights to use, copy, modify, merge, publish, distribute, sublicense,
13 : * and/or sell copies of the Software, and to permit persons to whom the
14 : * Software is furnished to do so, subject to the following conditions:
15 : *
16 : * The above copyright notice and this permission notice shall be included
17 : * in all copies or substantial portions of the Software.
18 : *
19 : * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
20 : * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21 : * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
22 : * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
23 : * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
24 : * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
25 : * DEALINAzureBlob IN THE SOFTWARE.
26 : ****************************************************************************/
27 :
28 : #include "cpl_azure.h"
29 : #include "cpl_json.h"
30 : #include "cpl_minixml.h"
31 : #include "cpl_vsi_error.h"
32 : #include "cpl_sha256.h"
33 : #include "cpl_time.h"
34 : #include "cpl_http.h"
35 : #include "cpl_multiproc.h"
36 : #include "cpl_vsi_virtual.h"
37 : #include "cpl_vsil_curl_class.h"
38 :
39 : #include <mutex>
40 :
41 : //! @cond Doxygen_Suppress
42 :
43 : #ifdef HAVE_CURL
44 :
45 : constexpr const char *X_MS_VERSION = "2019-12-12";
46 :
47 : /************************************************************************/
48 : /* RemoveTrailingSlash() */
49 : /************************************************************************/
50 :
51 663 : static std::string RemoveTrailingSlash(const std::string &osStr)
52 : {
53 663 : std::string osRet(osStr);
54 663 : if (!osRet.empty() && osRet.back() == '/')
55 1 : osRet.pop_back();
56 663 : return osRet;
57 : }
58 :
59 : /************************************************************************/
60 : /* CPLAzureGetSignature() */
61 : /************************************************************************/
62 :
63 218 : static std::string CPLAzureGetSignature(const std::string &osStringToSign,
64 : const std::string &osStorageKeyB64)
65 : {
66 :
67 : /* -------------------------------------------------------------------- */
68 : /* Compute signature. */
69 : /* -------------------------------------------------------------------- */
70 :
71 436 : std::string osStorageKeyUnbase64(osStorageKeyB64);
72 436 : int nB64Length = CPLBase64DecodeInPlace(
73 218 : reinterpret_cast<GByte *>(&osStorageKeyUnbase64[0]));
74 218 : osStorageKeyUnbase64.resize(nB64Length);
75 : #ifdef DEBUG_VERBOSE
76 : CPLDebug("AZURE", "signing key size: %d", nB64Length);
77 : #endif
78 :
79 218 : GByte abySignature[CPL_SHA256_HASH_SIZE] = {};
80 436 : CPL_HMAC_SHA256(osStorageKeyUnbase64.c_str(), nB64Length,
81 218 : osStringToSign.c_str(), osStringToSign.size(),
82 : abySignature);
83 :
84 218 : char *pszB64Signature = CPLBase64Encode(CPL_SHA256_HASH_SIZE, abySignature);
85 218 : std::string osSignature(pszB64Signature);
86 218 : CPLFree(pszB64Signature);
87 436 : return osSignature;
88 : }
89 :
90 : /************************************************************************/
91 : /* GetAzureBlobHeaders() */
92 : /************************************************************************/
93 :
94 224 : static struct curl_slist *GetAzureBlobHeaders(
95 : const std::string &osVerb, struct curl_slist *psHeaders,
96 : const std::string &osResource,
97 : const std::map<std::string, std::string> &oMapQueryParameters,
98 : const std::string &osStorageAccount, const std::string &osStorageKeyB64,
99 : bool bIncludeMSVersion)
100 : {
101 : /* See
102 : * https://docs.microsoft.com/en-us/rest/api/storageservices/authentication-for-the-azure-storage-services
103 : */
104 :
105 224 : const auto AddHeaders = [bIncludeMSVersion](struct curl_slist *l_psHeaders,
106 224 : const std::string &osDate)
107 : {
108 224 : l_psHeaders = curl_slist_append(
109 : l_psHeaders, CPLSPrintf("x-ms-date: %s", osDate.c_str()));
110 224 : if (bIncludeMSVersion)
111 : {
112 217 : l_psHeaders = curl_slist_append(
113 : l_psHeaders, CPLSPrintf("x-ms-version: %s", X_MS_VERSION));
114 : }
115 224 : return l_psHeaders;
116 224 : };
117 :
118 448 : std::string osDate = CPLGetConfigOption("CPL_AZURE_TIMESTAMP", "");
119 224 : if (osDate.empty())
120 : {
121 7 : osDate = IVSIS3LikeHandleHelper::GetRFC822DateTime();
122 : }
123 :
124 224 : if (osStorageKeyB64.empty())
125 : {
126 10 : psHeaders = AddHeaders(psHeaders, osDate);
127 10 : return psHeaders;
128 : }
129 :
130 428 : std::map<std::string, std::string> oSortedMapMSHeaders;
131 214 : if (bIncludeMSVersion)
132 207 : oSortedMapMSHeaders["x-ms-version"] = X_MS_VERSION;
133 214 : oSortedMapMSHeaders["x-ms-date"] = osDate;
134 : std::string osCanonicalizedHeaders(
135 : IVSIS3LikeHandleHelper::BuildCanonicalizedHeaders(oSortedMapMSHeaders,
136 428 : psHeaders, "x-ms-"));
137 :
138 428 : std::string osCanonicalizedResource;
139 214 : osCanonicalizedResource += "/" + osStorageAccount;
140 214 : osCanonicalizedResource += osResource;
141 :
142 : // We assume query parameters are in lower case and they are not repeated
143 : std::map<std::string, std::string>::const_iterator oIter =
144 214 : oMapQueryParameters.begin();
145 551 : for (; oIter != oMapQueryParameters.end(); ++oIter)
146 : {
147 337 : osCanonicalizedResource += "\n";
148 337 : osCanonicalizedResource += oIter->first;
149 337 : osCanonicalizedResource += ":";
150 337 : osCanonicalizedResource += oIter->second;
151 : }
152 :
153 428 : std::string osStringToSign;
154 214 : osStringToSign += osVerb + "\n";
155 214 : osStringToSign += CPLAWSGetHeaderVal(psHeaders, "Content-Encoding") + "\n";
156 214 : osStringToSign += CPLAWSGetHeaderVal(psHeaders, "Content-Language") + "\n";
157 : std::string osContentLength(
158 428 : CPLAWSGetHeaderVal(psHeaders, "Content-Length"));
159 214 : if (osContentLength == "0")
160 34 : osContentLength.clear(); // since x-ms-version 2015-02-21
161 214 : osStringToSign += osContentLength + "\n";
162 214 : osStringToSign += CPLAWSGetHeaderVal(psHeaders, "Content-MD5") + "\n";
163 214 : osStringToSign += CPLAWSGetHeaderVal(psHeaders, "Content-Type") + "\n";
164 214 : osStringToSign += CPLAWSGetHeaderVal(psHeaders, "Date") + "\n";
165 214 : osStringToSign += CPLAWSGetHeaderVal(psHeaders, "If-Modified-Since") + "\n";
166 214 : osStringToSign += CPLAWSGetHeaderVal(psHeaders, "If-Match") + "\n";
167 214 : osStringToSign += CPLAWSGetHeaderVal(psHeaders, "If-None-Match") + "\n";
168 : osStringToSign +=
169 214 : CPLAWSGetHeaderVal(psHeaders, "If-Unmodified-Since") + "\n";
170 214 : osStringToSign += CPLAWSGetHeaderVal(psHeaders, "Range") + "\n";
171 214 : osStringToSign += osCanonicalizedHeaders;
172 214 : osStringToSign += osCanonicalizedResource;
173 :
174 : #ifdef DEBUG_VERBOSE
175 : CPLDebug("AZURE", "osStringToSign = '%s'", osStringToSign.c_str());
176 : #endif
177 :
178 : /* -------------------------------------------------------------------- */
179 : /* Compute signature. */
180 : /* -------------------------------------------------------------------- */
181 :
182 : std::string osAuthorization(
183 428 : "SharedKey " + osStorageAccount + ":" +
184 428 : CPLAzureGetSignature(osStringToSign, osStorageKeyB64));
185 :
186 214 : psHeaders = AddHeaders(psHeaders, osDate);
187 214 : psHeaders = curl_slist_append(
188 : psHeaders, CPLSPrintf("Authorization: %s", osAuthorization.c_str()));
189 :
190 214 : return psHeaders;
191 : }
192 :
193 : /************************************************************************/
194 : /* VSIAzureBlobHandleHelper() */
195 : /************************************************************************/
196 343 : VSIAzureBlobHandleHelper::VSIAzureBlobHandleHelper(
197 : const std::string &osPathForOption, const std::string &osEndpoint,
198 : const std::string &osBucket, const std::string &osObjectKey,
199 : const std::string &osStorageAccount, const std::string &osStorageKey,
200 : const std::string &osSAS, const std::string &osAccessToken,
201 343 : bool bFromManagedIdentities)
202 : : m_osPathForOption(osPathForOption),
203 : m_osURL(BuildURL(osEndpoint, osBucket, osObjectKey, osSAS)),
204 : m_osEndpoint(osEndpoint), m_osBucket(osBucket),
205 : m_osObjectKey(osObjectKey), m_osStorageAccount(osStorageAccount),
206 : m_osStorageKey(osStorageKey), m_osSAS(osSAS),
207 : m_osAccessToken(osAccessToken),
208 343 : m_bFromManagedIdentities(bFromManagedIdentities)
209 : {
210 343 : }
211 :
212 : /************************************************************************/
213 : /* ~VSIAzureBlobHandleHelper() */
214 : /************************************************************************/
215 :
216 686 : VSIAzureBlobHandleHelper::~VSIAzureBlobHandleHelper()
217 : {
218 686 : }
219 :
220 : /************************************************************************/
221 : /* AzureCSGetParameter() */
222 : /************************************************************************/
223 :
224 1145 : static std::string AzureCSGetParameter(const std::string &osStr,
225 : const char *pszKey, bool bErrorIfMissing)
226 : {
227 3435 : std::string osKey(pszKey + std::string("="));
228 1145 : size_t nPos = osStr.find(osKey);
229 1145 : if (nPos == std::string::npos)
230 : {
231 : const char *pszMsg =
232 21 : CPLSPrintf("%s missing in AZURE_STORAGE_CONNECTION_STRING", pszKey);
233 21 : if (bErrorIfMissing)
234 : {
235 0 : CPLDebug("AZURE", "%s", pszMsg);
236 0 : VSIError(VSIE_InvalidCredentials, "%s", pszMsg);
237 : }
238 21 : return std::string();
239 : }
240 1124 : size_t nPos2 = osStr.find(";", nPos);
241 1124 : return osStr.substr(nPos + osKey.size(), nPos2 == std::string::npos
242 : ? nPos2
243 2248 : : nPos2 - nPos - osKey.size());
244 : }
245 :
246 : /************************************************************************/
247 : /* CPLAzureCachedToken */
248 : /************************************************************************/
249 :
250 : std::mutex gMutex;
251 :
252 : struct CPLAzureCachedToken
253 : {
254 : std::string osAccessToken{};
255 : GIntBig nExpiresOn = 0;
256 : };
257 :
258 : static std::map<std::string, CPLAzureCachedToken> goMapIMDSURLToCachedToken;
259 :
260 : /************************************************************************/
261 : /* GetConfigurationFromIMDSCredentials() */
262 : /************************************************************************/
263 :
264 : static bool
265 27 : GetConfigurationFromIMDSCredentials(const std::string &osPathForOption,
266 : std::string &osAccessToken)
267 : {
268 : // coverity[tainted_data]
269 : const std::string osRootURL(CPLGetConfigOption("CPL_AZURE_VM_API_ROOT_URL",
270 54 : "http://169.254.169.254"));
271 27 : if (osRootURL == "disabled")
272 3 : return false;
273 :
274 : std::string osURLResource("/metadata/identity/oauth2/"
275 : "token?api-version=2018-02-01&resource=https%"
276 48 : "3A%2F%2Fstorage.azure.com%2F");
277 24 : const char *pszObjectId = VSIGetPathSpecificOption(
278 : osPathForOption.c_str(), "AZURE_IMDS_OBJECT_ID", nullptr);
279 24 : if (pszObjectId)
280 12 : osURLResource += "&object_id=" + CPLAWSURLEncode(pszObjectId, false);
281 24 : const char *pszClientId = VSIGetPathSpecificOption(
282 : osPathForOption.c_str(), "AZURE_IMDS_CLIENT_ID", nullptr);
283 24 : if (pszClientId)
284 12 : osURLResource += "&client_id=" + CPLAWSURLEncode(pszClientId, false);
285 24 : const char *pszMsiResId = VSIGetPathSpecificOption(
286 : osPathForOption.c_str(), "AZURE_IMDS_MSI_RES_ID", nullptr);
287 24 : if (pszMsiResId)
288 12 : osURLResource += "&msi_res_id=" + CPLAWSURLEncode(pszMsiResId, false);
289 :
290 48 : std::lock_guard<std::mutex> guard(gMutex);
291 :
292 : // Look for cached token corresponding to this IMDS request URL
293 24 : auto oIter = goMapIMDSURLToCachedToken.find(osURLResource);
294 24 : if (oIter != goMapIMDSURLToCachedToken.end())
295 : {
296 20 : const auto &oCachedToken = oIter->second;
297 : time_t nCurTime;
298 20 : time(&nCurTime);
299 : // Try to reuse credentials if they are still valid, but
300 : // keep one minute of margin...
301 20 : if (nCurTime < oCachedToken.nExpiresOn - 60)
302 : {
303 17 : osAccessToken = oCachedToken.osAccessToken;
304 17 : return true;
305 : }
306 : }
307 :
308 : // Fetch credentials
309 7 : CPLStringList oResponse;
310 7 : const char *const apszOptions[] = {"HEADERS=Metadata: true", nullptr};
311 : CPLHTTPResult *psResult =
312 7 : CPLHTTPFetch((osRootURL + osURLResource).c_str(), apszOptions);
313 7 : if (psResult)
314 : {
315 7 : if (psResult->nStatus == 0 && psResult->pabyData != nullptr)
316 : {
317 : const std::string osJSon =
318 14 : reinterpret_cast<char *>(psResult->pabyData);
319 7 : oResponse = CPLParseKeyValueJson(osJSon.c_str());
320 7 : if (oResponse.FetchNameValue("error"))
321 : {
322 0 : CPLDebug("AZURE",
323 : "Cannot retrieve managed identities credentials: %s",
324 : osJSon.c_str());
325 : }
326 : }
327 7 : CPLHTTPDestroyResult(psResult);
328 : }
329 7 : osAccessToken = oResponse.FetchNameValueDef("access_token", "");
330 : const GIntBig nExpiresOn =
331 7 : CPLAtoGIntBig(oResponse.FetchNameValueDef("expires_on", ""));
332 7 : if (!osAccessToken.empty() && nExpiresOn > 0)
333 : {
334 14 : CPLAzureCachedToken cachedToken;
335 7 : cachedToken.osAccessToken = osAccessToken;
336 7 : cachedToken.nExpiresOn = nExpiresOn;
337 7 : goMapIMDSURLToCachedToken[osURLResource] = std::move(cachedToken);
338 7 : CPLDebug("AZURE", "Storing credentials for %s until " CPL_FRMT_GIB,
339 : osURLResource.c_str(), nExpiresOn);
340 : }
341 :
342 7 : return !osAccessToken.empty();
343 : }
344 :
345 : /************************************************************************/
346 : /* GetConfigurationFromWorkloadIdentity() */
347 : /************************************************************************/
348 :
349 : // Last timestamp AZURE_FEDERATED_TOKEN_FILE was read
350 : static GIntBig gnLastReadFederatedTokenFile = 0;
351 : static std::string gosFederatedToken{};
352 :
353 : // Azure Active Directory Workload Identity, typically for Azure Kubernetes
354 : // Cf https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/identity/azure-identity/azure/identity/_credentials/workload_identity.py
355 43 : static bool GetConfigurationFromWorkloadIdentity(std::string &osAccessToken)
356 : {
357 : const std::string AZURE_CLIENT_ID(
358 86 : CPLGetConfigOption("AZURE_CLIENT_ID", ""));
359 : const std::string AZURE_TENANT_ID(
360 86 : CPLGetConfigOption("AZURE_TENANT_ID", ""));
361 : const std::string AZURE_AUTHORITY_HOST(
362 86 : CPLGetConfigOption("AZURE_AUTHORITY_HOST", ""));
363 : const std::string AZURE_FEDERATED_TOKEN_FILE(
364 86 : CPLGetConfigOption("AZURE_FEDERATED_TOKEN_FILE", ""));
365 75 : if (AZURE_CLIENT_ID.empty() || AZURE_TENANT_ID.empty() ||
366 75 : AZURE_AUTHORITY_HOST.empty() || AZURE_FEDERATED_TOKEN_FILE.empty())
367 : {
368 27 : return false;
369 : }
370 :
371 32 : std::lock_guard<std::mutex> guard(gMutex);
372 :
373 : time_t nCurTime;
374 16 : time(&nCurTime);
375 :
376 : // Look for cached token corresponding to this request URL
377 16 : const std::string osURL(AZURE_AUTHORITY_HOST + AZURE_TENANT_ID +
378 32 : "/oauth2/v2.0/token");
379 16 : auto oIter = goMapIMDSURLToCachedToken.find(osURL);
380 16 : if (oIter != goMapIMDSURLToCachedToken.end())
381 : {
382 14 : const auto &oCachedToken = oIter->second;
383 : // Try to reuse credentials if they are still valid, but
384 : // keep one minute of margin...
385 14 : if (nCurTime < oCachedToken.nExpiresOn - 60)
386 : {
387 10 : osAccessToken = oCachedToken.osAccessToken;
388 10 : return true;
389 : }
390 : }
391 :
392 : // Ingest content of AZURE_FEDERATED_TOKEN_FILE if last time was more than
393 : // 600 seconds.
394 6 : if (nCurTime - gnLastReadFederatedTokenFile > 600)
395 : {
396 : auto fp = VSIVirtualHandleUniquePtr(
397 2 : VSIFOpenL(AZURE_FEDERATED_TOKEN_FILE.c_str(), "rb"));
398 2 : if (!fp)
399 : {
400 0 : CPLDebug("AZURE", "Cannot open AZURE_FEDERATED_TOKEN_FILE = %s",
401 : AZURE_FEDERATED_TOKEN_FILE.c_str());
402 0 : return false;
403 : }
404 2 : fp->Seek(0, SEEK_END);
405 2 : const auto nSize = fp->Tell();
406 2 : if (nSize == 0 || nSize > 100 * 1024)
407 : {
408 0 : CPLDebug(
409 : "AZURE",
410 : "Invalid size for AZURE_FEDERATED_TOKEN_FILE = " CPL_FRMT_GUIB,
411 : static_cast<GUIntBig>(nSize));
412 0 : return false;
413 : }
414 2 : fp->Seek(0, SEEK_SET);
415 2 : gosFederatedToken.resize(static_cast<size_t>(nSize));
416 2 : if (fp->Read(&gosFederatedToken[0], gosFederatedToken.size(), 1) != 1)
417 : {
418 0 : CPLDebug("AZURE", "Cannot read AZURE_FEDERATED_TOKEN_FILE");
419 0 : return false;
420 : }
421 2 : gnLastReadFederatedTokenFile = nCurTime;
422 : }
423 :
424 : /* -------------------------------------------------------------------- */
425 : /* Prepare POST request. */
426 : /* -------------------------------------------------------------------- */
427 12 : CPLStringList aosOptions;
428 :
429 : aosOptions.AddString(
430 6 : "HEADERS=Content-Type: application/x-www-form-urlencoded");
431 :
432 12 : std::string osItem("POSTFIELDS=client_assertion=");
433 6 : osItem += CPLAWSURLEncode(gosFederatedToken);
434 : osItem += "&client_assertion_type=urn:ietf:params:oauth:client-assertion-"
435 6 : "type:jwt-bearer";
436 6 : osItem += "&client_id=";
437 6 : osItem += CPLAWSURLEncode(AZURE_CLIENT_ID);
438 6 : osItem += "&grant_type=client_credentials";
439 6 : osItem += "&scope=https://storage.azure.com/.default";
440 6 : aosOptions.AddString(osItem.c_str());
441 :
442 : /* -------------------------------------------------------------------- */
443 : /* Submit request by HTTP. */
444 : /* -------------------------------------------------------------------- */
445 6 : CPLHTTPResult *psResult = CPLHTTPFetch(osURL.c_str(), aosOptions.List());
446 6 : if (!psResult)
447 0 : return false;
448 :
449 6 : if (!psResult->pabyData || psResult->pszErrBuf)
450 : {
451 0 : if (psResult->pszErrBuf)
452 0 : CPLDebug("AZURE", "%s", psResult->pszErrBuf);
453 0 : if (psResult->pabyData)
454 0 : CPLDebug("AZURE", "%s", psResult->pabyData);
455 :
456 0 : CPLDebug("AZURE",
457 : "Fetching OAuth2 access code from workload identity failed.");
458 0 : CPLHTTPDestroyResult(psResult);
459 0 : return false;
460 : }
461 :
462 : CPLStringList oResponse =
463 6 : CPLParseKeyValueJson(reinterpret_cast<char *>(psResult->pabyData));
464 6 : CPLHTTPDestroyResult(psResult);
465 :
466 6 : osAccessToken = oResponse.FetchNameValueDef("access_token", "");
467 6 : const int nExpiresIn = atoi(oResponse.FetchNameValueDef("expires_in", ""));
468 6 : if (!osAccessToken.empty() && nExpiresIn > 0)
469 : {
470 12 : CPLAzureCachedToken cachedToken;
471 6 : cachedToken.osAccessToken = osAccessToken;
472 6 : cachedToken.nExpiresOn = nCurTime + nExpiresIn;
473 6 : goMapIMDSURLToCachedToken[osURL] = cachedToken;
474 6 : CPLDebug("AZURE", "Storing credentials for %s until " CPL_FRMT_GIB,
475 : osURL.c_str(), cachedToken.nExpiresOn);
476 : }
477 :
478 6 : return !osAccessToken.empty();
479 : }
480 :
481 : /************************************************************************/
482 : /* GetConfigurationFromManagedIdentities() */
483 : /************************************************************************/
484 :
485 : static bool
486 43 : GetConfigurationFromManagedIdentities(const std::string &osPathForOption,
487 : std::string &osAccessToken)
488 : {
489 43 : if (GetConfigurationFromWorkloadIdentity(osAccessToken))
490 16 : return true;
491 27 : return GetConfigurationFromIMDSCredentials(osPathForOption, osAccessToken);
492 : }
493 :
494 : /************************************************************************/
495 : /* ClearCache() */
496 : /************************************************************************/
497 :
498 2820 : void VSIAzureBlobHandleHelper::ClearCache()
499 : {
500 5640 : std::lock_guard<std::mutex> guard(gMutex);
501 2820 : goMapIMDSURLToCachedToken.clear();
502 2820 : gnLastReadFederatedTokenFile = 0;
503 2820 : gosFederatedToken.clear();
504 2820 : }
505 :
506 : /************************************************************************/
507 : /* ParseStorageConnectionString() */
508 : /************************************************************************/
509 :
510 : static bool
511 285 : ParseStorageConnectionString(const std::string &osStorageConnectionString,
512 : const std::string &osServicePrefix,
513 : bool &bUseHTTPS, std::string &osEndpoint,
514 : std::string &osStorageAccount,
515 : std::string &osStorageKey, std::string &osSAS)
516 : {
517 : osStorageAccount =
518 285 : AzureCSGetParameter(osStorageConnectionString, "AccountName", false);
519 : osStorageKey =
520 285 : AzureCSGetParameter(osStorageConnectionString, "AccountKey", false);
521 :
522 : const std::string osProtocol(AzureCSGetParameter(
523 570 : osStorageConnectionString, "DefaultEndpointsProtocol", false));
524 285 : bUseHTTPS = (osProtocol != "http");
525 :
526 285 : if (osStorageAccount.empty() || osStorageKey.empty())
527 : {
528 5 : osStorageAccount.clear();
529 5 : osStorageKey.clear();
530 :
531 5 : std::string osBlobEndpoint = RemoveTrailingSlash(AzureCSGetParameter(
532 10 : osStorageConnectionString, "BlobEndpoint", false));
533 10 : osSAS = AzureCSGetParameter(osStorageConnectionString,
534 5 : "SharedAccessSignature", false);
535 5 : if (!osBlobEndpoint.empty() && !osSAS.empty())
536 : {
537 2 : osEndpoint = std::move(osBlobEndpoint);
538 2 : return true;
539 : }
540 :
541 3 : return false;
542 : }
543 :
544 : const std::string osBlobEndpoint =
545 280 : AzureCSGetParameter(osStorageConnectionString, "BlobEndpoint", false);
546 280 : if (!osBlobEndpoint.empty())
547 : {
548 280 : osEndpoint = RemoveTrailingSlash(osBlobEndpoint);
549 : }
550 : else
551 : {
552 : const std::string osEndpointSuffix(AzureCSGetParameter(
553 0 : osStorageConnectionString, "EndpointSuffix", false));
554 0 : if (!osEndpointSuffix.empty())
555 0 : osEndpoint = (bUseHTTPS ? "https://" : "http://") +
556 0 : osStorageAccount + "." + osServicePrefix + "." +
557 0 : RemoveTrailingSlash(osEndpointSuffix);
558 : }
559 :
560 280 : return true;
561 : }
562 :
563 : /************************************************************************/
564 : /* GetConfigurationFromCLIConfigFile() */
565 : /************************************************************************/
566 :
567 38 : static bool GetConfigurationFromCLIConfigFile(
568 : const std::string &osPathForOption, const std::string &osServicePrefix,
569 : bool &bUseHTTPS, std::string &osEndpoint, std::string &osStorageAccount,
570 : std::string &osStorageKey, std::string &osSAS, std::string &osAccessToken,
571 : bool &bFromManagedIdentities)
572 : {
573 : #ifdef _WIN32
574 : const char *pszHome = CPLGetConfigOption("USERPROFILE", nullptr);
575 : constexpr char SEP_STRING[] = "\\";
576 : #else
577 38 : const char *pszHome = CPLGetConfigOption("HOME", nullptr);
578 38 : constexpr char SEP_STRING[] = "/";
579 : #endif
580 :
581 76 : std::string osDotAzure(pszHome ? pszHome : "");
582 38 : osDotAzure += SEP_STRING;
583 38 : osDotAzure += ".azure";
584 :
585 : const char *pszAzureConfigDir =
586 38 : CPLGetConfigOption("AZURE_CONFIG_DIR", osDotAzure.c_str());
587 38 : if (pszAzureConfigDir[0] == '\0')
588 7 : return false;
589 :
590 62 : std::string osConfigFilename = pszAzureConfigDir;
591 31 : osConfigFilename += SEP_STRING;
592 31 : osConfigFilename += "config";
593 :
594 31 : VSILFILE *fp = VSIFOpenL(osConfigFilename.c_str(), "rb");
595 62 : std::string osStorageConnectionString;
596 31 : if (fp == nullptr)
597 19 : return false;
598 :
599 12 : bool bInStorageSection = false;
600 75 : while (const char *pszLine = CPLReadLineL(fp))
601 : {
602 63 : if (pszLine[0] == '#' || pszLine[0] == ';')
603 : {
604 : // comment line
605 : }
606 63 : else if (strcmp(pszLine, "[storage]") == 0)
607 : {
608 12 : bInStorageSection = true;
609 : }
610 51 : else if (pszLine[0] == '[')
611 : {
612 12 : bInStorageSection = false;
613 : }
614 39 : else if (bInStorageSection)
615 : {
616 15 : char *pszKey = nullptr;
617 15 : const char *pszValue = CPLParseNameValue(pszLine, &pszKey);
618 15 : if (pszKey && pszValue)
619 : {
620 15 : if (EQUAL(pszKey, "account"))
621 : {
622 6 : osStorageAccount = pszValue;
623 : }
624 9 : else if (EQUAL(pszKey, "connection_string"))
625 : {
626 3 : osStorageConnectionString = pszValue;
627 : }
628 6 : else if (EQUAL(pszKey, "key"))
629 : {
630 3 : osStorageKey = pszValue;
631 : }
632 3 : else if (EQUAL(pszKey, "sas_token"))
633 : {
634 3 : osSAS = pszValue;
635 : // Az CLI apparently uses configparser with
636 : // BasicInterpolation where the % character has a special
637 : // meaning See
638 : // https://docs.python.org/3/library/configparser.html#configparser.BasicInterpolation
639 : // A token might end with %%3D which must be transformed to
640 : // %3D
641 3 : osSAS = CPLString(osSAS).replaceAll("%%", '%');
642 : }
643 : }
644 15 : CPLFree(pszKey);
645 : }
646 63 : }
647 12 : VSIFCloseL(fp);
648 :
649 12 : if (!osStorageConnectionString.empty())
650 : {
651 3 : return ParseStorageConnectionString(
652 : osStorageConnectionString, osServicePrefix, bUseHTTPS, osEndpoint,
653 3 : osStorageAccount, osStorageKey, osSAS);
654 : }
655 :
656 9 : if (osStorageAccount.empty())
657 : {
658 3 : CPLDebug("AZURE", "Missing storage.account in %s",
659 : osConfigFilename.c_str());
660 3 : return false;
661 : }
662 :
663 6 : if (osEndpoint.empty())
664 0 : osEndpoint = (bUseHTTPS ? "https://" : "http://") + osStorageAccount +
665 0 : "." + osServicePrefix + ".core.windows.net";
666 :
667 6 : osAccessToken = CPLGetConfigOption("AZURE_STORAGE_ACCESS_TOKEN", "");
668 6 : if (!osAccessToken.empty())
669 0 : return true;
670 :
671 6 : if (osStorageKey.empty() && osSAS.empty())
672 : {
673 0 : if (CPLTestBool(CPLGetConfigOption("AZURE_NO_SIGN_REQUEST", "NO")))
674 : {
675 0 : return true;
676 : }
677 :
678 0 : std::string osTmpAccessToken;
679 0 : if (GetConfigurationFromManagedIdentities(osPathForOption,
680 : osTmpAccessToken))
681 : {
682 0 : bFromManagedIdentities = true;
683 0 : return true;
684 : }
685 :
686 0 : CPLDebug("AZURE", "Missing storage.key or storage.sas_token in %s",
687 : osConfigFilename.c_str());
688 0 : return false;
689 : }
690 :
691 6 : return true;
692 : }
693 :
694 : /************************************************************************/
695 : /* GetSAS() */
696 : /************************************************************************/
697 :
698 : /* static */
699 55 : std::string VSIAzureBlobHandleHelper::GetSAS(const char *pszFilename)
700 : {
701 : return VSIGetPathSpecificOption(
702 : pszFilename, "AZURE_STORAGE_SAS_TOKEN",
703 : CPLGetConfigOption("AZURE_SAS",
704 55 : "")); // AZURE_SAS for GDAL < 3.5
705 : }
706 :
707 : /************************************************************************/
708 : /* IsNoSignRequest() */
709 : /************************************************************************/
710 :
711 : /* static */
712 388 : bool VSIAzureBlobHandleHelper::IsNoSignRequest(const char *pszFilename)
713 : {
714 388 : return CPLTestBool(
715 388 : VSIGetPathSpecificOption(pszFilename, "AZURE_NO_SIGN_REQUEST", "NO"));
716 : }
717 :
718 : /************************************************************************/
719 : /* GetConfiguration() */
720 : /************************************************************************/
721 :
722 378 : bool VSIAzureBlobHandleHelper::GetConfiguration(
723 : const std::string &osPathForOption, CSLConstList papszOptions,
724 : Service eService, bool &bUseHTTPS, std::string &osEndpoint,
725 : std::string &osStorageAccount, std::string &osStorageKey,
726 : std::string &osSAS, std::string &osAccessToken,
727 : bool &bFromManagedIdentities)
728 : {
729 378 : bFromManagedIdentities = false;
730 :
731 : const std::string osServicePrefix(
732 756 : eService == Service::SERVICE_BLOB ? "blob" : "dfs");
733 378 : bUseHTTPS = CPLTestBool(VSIGetPathSpecificOption(
734 : osPathForOption.c_str(), "CPL_AZURE_USE_HTTPS", "YES"));
735 756 : osEndpoint = RemoveTrailingSlash(VSIGetPathSpecificOption(
736 378 : osPathForOption.c_str(), "CPL_AZURE_ENDPOINT", ""));
737 :
738 : const std::string osStorageConnectionString(CSLFetchNameValueDef(
739 : papszOptions, "AZURE_STORAGE_CONNECTION_STRING",
740 : VSIGetPathSpecificOption(osPathForOption.c_str(),
741 756 : "AZURE_STORAGE_CONNECTION_STRING", "")));
742 378 : if (!osStorageConnectionString.empty())
743 : {
744 282 : return ParseStorageConnectionString(
745 : osStorageConnectionString, osServicePrefix, bUseHTTPS, osEndpoint,
746 282 : osStorageAccount, osStorageKey, osSAS);
747 : }
748 : else
749 : {
750 : osStorageAccount = CSLFetchNameValueDef(
751 : papszOptions, "AZURE_STORAGE_ACCOUNT",
752 : VSIGetPathSpecificOption(osPathForOption.c_str(),
753 96 : "AZURE_STORAGE_ACCOUNT", ""));
754 96 : if (!osStorageAccount.empty())
755 : {
756 58 : if (osEndpoint.empty())
757 46 : osEndpoint = (bUseHTTPS ? "https://" : "http://") +
758 46 : osStorageAccount + "." + osServicePrefix +
759 23 : ".core.windows.net";
760 :
761 : osAccessToken = CSLFetchNameValueDef(
762 : papszOptions, "AZURE_STORAGE_ACCESS_TOKEN",
763 : VSIGetPathSpecificOption(osPathForOption.c_str(),
764 58 : "AZURE_STORAGE_ACCESS_TOKEN", ""));
765 58 : if (!osAccessToken.empty())
766 3 : return true;
767 :
768 : osStorageKey = CSLFetchNameValueDef(
769 : papszOptions, "AZURE_STORAGE_ACCESS_KEY",
770 : VSIGetPathSpecificOption(osPathForOption.c_str(),
771 55 : "AZURE_STORAGE_ACCESS_KEY", ""));
772 55 : if (osStorageKey.empty())
773 : {
774 49 : osSAS = GetSAS(osPathForOption.c_str());
775 49 : if (osSAS.empty())
776 : {
777 39 : if (IsNoSignRequest(osPathForOption.c_str()))
778 : {
779 7 : return true;
780 : }
781 :
782 64 : std::string osTmpAccessToken;
783 32 : if (GetConfigurationFromManagedIdentities(osPathForOption,
784 : osTmpAccessToken))
785 : {
786 29 : bFromManagedIdentities = true;
787 29 : return true;
788 : }
789 :
790 3 : const char *pszMsg =
791 : "AZURE_STORAGE_ACCESS_KEY or AZURE_STORAGE_SAS_TOKEN "
792 : "or AZURE_NO_SIGN_REQUEST configuration option "
793 : "not defined";
794 3 : CPLDebug("AZURE", "%s", pszMsg);
795 3 : VSIError(VSIE_InvalidCredentials, "%s", pszMsg);
796 3 : return false;
797 : }
798 : }
799 16 : return true;
800 : }
801 : }
802 :
803 38 : if (GetConfigurationFromCLIConfigFile(
804 : osPathForOption, osServicePrefix, bUseHTTPS, osEndpoint,
805 : osStorageAccount, osStorageKey, osSAS, osAccessToken,
806 : bFromManagedIdentities))
807 : {
808 9 : return true;
809 : }
810 :
811 29 : const char *pszMsg =
812 : "No valid Azure credentials found. "
813 : "For authenticated requests, you need to set "
814 : "AZURE_STORAGE_ACCOUNT, AZURE_STORAGE_ACCESS_KEY, "
815 : "AZURE_STORAGE_SAS_TOKEN, "
816 : "AZURE_STORAGE_CONNECTION_STRING, or other configuration "
817 : "options. Consult "
818 : "https://gdal.org/en/stable/user/"
819 : "virtual_file_systems.html#vsiaz-microsoft-azure-blob-files "
820 : "for more details. "
821 : "For unauthenticated requests on public resources, set the "
822 : "AZURE_NO_SIGN_REQUEST configuration option to YES.";
823 29 : CPLDebug("AZURE", "%s", pszMsg);
824 29 : VSIError(VSIE_InvalidCredentials, "%s", pszMsg);
825 29 : return false;
826 : }
827 :
828 : /************************************************************************/
829 : /* BuildFromURI() */
830 : /************************************************************************/
831 :
832 378 : VSIAzureBlobHandleHelper *VSIAzureBlobHandleHelper::BuildFromURI(
833 : const char *pszURI, const char *pszFSPrefix,
834 : const char *pszURIForPathSpecificOption, CSLConstList papszOptions)
835 : {
836 378 : if (strcmp(pszFSPrefix, "/vsiaz/") != 0 &&
837 107 : strcmp(pszFSPrefix, "/vsiaz_streaming/") != 0 &&
838 95 : strcmp(pszFSPrefix, "/vsiadls/") != 0)
839 : {
840 0 : CPLError(CE_Failure, CPLE_AppDefined, "Unsupported FS prefix");
841 0 : return nullptr;
842 : }
843 :
844 756 : const auto eService = strcmp(pszFSPrefix, "/vsiaz/") == 0 ||
845 107 : strcmp(pszFSPrefix, "/vsiaz_streaming/") == 0
846 378 : ? Service::SERVICE_BLOB
847 : : Service::SERVICE_ADLS;
848 :
849 : std::string osPathForOption(
850 756 : eService == Service::SERVICE_BLOB ? "/vsiaz/" : "/vsiadls/");
851 : osPathForOption +=
852 378 : pszURIForPathSpecificOption ? pszURIForPathSpecificOption : pszURI;
853 :
854 378 : bool bUseHTTPS = true;
855 756 : std::string osStorageAccount;
856 756 : std::string osStorageKey;
857 756 : std::string osEndpoint;
858 756 : std::string osSAS;
859 756 : std::string osAccessToken;
860 378 : bool bFromManagedIdentities = false;
861 :
862 378 : if (!GetConfiguration(osPathForOption, papszOptions, eService, bUseHTTPS,
863 : osEndpoint, osStorageAccount, osStorageKey, osSAS,
864 : osAccessToken, bFromManagedIdentities))
865 : {
866 35 : return nullptr;
867 : }
868 :
869 343 : if (IsNoSignRequest(osPathForOption.c_str()))
870 : {
871 15 : osStorageKey.clear();
872 15 : osSAS.clear();
873 15 : osAccessToken.clear();
874 : }
875 :
876 : // pszURI == bucket/object
877 686 : const std::string osBucketObject(pszURI);
878 686 : std::string osBucket(osBucketObject);
879 343 : std::string osObjectKey;
880 343 : size_t nSlashPos = osBucketObject.find('/');
881 343 : if (nSlashPos != std::string::npos)
882 : {
883 253 : osBucket = osBucketObject.substr(0, nSlashPos);
884 253 : osObjectKey = osBucketObject.substr(nSlashPos + 1);
885 : }
886 :
887 : return new VSIAzureBlobHandleHelper(
888 : osPathForOption, osEndpoint, osBucket, osObjectKey, osStorageAccount,
889 343 : osStorageKey, osSAS, osAccessToken, bFromManagedIdentities);
890 : }
891 :
892 : /************************************************************************/
893 : /* BuildURL() */
894 : /************************************************************************/
895 :
896 991 : std::string VSIAzureBlobHandleHelper::BuildURL(const std::string &osEndpoint,
897 : const std::string &osBucket,
898 : const std::string &osObjectKey,
899 : const std::string &osSAS)
900 : {
901 991 : std::string osURL = osEndpoint;
902 991 : osURL += "/";
903 991 : osURL += CPLAWSURLEncode(osBucket, false);
904 991 : if (!osObjectKey.empty())
905 493 : osURL += "/" + CPLAWSURLEncode(osObjectKey, false);
906 991 : if (!osSAS.empty())
907 15 : osURL += '?' + osSAS;
908 991 : return osURL;
909 : }
910 :
911 : /************************************************************************/
912 : /* RebuildURL() */
913 : /************************************************************************/
914 :
915 648 : void VSIAzureBlobHandleHelper::RebuildURL()
916 : {
917 648 : m_osURL = BuildURL(m_osEndpoint, m_osBucket, m_osObjectKey, std::string());
918 648 : m_osURL += GetQueryString(false);
919 648 : if (!m_osSAS.empty())
920 12 : m_osURL += (m_oMapQueryParameters.empty() ? '?' : '&') + m_osSAS;
921 648 : }
922 :
923 : /************************************************************************/
924 : /* GetSASQueryString() */
925 : /************************************************************************/
926 :
927 73 : std::string VSIAzureBlobHandleHelper::GetSASQueryString() const
928 : {
929 73 : if (!m_osSAS.empty())
930 4 : return '?' + m_osSAS;
931 69 : return std::string();
932 : }
933 :
934 : /************************************************************************/
935 : /* GetCurlHeaders() */
936 : /************************************************************************/
937 :
938 : struct curl_slist *
939 236 : VSIAzureBlobHandleHelper::GetCurlHeaders(const std::string &osVerb,
940 : struct curl_slist *psHeaders,
941 : const void *, size_t) const
942 : {
943 236 : if (m_bFromManagedIdentities || !m_osAccessToken.empty())
944 : {
945 12 : psHeaders = curl_slist_append(
946 : psHeaders,
947 24 : std::string("x-ms-version: ").append(X_MS_VERSION).c_str());
948 :
949 24 : std::string osAccessToken;
950 12 : if (m_bFromManagedIdentities)
951 : {
952 11 : if (!GetConfigurationFromManagedIdentities(m_osPathForOption,
953 : osAccessToken))
954 0 : return psHeaders;
955 : }
956 : else
957 : {
958 1 : osAccessToken = m_osAccessToken;
959 : }
960 :
961 : // Do not use CPLSPrintf() as we could get over the 8K character limit
962 : // with very large SAS tokens
963 12 : std::string osAuthorization = "Authorization: Bearer ";
964 12 : osAuthorization += osAccessToken;
965 12 : psHeaders = curl_slist_append(psHeaders, osAuthorization.c_str());
966 12 : return psHeaders;
967 : }
968 :
969 448 : std::string osResource;
970 224 : const auto nSlashSlashPos = m_osEndpoint.find("//");
971 224 : if (nSlashSlashPos != std::string::npos)
972 : {
973 224 : const auto nResourcePos = m_osEndpoint.find('/', nSlashSlashPos + 2);
974 224 : if (nResourcePos != std::string::npos)
975 217 : osResource = m_osEndpoint.substr(nResourcePos);
976 : }
977 224 : osResource += "/" + m_osBucket;
978 224 : if (!m_osObjectKey.empty())
979 141 : osResource += "/" + CPLAWSURLEncode(m_osObjectKey, false);
980 :
981 : // If accessing a Microsoft Azure account from a Azure VM, check that
982 : // Microsoft is still a sponsor, and if not, make some (kind) noise.
983 0 : if ((m_bFromManagedIdentities &&
984 0 : m_osEndpoint.find("core.windows.net") != std::string::npos)
985 : #ifdef DEBUG
986 224 : || CPLTestBool(CPLGetConfigOption("GDAL_TEST_NAME_AND_SHAME", "NO"))
987 : #endif
988 : )
989 : {
990 0 : static const bool bCheckSponsoring = []()
991 : {
992 0 : if (!CPLTestBool(CPLGetConfigOption("GDAL_NAME_AND_SHAME", "YES")))
993 0 : return true;
994 :
995 0 : const std::string osCacheDir = []()
996 : {
997 : #ifdef _WIN32
998 : const char *pszHome =
999 : CPLGetConfigOption("USERPROFILE", nullptr);
1000 : #else
1001 0 : const char *pszHome = CPLGetConfigOption("HOME", nullptr);
1002 : #endif
1003 0 : if (pszHome != nullptr)
1004 : {
1005 0 : return CPLFormFilenameSafe(pszHome, ".gdal", nullptr);
1006 : }
1007 : else
1008 : {
1009 0 : const char *pszDir = CPLGetConfigOption("TEMP", "/tmp");
1010 : VSIStatBufL sStat;
1011 0 : if (VSIStatL(pszDir, &sStat) == 0)
1012 : {
1013 : const char *pszUsername =
1014 0 : CPLGetConfigOption("USERNAME", nullptr);
1015 0 : if (pszUsername == nullptr)
1016 0 : pszUsername = CPLGetConfigOption("USER", nullptr);
1017 :
1018 0 : if (pszUsername != nullptr)
1019 : {
1020 : return CPLFormFilenameSafe(
1021 : pszDir, CPLSPrintf(".gdal_%s", pszUsername),
1022 0 : nullptr);
1023 : }
1024 : }
1025 : }
1026 0 : return std::string();
1027 0 : }();
1028 0 : if (!osCacheDir.empty())
1029 : {
1030 : VSIStatBufL sStat;
1031 0 : if (VSIStatL(osCacheDir.c_str(), &sStat) != 0)
1032 0 : VSIMkdir(osCacheDir.c_str(), 0755);
1033 : const std::string osCloudCheck = CPLFormFilenameSafe(
1034 0 : osCacheDir.c_str(), "cloud_check_ms.txt", nullptr);
1035 : // Sidereal day, why not? "Aim for the stars, expect dust"
1036 0 : constexpr int ONE_DAY_IN_SECS = 86164;
1037 0 : if (VSIStatL(osCloudCheck.c_str(), &sStat) == 0 &&
1038 0 : sStat.st_mtime + ONE_DAY_IN_SECS >= time(nullptr))
1039 : {
1040 0 : CPLDebugOnly("GDAL", "%s checked", osCloudCheck.c_str());
1041 : }
1042 : else
1043 : {
1044 0 : FILE *f = fopen(osCloudCheck.c_str(), "wb");
1045 0 : if (f)
1046 0 : fclose(f);
1047 :
1048 0 : const auto PingURL = [](const char *pszURL)
1049 : {
1050 0 : CPLErrorStateBackuper oBackuper(CPLQuietErrorHandler);
1051 0 : const char *const apszOptions[] = {
1052 : "CUSTOMREQUEST=HEAD", "TIMEOUT=1", nullptr};
1053 0 : auto res = CPLHTTPFetch(pszURL, apszOptions);
1054 0 : const bool bOK = res && !res->pszErrBuf;
1055 0 : CPLHTTPDestroyResult(res);
1056 0 : return bOK;
1057 : };
1058 0 : if (!PingURL("https://gdal.org/en/latest/sponsors/"
1059 0 : "did_microsoft_sponsor.html") &&
1060 : // check that gdal.org is responding to avoid false positive
1061 0 : PingURL("https://gdal.org/en/latest/index.html"))
1062 : {
1063 0 : const auto CPLE_NonCooperativeSponsor = CPLE_AppDefined;
1064 0 : CPLError(
1065 : CE_Warning, CPLE_NonCooperativeSponsor,
1066 : "Due to lack of resources, Azure Cloud Storage "
1067 : "access is undergoing minimal maintenance and may "
1068 : "be removed in the future unless Microsoft Azure "
1069 : "re-evaluates its decision to stop sponsoring "
1070 : "GDAL. If you are interested in keeping this "
1071 : "functionality please get in touch with your "
1072 : "Microsoft Azure representative.");
1073 : }
1074 : }
1075 : }
1076 0 : return true;
1077 0 : }();
1078 0 : CPL_IGNORE_RET_VAL(bCheckSponsoring);
1079 : }
1080 :
1081 448 : return GetAzureBlobHeaders(osVerb, psHeaders, osResource,
1082 224 : m_oMapQueryParameters, m_osStorageAccount,
1083 224 : m_osStorageKey, m_bIncludeMSVersion);
1084 : }
1085 :
1086 : /************************************************************************/
1087 : /* CanRestartOnError() */
1088 : /************************************************************************/
1089 :
1090 18 : bool VSIAzureBlobHandleHelper::CanRestartOnError(const char *pszErrorMsg,
1091 : const char *pszHeaders,
1092 : bool bSetError)
1093 : {
1094 18 : if (pszErrorMsg[0] == '\xEF' && pszErrorMsg[1] == '\xBB' &&
1095 2 : pszErrorMsg[2] == '\xBF')
1096 2 : pszErrorMsg += 3;
1097 :
1098 : #ifdef DEBUG_VERBOSE
1099 : CPLDebug("AZURE", "%s", pszErrorMsg);
1100 : CPLDebug("AZURE", "%s", pszHeaders ? pszHeaders : "");
1101 : #endif
1102 :
1103 18 : if (STARTS_WITH(pszErrorMsg, "HTTP/") && pszHeaders &&
1104 16 : STARTS_WITH(pszHeaders, "HTTP/"))
1105 : {
1106 16 : if (bSetError)
1107 : {
1108 32 : std::string osMessage;
1109 32 : std::string osTmpMessage(pszHeaders);
1110 16 : auto nPos = osTmpMessage.find(' ');
1111 16 : if (nPos != std::string::npos)
1112 : {
1113 16 : nPos = osTmpMessage.find(' ', nPos + 1);
1114 16 : if (nPos != std::string::npos)
1115 : {
1116 16 : auto nPos2 = osTmpMessage.find('\r', nPos + 1);
1117 16 : if (nPos2 != std::string::npos)
1118 : osMessage =
1119 16 : osTmpMessage.substr(nPos + 1, nPos2 - nPos - 1);
1120 : }
1121 : }
1122 16 : if (strstr(pszHeaders, "x-ms-error-code: BlobNotFound") || // vsiaz
1123 16 : strstr(pszHeaders, "x-ms-error-code: PathNotFound") // vsiadls
1124 : )
1125 : {
1126 0 : VSIError(VSIE_ObjectNotFound, "%s", osMessage.c_str());
1127 : }
1128 16 : else if (strstr(pszHeaders,
1129 16 : "x-ms-error-code: InvalidAuthenticationInfo") ||
1130 16 : strstr(pszHeaders,
1131 : "x-ms-error-code: AuthenticationFailed"))
1132 : {
1133 1 : VSIError(VSIE_InvalidCredentials, "%s", osMessage.c_str());
1134 : }
1135 : // /vsiadls
1136 15 : else if (strstr(pszHeaders, "x-ms-error-code: FilesystemNotFound"))
1137 : {
1138 0 : VSIError(VSIE_BucketNotFound, "%s", osMessage.c_str());
1139 : }
1140 : else
1141 : {
1142 15 : CPLDebug("AZURE", "%s", pszHeaders);
1143 : }
1144 : }
1145 16 : return false;
1146 : }
1147 :
1148 2 : if (!STARTS_WITH(pszErrorMsg, "<?xml") &&
1149 0 : !STARTS_WITH(pszErrorMsg, "<Error>"))
1150 : {
1151 0 : if (bSetError)
1152 : {
1153 0 : VSIError(VSIE_ObjectStorageGenericError,
1154 : "Invalid Azure response: %s", pszErrorMsg);
1155 : }
1156 0 : return false;
1157 : }
1158 :
1159 4 : auto psTree = CPLXMLTreeCloser(CPLParseXMLString(pszErrorMsg));
1160 2 : if (psTree == nullptr)
1161 : {
1162 0 : if (bSetError)
1163 : {
1164 0 : VSIError(VSIE_ObjectStorageGenericError,
1165 : "Malformed Azure XML response: %s", pszErrorMsg);
1166 : }
1167 0 : return false;
1168 : }
1169 :
1170 2 : const char *pszCode = CPLGetXMLValue(psTree.get(), "=Error.Code", nullptr);
1171 2 : if (pszCode == nullptr)
1172 : {
1173 0 : if (bSetError)
1174 : {
1175 0 : VSIError(VSIE_ObjectStorageGenericError,
1176 : "Malformed Azure XML response: %s", pszErrorMsg);
1177 : }
1178 0 : return false;
1179 : }
1180 :
1181 2 : if (bSetError)
1182 : {
1183 : // Translate AWS errors into VSI errors.
1184 : const char *pszMessage =
1185 2 : CPLGetXMLValue(psTree.get(), "=Error.Message", nullptr);
1186 4 : std::string osMessage;
1187 2 : if (pszMessage)
1188 : {
1189 2 : osMessage = pszMessage;
1190 2 : const auto nPos = osMessage.find("\nRequestId:");
1191 2 : if (nPos != std::string::npos)
1192 2 : osMessage.resize(nPos);
1193 : }
1194 :
1195 2 : if (pszMessage == nullptr)
1196 : {
1197 0 : VSIError(VSIE_ObjectStorageGenericError, "%s", pszErrorMsg);
1198 : }
1199 2 : else if (EQUAL(pszCode, "ContainerNotFound"))
1200 : {
1201 0 : VSIError(VSIE_BucketNotFound, "%s", osMessage.c_str());
1202 : }
1203 : else
1204 : {
1205 2 : VSIError(VSIE_ObjectStorageGenericError, "%s: %s", pszCode,
1206 : pszMessage);
1207 : }
1208 : }
1209 :
1210 2 : return false;
1211 : }
1212 :
1213 : /************************************************************************/
1214 : /* GetSignedURL() */
1215 : /************************************************************************/
1216 :
1217 7 : std::string VSIAzureBlobHandleHelper::GetSignedURL(CSLConstList papszOptions)
1218 : {
1219 7 : if (m_osStorageKey.empty())
1220 3 : return m_osURL;
1221 :
1222 8 : std::string osStartDate(CPLGetAWS_SIGN4_Timestamp(time(nullptr)));
1223 4 : const char *pszStartDate = CSLFetchNameValue(papszOptions, "START_DATE");
1224 4 : if (pszStartDate)
1225 2 : osStartDate = pszStartDate;
1226 4 : int nYear, nMonth, nDay, nHour = 0, nMin = 0, nSec = 0;
1227 4 : if (sscanf(osStartDate.c_str(), "%04d%02d%02dT%02d%02d%02dZ", &nYear,
1228 4 : &nMonth, &nDay, &nHour, &nMin, &nSec) < 3)
1229 : {
1230 0 : return std::string();
1231 : }
1232 : osStartDate = CPLSPrintf("%04d-%02d-%02dT%02d:%02d:%02dZ", nYear, nMonth,
1233 4 : nDay, nHour, nMin, nSec);
1234 :
1235 : struct tm brokendowntime;
1236 4 : brokendowntime.tm_year = nYear - 1900;
1237 4 : brokendowntime.tm_mon = nMonth - 1;
1238 4 : brokendowntime.tm_mday = nDay;
1239 4 : brokendowntime.tm_hour = nHour;
1240 4 : brokendowntime.tm_min = nMin;
1241 4 : brokendowntime.tm_sec = nSec;
1242 4 : GIntBig nStartDate = CPLYMDHMSToUnixTime(&brokendowntime);
1243 : GIntBig nEndDate =
1244 : nStartDate +
1245 4 : atoi(CSLFetchNameValueDef(papszOptions, "EXPIRATION_DELAY", "3600"));
1246 4 : CPLUnixTimeToYMDHMS(nEndDate, &brokendowntime);
1247 4 : nYear = brokendowntime.tm_year + 1900;
1248 4 : nMonth = brokendowntime.tm_mon + 1;
1249 4 : nDay = brokendowntime.tm_mday;
1250 4 : nHour = brokendowntime.tm_hour;
1251 4 : nMin = brokendowntime.tm_min;
1252 4 : nSec = brokendowntime.tm_sec;
1253 : std::string osEndDate = CPLSPrintf("%04d-%02d-%02dT%02d:%02d:%02dZ", nYear,
1254 8 : nMonth, nDay, nHour, nMin, nSec);
1255 :
1256 8 : std::string osVerb(CSLFetchNameValueDef(papszOptions, "VERB", "GET"));
1257 : std::string osSignedPermissions(CSLFetchNameValueDef(
1258 : papszOptions, "SIGNEDPERMISSIONS",
1259 4 : (EQUAL(osVerb.c_str(), "GET") || EQUAL(osVerb.c_str(), "HEAD")) ? "r"
1260 12 : : "w"));
1261 :
1262 : std::string osSignedIdentifier(
1263 8 : CSLFetchNameValueDef(papszOptions, "SIGNEDIDENTIFIER", ""));
1264 :
1265 8 : const std::string osSignedVersion("2020-12-06");
1266 8 : const std::string osSignedProtocol("https");
1267 8 : const std::string osSignedResource("b"); // blob
1268 :
1269 8 : std::string osCanonicalizedResource("/blob/");
1270 4 : osCanonicalizedResource += CPLAWSURLEncode(m_osStorageAccount, false);
1271 4 : osCanonicalizedResource += '/';
1272 4 : osCanonicalizedResource += CPLAWSURLEncode(m_osBucket, false);
1273 4 : osCanonicalizedResource += '/';
1274 4 : osCanonicalizedResource += CPLAWSURLEncode(m_osObjectKey, false);
1275 :
1276 : // Cf https://learn.microsoft.com/en-us/rest/api/storageservices/create-service-sas
1277 8 : std::string osStringToSign;
1278 4 : osStringToSign += osSignedPermissions + "\n";
1279 4 : osStringToSign += osStartDate + "\n";
1280 4 : osStringToSign += osEndDate + "\n";
1281 4 : osStringToSign += osCanonicalizedResource + "\n";
1282 4 : osStringToSign += osSignedIdentifier + "\n";
1283 4 : osStringToSign += "\n"; // signedIP
1284 4 : osStringToSign += osSignedProtocol + "\n";
1285 4 : osStringToSign += osSignedVersion + "\n";
1286 4 : osStringToSign += osSignedResource + "\n";
1287 4 : osStringToSign += "\n"; // signedSnapshotTime
1288 4 : osStringToSign += "\n"; // signedEncryptionScope
1289 4 : osStringToSign += "\n"; // rscc
1290 4 : osStringToSign += "\n"; // rscd
1291 4 : osStringToSign += "\n"; // rsce
1292 4 : osStringToSign += "\n"; // rscl
1293 :
1294 : #ifdef DEBUG_VERBOSE
1295 : CPLDebug("AZURE", "osStringToSign = %s", osStringToSign.c_str());
1296 : #endif
1297 :
1298 : /* -------------------------------------------------------------------- */
1299 : /* Compute signature. */
1300 : /* -------------------------------------------------------------------- */
1301 : std::string osSignature(
1302 8 : CPLAzureGetSignature(osStringToSign, m_osStorageKey));
1303 :
1304 4 : ResetQueryParameters();
1305 4 : AddQueryParameter("sv", osSignedVersion);
1306 4 : AddQueryParameter("st", osStartDate);
1307 4 : AddQueryParameter("se", osEndDate);
1308 4 : AddQueryParameter("sr", osSignedResource);
1309 4 : AddQueryParameter("sp", osSignedPermissions);
1310 4 : AddQueryParameter("spr", osSignedProtocol);
1311 4 : AddQueryParameter("sig", osSignature);
1312 4 : if (!osSignedIdentifier.empty())
1313 0 : AddQueryParameter("si", osSignedIdentifier);
1314 4 : return m_osURL;
1315 : }
1316 :
1317 : /************************************************************************/
1318 : /* GetOptions() */
1319 : /************************************************************************/
1320 :
1321 : /* static */
1322 3 : const char *VSIAzureBlobHandleHelper::GetOptions()
1323 : {
1324 : static std::string osOptions(
1325 2 : std::string("<Options>") +
1326 : " <Option name='AZURE_STORAGE_CONNECTION_STRING' type='string' "
1327 : "description='Connection string that contains account name and "
1328 : "secret key'/>"
1329 : " <Option name='AZURE_STORAGE_ACCOUNT' type='string' "
1330 : "description='Storage account. To use with AZURE_STORAGE_ACCESS_KEY'/>"
1331 : " <Option name='AZURE_STORAGE_ACCESS_KEY' type='string' "
1332 : "description='Secret key'/>"
1333 : " <Option name='AZURE_STORAGE_ACCESS_TOKEN' type='string' "
1334 : "description='Access token typically obtained using Microsoft "
1335 : "Authentication Library (MSAL).'/>"
1336 : " <Option name='AZURE_STORAGE_SAS_TOKEN' type='string' "
1337 : "description='Shared Access Signature'/>"
1338 : " <Option name='AZURE_NO_SIGN_REQUEST' type='boolean' "
1339 : "description='Whether to disable signing of requests' default='NO'/>"
1340 : " <Option name='VSIAZ_CHUNK_SIZE' type='int' "
1341 : "description='Size in MB for chunks of files that are uploaded' "
1342 3 : "default='4' min='1' max='4'/>" +
1343 4 : cpl::VSICurlFilesystemHandlerBase::GetOptionsStatic() + "</Options>");
1344 3 : return osOptions.c_str();
1345 : }
1346 :
1347 : #endif // HAVE_CURL
1348 :
1349 : //! @endcond
|
