Line data Source code
1 : /**********************************************************************
2 : * Project: CPL - Common Portability Library
3 : * Purpose: Microsoft Azure Storage Blob routines
4 : * Author: Even Rouault <even.rouault at spatialys.com>
5 : *
6 : **********************************************************************
7 : * Copyright (c) 2017, Even Rouault <even.rouault at spatialys.com>
8 : *
9 : * Permission is hereby granted, free of charge, to any person obtaining a
10 : * copy of this software and associated documentation files (the "Software"),
11 : * to deal in the Software without restriction, including without limitation
12 : * the rights to use, copy, modify, merge, publish, distribute, sublicense,
13 : * and/or sell copies of the Software, and to permit persons to whom the
14 : * Software is furnished to do so, subject to the following conditions:
15 : *
16 : * The above copyright notice and this permission notice shall be included
17 : * in all copies or substantial portions of the Software.
18 : *
19 : * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
20 : * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21 : * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
22 : * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
23 : * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
24 : * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
25 : * DEALINAzureBlob IN THE SOFTWARE.
26 : ****************************************************************************/
27 :
28 : #include "cpl_azure.h"
29 : #include "cpl_json.h"
30 : #include "cpl_vsi_error.h"
31 : #include "cpl_sha256.h"
32 : #include "cpl_time.h"
33 : #include "cpl_http.h"
34 : #include "cpl_multiproc.h"
35 : #include "cpl_vsi_virtual.h"
36 :
37 : #include <mutex>
38 :
39 : //! @cond Doxygen_Suppress
40 :
41 : #ifdef HAVE_CURL
42 :
43 : /************************************************************************/
44 : /* RemoveTrailingSlash() */
45 : /************************************************************************/
46 :
47 560 : static std::string RemoveTrailingSlash(const std::string &osStr)
48 : {
49 560 : std::string osRet(osStr);
50 560 : if (!osRet.empty() && osRet.back() == '/')
51 1 : osRet.pop_back();
52 560 : return osRet;
53 : }
54 :
55 : /************************************************************************/
56 : /* CPLAzureGetSignature() */
57 : /************************************************************************/
58 :
59 205 : static std::string CPLAzureGetSignature(const std::string &osStringToSign,
60 : const std::string &osStorageKeyB64)
61 : {
62 :
63 : /* -------------------------------------------------------------------- */
64 : /* Compute signature. */
65 : /* -------------------------------------------------------------------- */
66 :
67 410 : std::string osStorageKeyUnbase64(osStorageKeyB64);
68 410 : int nB64Length = CPLBase64DecodeInPlace(
69 205 : reinterpret_cast<GByte *>(&osStorageKeyUnbase64[0]));
70 205 : osStorageKeyUnbase64.resize(nB64Length);
71 : #ifdef DEBUG_VERBOSE
72 : CPLDebug("AZURE", "signing key size: %d", nB64Length);
73 : #endif
74 :
75 205 : GByte abySignature[CPL_SHA256_HASH_SIZE] = {};
76 410 : CPL_HMAC_SHA256(osStorageKeyUnbase64.c_str(), nB64Length,
77 205 : osStringToSign.c_str(), osStringToSign.size(),
78 : abySignature);
79 :
80 205 : char *pszB64Signature = CPLBase64Encode(CPL_SHA256_HASH_SIZE, abySignature);
81 205 : std::string osSignature(pszB64Signature);
82 205 : CPLFree(pszB64Signature);
83 410 : return osSignature;
84 : }
85 :
86 : /************************************************************************/
87 : /* GetAzureBlobHeaders() */
88 : /************************************************************************/
89 :
90 209 : static struct curl_slist *GetAzureBlobHeaders(
91 : const std::string &osVerb, const struct curl_slist *psExistingHeaders,
92 : const std::string &osResource,
93 : const std::map<std::string, std::string> &oMapQueryParameters,
94 : const std::string &osStorageAccount, const std::string &osStorageKeyB64,
95 : bool bIncludeMSVersion)
96 : {
97 : /* See
98 : * https://docs.microsoft.com/en-us/rest/api/storageservices/authentication-for-the-azure-storage-services
99 : */
100 :
101 418 : std::string osDate = CPLGetConfigOption("CPL_AZURE_TIMESTAMP", "");
102 209 : if (osDate.empty())
103 : {
104 7 : osDate = IVSIS3LikeHandleHelper::GetRFC822DateTime();
105 : }
106 209 : if (osStorageKeyB64.empty())
107 : {
108 8 : struct curl_slist *headers = nullptr;
109 8 : headers = curl_slist_append(
110 : headers, CPLSPrintf("x-ms-date: %s", osDate.c_str()));
111 8 : return headers;
112 : }
113 :
114 402 : std::string osMsVersion("2019-12-12");
115 402 : std::map<std::string, std::string> oSortedMapMSHeaders;
116 201 : if (bIncludeMSVersion)
117 194 : oSortedMapMSHeaders["x-ms-version"] = osMsVersion;
118 201 : oSortedMapMSHeaders["x-ms-date"] = osDate;
119 : std::string osCanonicalizedHeaders(
120 : IVSIS3LikeHandleHelper::BuildCanonicalizedHeaders(
121 402 : oSortedMapMSHeaders, psExistingHeaders, "x-ms-"));
122 :
123 402 : std::string osCanonicalizedResource;
124 201 : osCanonicalizedResource += "/" + osStorageAccount;
125 201 : osCanonicalizedResource += osResource;
126 :
127 : // We assume query parameters are in lower case and they are not repeated
128 : std::map<std::string, std::string>::const_iterator oIter =
129 201 : oMapQueryParameters.begin();
130 470 : for (; oIter != oMapQueryParameters.end(); ++oIter)
131 : {
132 269 : osCanonicalizedResource += "\n";
133 269 : osCanonicalizedResource += oIter->first;
134 269 : osCanonicalizedResource += ":";
135 269 : osCanonicalizedResource += oIter->second;
136 : }
137 :
138 402 : std::string osStringToSign;
139 201 : osStringToSign += osVerb + "\n";
140 : osStringToSign +=
141 201 : CPLAWSGetHeaderVal(psExistingHeaders, "Content-Encoding") + "\n";
142 : osStringToSign +=
143 201 : CPLAWSGetHeaderVal(psExistingHeaders, "Content-Language") + "\n";
144 : std::string osContentLength(
145 402 : CPLAWSGetHeaderVal(psExistingHeaders, "Content-Length"));
146 201 : if (osContentLength == "0")
147 34 : osContentLength.clear(); // since x-ms-version 2015-02-21
148 201 : osStringToSign += osContentLength + "\n";
149 : osStringToSign +=
150 201 : CPLAWSGetHeaderVal(psExistingHeaders, "Content-MD5") + "\n";
151 : osStringToSign +=
152 201 : CPLAWSGetHeaderVal(psExistingHeaders, "Content-Type") + "\n";
153 201 : osStringToSign += CPLAWSGetHeaderVal(psExistingHeaders, "Date") + "\n";
154 : osStringToSign +=
155 201 : CPLAWSGetHeaderVal(psExistingHeaders, "If-Modified-Since") + "\n";
156 201 : osStringToSign += CPLAWSGetHeaderVal(psExistingHeaders, "If-Match") + "\n";
157 : osStringToSign +=
158 201 : CPLAWSGetHeaderVal(psExistingHeaders, "If-None-Match") + "\n";
159 : osStringToSign +=
160 201 : CPLAWSGetHeaderVal(psExistingHeaders, "If-Unmodified-Since") + "\n";
161 201 : osStringToSign += CPLAWSGetHeaderVal(psExistingHeaders, "Range") + "\n";
162 201 : osStringToSign += osCanonicalizedHeaders;
163 201 : osStringToSign += osCanonicalizedResource;
164 :
165 : #ifdef DEBUG_VERBOSE
166 : CPLDebug("AZURE", "osStringToSign = '%s'", osStringToSign.c_str());
167 : #endif
168 :
169 : /* -------------------------------------------------------------------- */
170 : /* Compute signature. */
171 : /* -------------------------------------------------------------------- */
172 :
173 : std::string osAuthorization(
174 402 : "SharedKey " + osStorageAccount + ":" +
175 402 : CPLAzureGetSignature(osStringToSign, osStorageKeyB64));
176 :
177 201 : struct curl_slist *headers = nullptr;
178 : headers =
179 201 : curl_slist_append(headers, CPLSPrintf("x-ms-date: %s", osDate.c_str()));
180 201 : if (bIncludeMSVersion)
181 : {
182 194 : headers = curl_slist_append(
183 : headers, CPLSPrintf("x-ms-version: %s", osMsVersion.c_str()));
184 : }
185 201 : headers = curl_slist_append(
186 : headers, CPLSPrintf("Authorization: %s", osAuthorization.c_str()));
187 201 : return headers;
188 : }
189 :
190 : /************************************************************************/
191 : /* VSIAzureBlobHandleHelper() */
192 : /************************************************************************/
193 291 : VSIAzureBlobHandleHelper::VSIAzureBlobHandleHelper(
194 : const std::string &osPathForOption, const std::string &osEndpoint,
195 : const std::string &osBucket, const std::string &osObjectKey,
196 : const std::string &osStorageAccount, const std::string &osStorageKey,
197 : const std::string &osSAS, const std::string &osAccessToken,
198 291 : bool bFromManagedIdentities)
199 : : m_osPathForOption(osPathForOption),
200 : m_osURL(BuildURL(osEndpoint, osBucket, osObjectKey, osSAS)),
201 : m_osEndpoint(osEndpoint), m_osBucket(osBucket),
202 : m_osObjectKey(osObjectKey), m_osStorageAccount(osStorageAccount),
203 : m_osStorageKey(osStorageKey), m_osSAS(osSAS),
204 : m_osAccessToken(osAccessToken),
205 291 : m_bFromManagedIdentities(bFromManagedIdentities)
206 : {
207 291 : }
208 :
209 : /************************************************************************/
210 : /* ~VSIAzureBlobHandleHelper() */
211 : /************************************************************************/
212 :
213 582 : VSIAzureBlobHandleHelper::~VSIAzureBlobHandleHelper()
214 : {
215 582 : }
216 :
217 : /************************************************************************/
218 : /* AzureCSGetParameter() */
219 : /************************************************************************/
220 :
221 1047 : static std::string AzureCSGetParameter(const std::string &osStr,
222 : const char *pszKey, bool bErrorIfMissing)
223 : {
224 3141 : std::string osKey(pszKey + std::string("="));
225 1047 : size_t nPos = osStr.find(osKey);
226 1047 : if (nPos == std::string::npos)
227 : {
228 : const char *pszMsg =
229 11 : CPLSPrintf("%s missing in AZURE_STORAGE_CONNECTION_STRING", pszKey);
230 11 : if (bErrorIfMissing)
231 : {
232 0 : CPLDebug("AZURE", "%s", pszMsg);
233 0 : VSIError(VSIE_AWSInvalidCredentials, "%s", pszMsg);
234 : }
235 11 : return std::string();
236 : }
237 1036 : size_t nPos2 = osStr.find(";", nPos);
238 1036 : return osStr.substr(nPos + osKey.size(), nPos2 == std::string::npos
239 : ? nPos2
240 2072 : : nPos2 - nPos - osKey.size());
241 : }
242 :
243 : /************************************************************************/
244 : /* CPLAzureCachedToken */
245 : /************************************************************************/
246 :
247 : std::mutex gMutex;
248 :
249 : struct CPLAzureCachedToken
250 : {
251 : std::string osAccessToken{};
252 : GIntBig nExpiresOn = 0;
253 : };
254 :
255 : static std::map<std::string, CPLAzureCachedToken> goMapIMDSURLToCachedToken;
256 :
257 : /************************************************************************/
258 : /* GetConfigurationFromIMDSCredentials() */
259 : /************************************************************************/
260 :
261 : static bool
262 15 : GetConfigurationFromIMDSCredentials(const std::string &osPathForOption,
263 : std::string &osAccessToken)
264 : {
265 : // coverity[tainted_data]
266 : const std::string osRootURL(CPLGetConfigOption("CPL_AZURE_VM_API_ROOT_URL",
267 30 : "http://169.254.169.254"));
268 15 : if (osRootURL == "disabled")
269 1 : return false;
270 :
271 : std::string osURLResource("/metadata/identity/oauth2/"
272 : "token?api-version=2018-02-01&resource=https%"
273 28 : "3A%2F%2Fstorage.azure.com%2F");
274 14 : const char *pszObjectId = VSIGetPathSpecificOption(
275 : osPathForOption.c_str(), "AZURE_IMDS_OBJECT_ID", nullptr);
276 14 : if (pszObjectId)
277 8 : osURLResource += "&object_id=" + CPLAWSURLEncode(pszObjectId, false);
278 14 : const char *pszClientId = VSIGetPathSpecificOption(
279 : osPathForOption.c_str(), "AZURE_IMDS_CLIENT_ID", nullptr);
280 14 : if (pszClientId)
281 8 : osURLResource += "&client_id=" + CPLAWSURLEncode(pszClientId, false);
282 14 : const char *pszMsiResId = VSIGetPathSpecificOption(
283 : osPathForOption.c_str(), "AZURE_IMDS_MSI_RES_ID", nullptr);
284 14 : if (pszMsiResId)
285 8 : osURLResource += "&msi_res_id=" + CPLAWSURLEncode(pszMsiResId, false);
286 :
287 28 : std::lock_guard<std::mutex> guard(gMutex);
288 :
289 : // Look for cached token corresponding to this IMDS request URL
290 14 : auto oIter = goMapIMDSURLToCachedToken.find(osURLResource);
291 14 : if (oIter != goMapIMDSURLToCachedToken.end())
292 : {
293 10 : const auto &oCachedToken = oIter->second;
294 : time_t nCurTime;
295 10 : time(&nCurTime);
296 : // Try to reuse credentials if they are still valid, but
297 : // keep one minute of margin...
298 10 : if (nCurTime < oCachedToken.nExpiresOn - 60)
299 : {
300 9 : osAccessToken = oCachedToken.osAccessToken;
301 9 : return true;
302 : }
303 : }
304 :
305 : // Fetch credentials
306 5 : CPLStringList oResponse;
307 5 : const char *const apszOptions[] = {"HEADERS=Metadata: true", nullptr};
308 : CPLHTTPResult *psResult =
309 5 : CPLHTTPFetch((osRootURL + osURLResource).c_str(), apszOptions);
310 5 : if (psResult)
311 : {
312 5 : if (psResult->nStatus == 0 && psResult->pabyData != nullptr)
313 : {
314 : const std::string osJSon =
315 10 : reinterpret_cast<char *>(psResult->pabyData);
316 5 : oResponse = CPLParseKeyValueJson(osJSon.c_str());
317 5 : if (oResponse.FetchNameValue("error"))
318 : {
319 0 : CPLDebug("AZURE",
320 : "Cannot retrieve managed identities credentials: %s",
321 : osJSon.c_str());
322 : }
323 : }
324 5 : CPLHTTPDestroyResult(psResult);
325 : }
326 5 : osAccessToken = oResponse.FetchNameValueDef("access_token", "");
327 : const GIntBig nExpiresOn =
328 5 : CPLAtoGIntBig(oResponse.FetchNameValueDef("expires_on", ""));
329 5 : if (!osAccessToken.empty() && nExpiresOn > 0)
330 : {
331 10 : CPLAzureCachedToken cachedToken;
332 5 : cachedToken.osAccessToken = osAccessToken;
333 5 : cachedToken.nExpiresOn = nExpiresOn;
334 5 : goMapIMDSURLToCachedToken[osURLResource] = std::move(cachedToken);
335 5 : CPLDebug("AZURE", "Storing credentials for %s until " CPL_FRMT_GIB,
336 : osURLResource.c_str(), nExpiresOn);
337 : }
338 :
339 5 : return !osAccessToken.empty();
340 : }
341 :
342 : /************************************************************************/
343 : /* GetConfigurationFromWorkloadIdentity() */
344 : /************************************************************************/
345 :
346 : // Last timestamp AZURE_FEDERATED_TOKEN_FILE was read
347 : static GIntBig gnLastReadFederatedTokenFile = 0;
348 : static std::string gosFederatedToken{};
349 :
350 : // Azure Active Directory Workload Identity, typically for Azure Kubernetes
351 : // Cf https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/identity/azure-identity/azure/identity/_credentials/workload_identity.py
352 23 : static bool GetConfigurationFromWorkloadIdentity(std::string &osAccessToken)
353 : {
354 : const std::string AZURE_CLIENT_ID(
355 46 : CPLGetConfigOption("AZURE_CLIENT_ID", ""));
356 : const std::string AZURE_TENANT_ID(
357 46 : CPLGetConfigOption("AZURE_TENANT_ID", ""));
358 : const std::string AZURE_AUTHORITY_HOST(
359 46 : CPLGetConfigOption("AZURE_AUTHORITY_HOST", ""));
360 : const std::string AZURE_FEDERATED_TOKEN_FILE(
361 46 : CPLGetConfigOption("AZURE_FEDERATED_TOKEN_FILE", ""));
362 39 : if (AZURE_CLIENT_ID.empty() || AZURE_TENANT_ID.empty() ||
363 39 : AZURE_AUTHORITY_HOST.empty() || AZURE_FEDERATED_TOKEN_FILE.empty())
364 : {
365 15 : return false;
366 : }
367 :
368 16 : std::lock_guard<std::mutex> guard(gMutex);
369 :
370 : time_t nCurTime;
371 8 : time(&nCurTime);
372 :
373 : // Look for cached token corresponding to this request URL
374 8 : const std::string osURL(AZURE_AUTHORITY_HOST + AZURE_TENANT_ID +
375 16 : "/oauth2/v2.0/token");
376 8 : auto oIter = goMapIMDSURLToCachedToken.find(osURL);
377 8 : if (oIter != goMapIMDSURLToCachedToken.end())
378 : {
379 6 : const auto &oCachedToken = oIter->second;
380 : // Try to reuse credentials if they are still valid, but
381 : // keep one minute of margin...
382 6 : if (nCurTime < oCachedToken.nExpiresOn - 60)
383 : {
384 4 : osAccessToken = oCachedToken.osAccessToken;
385 4 : return true;
386 : }
387 : }
388 :
389 : // Ingest content of AZURE_FEDERATED_TOKEN_FILE if last time was more than
390 : // 600 seconds.
391 4 : if (nCurTime - gnLastReadFederatedTokenFile > 600)
392 : {
393 : auto fp = VSIVirtualHandleUniquePtr(
394 2 : VSIFOpenL(AZURE_FEDERATED_TOKEN_FILE.c_str(), "rb"));
395 2 : if (!fp)
396 : {
397 0 : CPLDebug("AZURE", "Cannot open AZURE_FEDERATED_TOKEN_FILE = %s",
398 : AZURE_FEDERATED_TOKEN_FILE.c_str());
399 0 : return false;
400 : }
401 2 : fp->Seek(0, SEEK_END);
402 2 : const auto nSize = fp->Tell();
403 2 : if (nSize == 0 || nSize > 100 * 1024)
404 : {
405 0 : CPLDebug(
406 : "AZURE",
407 : "Invalid size for AZURE_FEDERATED_TOKEN_FILE = " CPL_FRMT_GUIB,
408 : static_cast<GUIntBig>(nSize));
409 0 : return false;
410 : }
411 2 : fp->Seek(0, SEEK_SET);
412 2 : gosFederatedToken.resize(static_cast<size_t>(nSize));
413 2 : if (fp->Read(&gosFederatedToken[0], gosFederatedToken.size(), 1) != 1)
414 : {
415 0 : CPLDebug("AZURE", "Cannot read AZURE_FEDERATED_TOKEN_FILE");
416 0 : return false;
417 : }
418 2 : gnLastReadFederatedTokenFile = nCurTime;
419 : }
420 :
421 : /* -------------------------------------------------------------------- */
422 : /* Prepare POST request. */
423 : /* -------------------------------------------------------------------- */
424 8 : CPLStringList aosOptions;
425 :
426 : aosOptions.AddString(
427 4 : "HEADERS=Content-Type: application/x-www-form-urlencoded");
428 :
429 8 : std::string osItem("POSTFIELDS=client_assertion=");
430 4 : osItem += CPLAWSURLEncode(gosFederatedToken);
431 : osItem += "&client_assertion_type=urn:ietf:params:oauth:client-assertion-"
432 4 : "type:jwt-bearer";
433 4 : osItem += "&client_id=";
434 4 : osItem += CPLAWSURLEncode(AZURE_CLIENT_ID);
435 4 : osItem += "&grant_type=client_credentials";
436 4 : osItem += "&scope=https://storage.azure.com/.default";
437 4 : aosOptions.AddString(osItem.c_str());
438 :
439 : /* -------------------------------------------------------------------- */
440 : /* Submit request by HTTP. */
441 : /* -------------------------------------------------------------------- */
442 4 : CPLHTTPResult *psResult = CPLHTTPFetch(osURL.c_str(), aosOptions.List());
443 4 : if (!psResult)
444 0 : return false;
445 :
446 4 : if (!psResult->pabyData || psResult->pszErrBuf)
447 : {
448 0 : if (psResult->pszErrBuf)
449 0 : CPLDebug("AZURE", "%s", psResult->pszErrBuf);
450 0 : if (psResult->pabyData)
451 0 : CPLDebug("AZURE", "%s", psResult->pabyData);
452 :
453 0 : CPLDebug("AZURE",
454 : "Fetching OAuth2 access code from workload identity failed.");
455 0 : CPLHTTPDestroyResult(psResult);
456 0 : return false;
457 : }
458 :
459 : CPLStringList oResponse =
460 4 : CPLParseKeyValueJson(reinterpret_cast<char *>(psResult->pabyData));
461 4 : CPLHTTPDestroyResult(psResult);
462 :
463 4 : osAccessToken = oResponse.FetchNameValueDef("access_token", "");
464 4 : const int nExpiresIn = atoi(oResponse.FetchNameValueDef("expires_in", ""));
465 4 : if (!osAccessToken.empty() && nExpiresIn > 0)
466 : {
467 8 : CPLAzureCachedToken cachedToken;
468 4 : cachedToken.osAccessToken = osAccessToken;
469 4 : cachedToken.nExpiresOn = nCurTime + nExpiresIn;
470 4 : goMapIMDSURLToCachedToken[osURL] = cachedToken;
471 4 : CPLDebug("AZURE", "Storing credentials for %s until " CPL_FRMT_GIB,
472 : osURL.c_str(), cachedToken.nExpiresOn);
473 : }
474 :
475 4 : return !osAccessToken.empty();
476 : }
477 :
478 : /************************************************************************/
479 : /* GetConfigurationFromManagedIdentities() */
480 : /************************************************************************/
481 :
482 : static bool
483 23 : GetConfigurationFromManagedIdentities(const std::string &osPathForOption,
484 : std::string &osAccessToken)
485 : {
486 23 : if (GetConfigurationFromWorkloadIdentity(osAccessToken))
487 8 : return true;
488 15 : return GetConfigurationFromIMDSCredentials(osPathForOption, osAccessToken);
489 : }
490 :
491 : /************************************************************************/
492 : /* ClearCache() */
493 : /************************************************************************/
494 :
495 598 : void VSIAzureBlobHandleHelper::ClearCache()
496 : {
497 1196 : std::lock_guard<std::mutex> guard(gMutex);
498 598 : goMapIMDSURLToCachedToken.clear();
499 598 : gnLastReadFederatedTokenFile = 0;
500 598 : gosFederatedToken.clear();
501 598 : }
502 :
503 : /************************************************************************/
504 : /* ParseStorageConnectionString() */
505 : /************************************************************************/
506 :
507 : static bool
508 261 : ParseStorageConnectionString(const std::string &osStorageConnectionString,
509 : const std::string &osServicePrefix,
510 : bool &bUseHTTPS, std::string &osEndpoint,
511 : std::string &osStorageAccount,
512 : std::string &osStorageKey, std::string &osSAS)
513 : {
514 : osStorageAccount =
515 261 : AzureCSGetParameter(osStorageConnectionString, "AccountName", false);
516 : osStorageKey =
517 261 : AzureCSGetParameter(osStorageConnectionString, "AccountKey", false);
518 :
519 : const std::string osProtocol(AzureCSGetParameter(
520 522 : osStorageConnectionString, "DefaultEndpointsProtocol", false));
521 261 : bUseHTTPS = (osProtocol != "http");
522 :
523 261 : if (osStorageAccount.empty() || osStorageKey.empty())
524 : {
525 3 : osStorageAccount.clear();
526 3 : osStorageKey.clear();
527 :
528 : const std::string osBlobEndpoint =
529 3 : RemoveTrailingSlash(AzureCSGetParameter(osStorageConnectionString,
530 6 : "BlobEndpoint", false));
531 6 : osSAS = AzureCSGetParameter(osStorageConnectionString,
532 3 : "SharedAccessSignature", false);
533 3 : if (!osBlobEndpoint.empty() && !osSAS.empty())
534 : {
535 2 : osEndpoint = osBlobEndpoint;
536 2 : return true;
537 : }
538 :
539 1 : return false;
540 : }
541 :
542 : const std::string osBlobEndpoint =
543 258 : AzureCSGetParameter(osStorageConnectionString, "BlobEndpoint", false);
544 258 : if (!osBlobEndpoint.empty())
545 : {
546 258 : osEndpoint = RemoveTrailingSlash(osBlobEndpoint);
547 : }
548 : else
549 : {
550 : const std::string osEndpointSuffix(AzureCSGetParameter(
551 0 : osStorageConnectionString, "EndpointSuffix", false));
552 0 : if (!osEndpointSuffix.empty())
553 0 : osEndpoint = (bUseHTTPS ? "https://" : "http://") +
554 0 : osStorageAccount + "." + osServicePrefix + "." +
555 0 : RemoveTrailingSlash(osEndpointSuffix);
556 : }
557 :
558 258 : return true;
559 : }
560 :
561 : /************************************************************************/
562 : /* GetConfigurationFromCLIConfigFile() */
563 : /************************************************************************/
564 :
565 9 : static bool GetConfigurationFromCLIConfigFile(
566 : const std::string &osPathForOption, const std::string &osServicePrefix,
567 : bool &bUseHTTPS, std::string &osEndpoint, std::string &osStorageAccount,
568 : std::string &osStorageKey, std::string &osSAS, std::string &osAccessToken,
569 : bool &bFromManagedIdentities)
570 : {
571 : #ifdef _WIN32
572 : const char *pszHome = CPLGetConfigOption("USERPROFILE", nullptr);
573 : constexpr char SEP_STRING[] = "\\";
574 : #else
575 9 : const char *pszHome = CPLGetConfigOption("HOME", nullptr);
576 9 : constexpr char SEP_STRING[] = "/";
577 : #endif
578 :
579 18 : std::string osDotAzure(pszHome ? pszHome : "");
580 9 : osDotAzure += SEP_STRING;
581 9 : osDotAzure += ".azure";
582 :
583 : const char *pszAzureConfigDir =
584 9 : CPLGetConfigOption("AZURE_CONFIG_DIR", osDotAzure.c_str());
585 9 : if (pszAzureConfigDir[0] == '\0')
586 5 : return false;
587 :
588 8 : std::string osConfigFilename = pszAzureConfigDir;
589 4 : osConfigFilename += SEP_STRING;
590 4 : osConfigFilename += "config";
591 :
592 4 : VSILFILE *fp = VSIFOpenL(osConfigFilename.c_str(), "rb");
593 8 : std::string osStorageConnectionString;
594 4 : if (fp == nullptr)
595 0 : return false;
596 :
597 4 : bool bInStorageSection = false;
598 25 : while (const char *pszLine = CPLReadLineL(fp))
599 : {
600 21 : if (pszLine[0] == '#' || pszLine[0] == ';')
601 : {
602 : // comment line
603 : }
604 21 : else if (strcmp(pszLine, "[storage]") == 0)
605 : {
606 4 : bInStorageSection = true;
607 : }
608 17 : else if (pszLine[0] == '[')
609 : {
610 4 : bInStorageSection = false;
611 : }
612 13 : else if (bInStorageSection)
613 : {
614 5 : char *pszKey = nullptr;
615 5 : const char *pszValue = CPLParseNameValue(pszLine, &pszKey);
616 5 : if (pszKey && pszValue)
617 : {
618 5 : if (EQUAL(pszKey, "account"))
619 : {
620 2 : osStorageAccount = pszValue;
621 : }
622 3 : else if (EQUAL(pszKey, "connection_string"))
623 : {
624 1 : osStorageConnectionString = pszValue;
625 : }
626 2 : else if (EQUAL(pszKey, "key"))
627 : {
628 1 : osStorageKey = pszValue;
629 : }
630 1 : else if (EQUAL(pszKey, "sas_token"))
631 : {
632 1 : osSAS = pszValue;
633 : // Az CLI apparently uses configparser with
634 : // BasicInterpolation where the % character has a special
635 : // meaning See
636 : // https://docs.python.org/3/library/configparser.html#configparser.BasicInterpolation
637 : // A token might end with %%3D which must be transformed to
638 : // %3D
639 1 : osSAS = CPLString(osSAS).replaceAll("%%", '%');
640 : }
641 : }
642 5 : CPLFree(pszKey);
643 : }
644 21 : }
645 4 : VSIFCloseL(fp);
646 :
647 4 : if (!osStorageConnectionString.empty())
648 : {
649 1 : return ParseStorageConnectionString(
650 : osStorageConnectionString, osServicePrefix, bUseHTTPS, osEndpoint,
651 1 : osStorageAccount, osStorageKey, osSAS);
652 : }
653 :
654 3 : if (osStorageAccount.empty())
655 : {
656 1 : CPLDebug("AZURE", "Missing storage.account in %s",
657 : osConfigFilename.c_str());
658 1 : return false;
659 : }
660 :
661 2 : if (osEndpoint.empty())
662 0 : osEndpoint = (bUseHTTPS ? "https://" : "http://") + osStorageAccount +
663 0 : "." + osServicePrefix + ".core.windows.net";
664 :
665 2 : osAccessToken = CPLGetConfigOption("AZURE_STORAGE_ACCESS_TOKEN", "");
666 2 : if (!osAccessToken.empty())
667 0 : return true;
668 :
669 2 : if (osStorageKey.empty() && osSAS.empty())
670 : {
671 0 : if (CPLTestBool(CPLGetConfigOption("AZURE_NO_SIGN_REQUEST", "NO")))
672 : {
673 0 : return true;
674 : }
675 :
676 0 : std::string osTmpAccessToken;
677 0 : if (GetConfigurationFromManagedIdentities(osPathForOption,
678 : osTmpAccessToken))
679 : {
680 0 : bFromManagedIdentities = true;
681 0 : return true;
682 : }
683 :
684 0 : CPLDebug("AZURE", "Missing storage.key or storage.sas_token in %s",
685 : osConfigFilename.c_str());
686 0 : return false;
687 : }
688 :
689 2 : return true;
690 : }
691 :
692 : /************************************************************************/
693 : /* GetConfiguration() */
694 : /************************************************************************/
695 :
696 299 : bool VSIAzureBlobHandleHelper::GetConfiguration(
697 : const std::string &osPathForOption, CSLConstList papszOptions,
698 : Service eService, bool &bUseHTTPS, std::string &osEndpoint,
699 : std::string &osStorageAccount, std::string &osStorageKey,
700 : std::string &osSAS, std::string &osAccessToken,
701 : bool &bFromManagedIdentities)
702 : {
703 299 : bFromManagedIdentities = false;
704 :
705 : const std::string osServicePrefix(
706 598 : eService == Service::SERVICE_BLOB ? "blob" : "dfs");
707 299 : bUseHTTPS = CPLTestBool(VSIGetPathSpecificOption(
708 : osPathForOption.c_str(), "CPL_AZURE_USE_HTTPS", "YES"));
709 598 : osEndpoint = RemoveTrailingSlash(VSIGetPathSpecificOption(
710 299 : osPathForOption.c_str(), "CPL_AZURE_ENDPOINT", ""));
711 :
712 : const std::string osStorageConnectionString(CSLFetchNameValueDef(
713 : papszOptions, "AZURE_STORAGE_CONNECTION_STRING",
714 : VSIGetPathSpecificOption(osPathForOption.c_str(),
715 598 : "AZURE_STORAGE_CONNECTION_STRING", "")));
716 299 : if (!osStorageConnectionString.empty())
717 : {
718 260 : return ParseStorageConnectionString(
719 : osStorageConnectionString, osServicePrefix, bUseHTTPS, osEndpoint,
720 260 : osStorageAccount, osStorageKey, osSAS);
721 : }
722 : else
723 : {
724 : osStorageAccount = CSLFetchNameValueDef(
725 : papszOptions, "AZURE_STORAGE_ACCOUNT",
726 : VSIGetPathSpecificOption(osPathForOption.c_str(),
727 39 : "AZURE_STORAGE_ACCOUNT", ""));
728 39 : if (!osStorageAccount.empty())
729 : {
730 30 : if (osEndpoint.empty())
731 30 : osEndpoint = (bUseHTTPS ? "https://" : "http://") +
732 30 : osStorageAccount + "." + osServicePrefix +
733 15 : ".core.windows.net";
734 :
735 : osAccessToken = CSLFetchNameValueDef(
736 : papszOptions, "AZURE_STORAGE_ACCESS_TOKEN",
737 : VSIGetPathSpecificOption(osPathForOption.c_str(),
738 30 : "AZURE_STORAGE_ACCESS_TOKEN", ""));
739 30 : if (!osAccessToken.empty())
740 1 : return true;
741 :
742 : osStorageKey = CSLFetchNameValueDef(
743 : papszOptions, "AZURE_STORAGE_ACCESS_KEY",
744 : VSIGetPathSpecificOption(osPathForOption.c_str(),
745 29 : "AZURE_STORAGE_ACCESS_KEY", ""));
746 29 : if (osStorageKey.empty())
747 : {
748 : osSAS = VSIGetPathSpecificOption(
749 : osPathForOption.c_str(), "AZURE_STORAGE_SAS_TOKEN",
750 : CPLGetConfigOption("AZURE_SAS",
751 25 : "")); // AZURE_SAS for GDAL < 3.5
752 25 : if (osSAS.empty())
753 : {
754 17 : if (CPLTestBool(VSIGetPathSpecificOption(
755 : osPathForOption.c_str(), "AZURE_NO_SIGN_REQUEST",
756 : "NO")))
757 : {
758 5 : return true;
759 : }
760 :
761 24 : std::string osTmpAccessToken;
762 12 : if (GetConfigurationFromManagedIdentities(osPathForOption,
763 : osTmpAccessToken))
764 : {
765 11 : bFromManagedIdentities = true;
766 11 : return true;
767 : }
768 :
769 1 : const char *pszMsg =
770 : "AZURE_STORAGE_ACCESS_KEY or AZURE_STORAGE_SAS_TOKEN "
771 : "or AZURE_NO_SIGN_REQUEST configuration option "
772 : "not defined";
773 1 : CPLDebug("AZURE", "%s", pszMsg);
774 1 : VSIError(VSIE_AWSInvalidCredentials, "%s", pszMsg);
775 1 : return false;
776 : }
777 : }
778 12 : return true;
779 : }
780 : }
781 :
782 9 : if (GetConfigurationFromCLIConfigFile(
783 : osPathForOption, osServicePrefix, bUseHTTPS, osEndpoint,
784 : osStorageAccount, osStorageKey, osSAS, osAccessToken,
785 : bFromManagedIdentities))
786 : {
787 3 : return true;
788 : }
789 :
790 6 : const char *pszMsg =
791 : "Missing AZURE_STORAGE_ACCOUNT+"
792 : "(AZURE_STORAGE_ACCESS_KEY or AZURE_STORAGE_SAS_TOKEN or "
793 : "AZURE_NO_SIGN_REQUEST) or "
794 : "AZURE_STORAGE_CONNECTION_STRING "
795 : "configuration options or Azure CLI configuration file";
796 6 : CPLDebug("AZURE", "%s", pszMsg);
797 6 : VSIError(VSIE_AWSInvalidCredentials, "%s", pszMsg);
798 6 : return false;
799 : }
800 :
801 : /************************************************************************/
802 : /* BuildFromURI() */
803 : /************************************************************************/
804 :
805 299 : VSIAzureBlobHandleHelper *VSIAzureBlobHandleHelper::BuildFromURI(
806 : const char *pszURI, const char *pszFSPrefix,
807 : const char *pszURIForPathSpecificOption, CSLConstList papszOptions)
808 : {
809 299 : if (strcmp(pszFSPrefix, "/vsiaz/") != 0 &&
810 97 : strcmp(pszFSPrefix, "/vsiaz_streaming/") != 0 &&
811 91 : strcmp(pszFSPrefix, "/vsiadls/") != 0)
812 : {
813 0 : CPLError(CE_Failure, CPLE_AppDefined, "Unsupported FS prefix");
814 0 : return nullptr;
815 : }
816 :
817 598 : const auto eService = strcmp(pszFSPrefix, "/vsiaz/") == 0 ||
818 97 : strcmp(pszFSPrefix, "/vsiaz_streaming/") == 0
819 299 : ? Service::SERVICE_BLOB
820 : : Service::SERVICE_ADLS;
821 :
822 : std::string osPathForOption(
823 598 : eService == Service::SERVICE_BLOB ? "/vsiaz/" : "/vsiadls/");
824 : osPathForOption +=
825 299 : pszURIForPathSpecificOption ? pszURIForPathSpecificOption : pszURI;
826 :
827 299 : bool bUseHTTPS = true;
828 598 : std::string osStorageAccount;
829 598 : std::string osStorageKey;
830 598 : std::string osEndpoint;
831 598 : std::string osSAS;
832 598 : std::string osAccessToken;
833 299 : bool bFromManagedIdentities = false;
834 :
835 299 : if (!GetConfiguration(osPathForOption, papszOptions, eService, bUseHTTPS,
836 : osEndpoint, osStorageAccount, osStorageKey, osSAS,
837 : osAccessToken, bFromManagedIdentities))
838 : {
839 8 : return nullptr;
840 : }
841 :
842 291 : if (CPLTestBool(VSIGetPathSpecificOption(osPathForOption.c_str(),
843 : "AZURE_NO_SIGN_REQUEST", "NO")))
844 : {
845 6 : osStorageKey.clear();
846 6 : osSAS.clear();
847 6 : osAccessToken.clear();
848 : }
849 :
850 : // pszURI == bucket/object
851 582 : const std::string osBucketObject(pszURI);
852 582 : std::string osBucket(osBucketObject);
853 291 : std::string osObjectKey;
854 291 : size_t nSlashPos = osBucketObject.find('/');
855 291 : if (nSlashPos != std::string::npos)
856 : {
857 210 : osBucket = osBucketObject.substr(0, nSlashPos);
858 210 : osObjectKey = osBucketObject.substr(nSlashPos + 1);
859 : }
860 :
861 : return new VSIAzureBlobHandleHelper(
862 : osPathForOption, osEndpoint, osBucket, osObjectKey, osStorageAccount,
863 291 : osStorageKey, osSAS, osAccessToken, bFromManagedIdentities);
864 : }
865 :
866 : /************************************************************************/
867 : /* BuildURL() */
868 : /************************************************************************/
869 :
870 832 : std::string VSIAzureBlobHandleHelper::BuildURL(const std::string &osEndpoint,
871 : const std::string &osBucket,
872 : const std::string &osObjectKey,
873 : const std::string &osSAS)
874 : {
875 832 : std::string osURL = osEndpoint;
876 832 : osURL += "/";
877 832 : osURL += CPLAWSURLEncode(osBucket, false);
878 832 : if (!osObjectKey.empty())
879 450 : osURL += "/" + CPLAWSURLEncode(osObjectKey, false);
880 832 : if (!osSAS.empty())
881 11 : osURL += '?' + osSAS;
882 832 : return osURL;
883 : }
884 :
885 : /************************************************************************/
886 : /* RebuildURL() */
887 : /************************************************************************/
888 :
889 541 : void VSIAzureBlobHandleHelper::RebuildURL()
890 : {
891 541 : m_osURL = BuildURL(m_osEndpoint, m_osBucket, m_osObjectKey, std::string());
892 541 : m_osURL += GetQueryString(false);
893 541 : if (!m_osSAS.empty())
894 11 : m_osURL += (m_oMapQueryParameters.empty() ? '?' : '&') + m_osSAS;
895 541 : }
896 :
897 : /************************************************************************/
898 : /* GetSASQueryString() */
899 : /************************************************************************/
900 :
901 72 : std::string VSIAzureBlobHandleHelper::GetSASQueryString() const
902 : {
903 72 : if (!m_osSAS.empty())
904 4 : return '?' + m_osSAS;
905 68 : return std::string();
906 : }
907 :
908 : /************************************************************************/
909 : /* GetCurlHeaders() */
910 : /************************************************************************/
911 :
912 221 : struct curl_slist *VSIAzureBlobHandleHelper::GetCurlHeaders(
913 : const std::string &osVerb, const struct curl_slist *psExistingHeaders,
914 : const void *, size_t) const
915 : {
916 221 : if (m_bFromManagedIdentities || !m_osAccessToken.empty())
917 : {
918 24 : std::string osAccessToken;
919 12 : if (m_bFromManagedIdentities)
920 : {
921 11 : if (!GetConfigurationFromManagedIdentities(m_osPathForOption,
922 : osAccessToken))
923 0 : return nullptr;
924 : }
925 : else
926 : {
927 1 : osAccessToken = m_osAccessToken;
928 : }
929 :
930 12 : struct curl_slist *headers = nullptr;
931 :
932 : // Do not use CPLSPrintf() as we could get over the 8K character limit
933 : // with very large SAS tokens
934 12 : std::string osAuthorization = "Authorization: Bearer ";
935 12 : osAuthorization += osAccessToken;
936 12 : headers = curl_slist_append(headers, osAuthorization.c_str());
937 12 : headers = curl_slist_append(headers, "x-ms-version: 2019-12-12");
938 12 : return headers;
939 : }
940 :
941 418 : std::string osResource;
942 209 : const auto nSlashSlashPos = m_osEndpoint.find("//");
943 209 : if (nSlashSlashPos != std::string::npos)
944 : {
945 209 : const auto nResourcePos = m_osEndpoint.find('/', nSlashSlashPos + 2);
946 209 : if (nResourcePos != std::string::npos)
947 202 : osResource = m_osEndpoint.substr(nResourcePos);
948 : }
949 209 : osResource += "/" + m_osBucket;
950 209 : if (!m_osObjectKey.empty())
951 141 : osResource += "/" + CPLAWSURLEncode(m_osObjectKey, false);
952 :
953 418 : return GetAzureBlobHeaders(osVerb, psExistingHeaders, osResource,
954 209 : m_oMapQueryParameters, m_osStorageAccount,
955 209 : m_osStorageKey, m_bIncludeMSVersion);
956 : }
957 :
958 : /************************************************************************/
959 : /* GetSignedURL() */
960 : /************************************************************************/
961 :
962 7 : std::string VSIAzureBlobHandleHelper::GetSignedURL(CSLConstList papszOptions)
963 : {
964 7 : if (m_osStorageKey.empty())
965 3 : return m_osURL;
966 :
967 8 : std::string osStartDate(CPLGetAWS_SIGN4_Timestamp(time(nullptr)));
968 4 : const char *pszStartDate = CSLFetchNameValue(papszOptions, "START_DATE");
969 4 : if (pszStartDate)
970 2 : osStartDate = pszStartDate;
971 4 : int nYear, nMonth, nDay, nHour = 0, nMin = 0, nSec = 0;
972 4 : if (sscanf(osStartDate.c_str(), "%04d%02d%02dT%02d%02d%02dZ", &nYear,
973 4 : &nMonth, &nDay, &nHour, &nMin, &nSec) < 3)
974 : {
975 0 : return std::string();
976 : }
977 : osStartDate = CPLSPrintf("%04d-%02d-%02dT%02d:%02d:%02dZ", nYear, nMonth,
978 4 : nDay, nHour, nMin, nSec);
979 :
980 : struct tm brokendowntime;
981 4 : brokendowntime.tm_year = nYear - 1900;
982 4 : brokendowntime.tm_mon = nMonth - 1;
983 4 : brokendowntime.tm_mday = nDay;
984 4 : brokendowntime.tm_hour = nHour;
985 4 : brokendowntime.tm_min = nMin;
986 4 : brokendowntime.tm_sec = nSec;
987 4 : GIntBig nStartDate = CPLYMDHMSToUnixTime(&brokendowntime);
988 : GIntBig nEndDate =
989 : nStartDate +
990 4 : atoi(CSLFetchNameValueDef(papszOptions, "EXPIRATION_DELAY", "3600"));
991 4 : CPLUnixTimeToYMDHMS(nEndDate, &brokendowntime);
992 4 : nYear = brokendowntime.tm_year + 1900;
993 4 : nMonth = brokendowntime.tm_mon + 1;
994 4 : nDay = brokendowntime.tm_mday;
995 4 : nHour = brokendowntime.tm_hour;
996 4 : nMin = brokendowntime.tm_min;
997 4 : nSec = brokendowntime.tm_sec;
998 : std::string osEndDate = CPLSPrintf("%04d-%02d-%02dT%02d:%02d:%02dZ", nYear,
999 8 : nMonth, nDay, nHour, nMin, nSec);
1000 :
1001 8 : std::string osVerb(CSLFetchNameValueDef(papszOptions, "VERB", "GET"));
1002 : std::string osSignedPermissions(CSLFetchNameValueDef(
1003 : papszOptions, "SIGNEDPERMISSIONS",
1004 4 : (EQUAL(osVerb.c_str(), "GET") || EQUAL(osVerb.c_str(), "HEAD")) ? "r"
1005 12 : : "w"));
1006 :
1007 : std::string osSignedIdentifier(
1008 8 : CSLFetchNameValueDef(papszOptions, "SIGNEDIDENTIFIER", ""));
1009 :
1010 8 : const std::string osSignedVersion("2020-12-06");
1011 8 : const std::string osSignedProtocol("https");
1012 8 : const std::string osSignedResource("b"); // blob
1013 :
1014 8 : std::string osCanonicalizedResource("/blob/");
1015 4 : osCanonicalizedResource += CPLAWSURLEncode(m_osStorageAccount, false);
1016 4 : osCanonicalizedResource += '/';
1017 4 : osCanonicalizedResource += CPLAWSURLEncode(m_osBucket, false);
1018 4 : osCanonicalizedResource += '/';
1019 4 : osCanonicalizedResource += CPLAWSURLEncode(m_osObjectKey, false);
1020 :
1021 : // Cf https://learn.microsoft.com/en-us/rest/api/storageservices/create-service-sas
1022 8 : std::string osStringToSign;
1023 4 : osStringToSign += osSignedPermissions + "\n";
1024 4 : osStringToSign += osStartDate + "\n";
1025 4 : osStringToSign += osEndDate + "\n";
1026 4 : osStringToSign += osCanonicalizedResource + "\n";
1027 4 : osStringToSign += osSignedIdentifier + "\n";
1028 4 : osStringToSign += "\n"; // signedIP
1029 4 : osStringToSign += osSignedProtocol + "\n";
1030 4 : osStringToSign += osSignedVersion + "\n";
1031 4 : osStringToSign += osSignedResource + "\n";
1032 4 : osStringToSign += "\n"; // signedSnapshotTime
1033 4 : osStringToSign += "\n"; // signedEncryptionScope
1034 4 : osStringToSign += "\n"; // rscc
1035 4 : osStringToSign += "\n"; // rscd
1036 4 : osStringToSign += "\n"; // rsce
1037 4 : osStringToSign += "\n"; // rscl
1038 :
1039 : #ifdef DEBUG_VERBOSE
1040 : CPLDebug("AZURE", "osStringToSign = %s", osStringToSign.c_str());
1041 : #endif
1042 :
1043 : /* -------------------------------------------------------------------- */
1044 : /* Compute signature. */
1045 : /* -------------------------------------------------------------------- */
1046 : std::string osSignature(
1047 8 : CPLAzureGetSignature(osStringToSign, m_osStorageKey));
1048 :
1049 4 : ResetQueryParameters();
1050 4 : AddQueryParameter("sv", osSignedVersion);
1051 4 : AddQueryParameter("st", osStartDate);
1052 4 : AddQueryParameter("se", osEndDate);
1053 4 : AddQueryParameter("sr", osSignedResource);
1054 4 : AddQueryParameter("sp", osSignedPermissions);
1055 4 : AddQueryParameter("spr", osSignedProtocol);
1056 4 : AddQueryParameter("sig", osSignature);
1057 4 : if (!osSignedIdentifier.empty())
1058 0 : AddQueryParameter("si", osSignedIdentifier);
1059 4 : return m_osURL;
1060 : }
1061 :
1062 : #endif // HAVE_CURL
1063 :
1064 : //! @endcond
|
