Line data Source code
1 : /******************************************************************************
2 : *
3 : * Project: VSI Virtual File System
4 : * Purpose: Implementation VSI*L File API and other file system access
5 : * methods going through file virtualization.
6 : * Author: Frank Warmerdam, warmerdam@pobox.com
7 : *
8 : ******************************************************************************
9 : * Copyright (c) 2005, Frank Warmerdam <warmerdam@pobox.com>
10 : * Copyright (c) 2008-2014, Even Rouault <even dot rouault at spatialys.com>
11 : *
12 : * SPDX-License-Identifier: MIT
13 : ****************************************************************************/
14 :
15 : #include "cpl_port.h"
16 : #include "cpl_vsi.h"
17 :
18 : #include <cassert>
19 : #include <cinttypes>
20 : #include <cstdarg>
21 : #include <cstddef>
22 : #include <cstring>
23 :
24 : #include <fcntl.h>
25 :
26 : #include <algorithm>
27 : #include <limits>
28 : #include <map>
29 : #include <memory>
30 : #include <mutex>
31 : #include <set>
32 : #include <string>
33 : #include <utility>
34 : #include <vector>
35 :
36 : #include "cpl_conv.h"
37 : #include "cpl_error.h"
38 : #include "cpl_multiproc.h"
39 : #include "cpl_string.h"
40 : #include "cpl_vsi_virtual.h"
41 : #include "cpl_vsil_curl_class.h"
42 :
43 : // To avoid aliasing to GetDiskFreeSpace to GetDiskFreeSpaceA on Windows
44 : #ifdef GetDiskFreeSpace
45 : #undef GetDiskFreeSpace
46 : #endif
47 :
48 : /************************************************************************/
49 : /* VSIReadDir() */
50 : /************************************************************************/
51 :
52 : /**
53 : * \brief Read names in a directory.
54 : *
55 : * This function abstracts access to directory contains. It returns a
56 : * list of strings containing the names of files, and directories in this
57 : * directory. The resulting string list becomes the responsibility of the
58 : * application and should be freed with CSLDestroy() when no longer needed.
59 : *
60 : * Note that no error is issued via CPLError() if the directory path is
61 : * invalid, though NULL is returned.
62 : *
63 : * This function used to be known as CPLReadDir(), but the old name is now
64 : * deprecated.
65 : *
66 : * @param pszPath the relative, or absolute path of a directory to read.
67 : * UTF-8 encoded.
68 : * @return The list of entries in the directory, or NULL if the directory
69 : * doesn't exist. Filenames are returned in UTF-8 encoding.
70 : */
71 :
72 15578 : char **VSIReadDir(const char *pszPath)
73 : {
74 15578 : return VSIReadDirEx(pszPath, 0);
75 : }
76 :
77 : /************************************************************************/
78 : /* VSIReadDirEx() */
79 : /************************************************************************/
80 :
81 : /**
82 : * \brief Read names in a directory.
83 : *
84 : * This function abstracts access to directory contains. It returns a
85 : * list of strings containing the names of files, and directories in this
86 : * directory. The resulting string list becomes the responsibility of the
87 : * application and should be freed with CSLDestroy() when no longer needed.
88 : *
89 : * Note that no error is issued via CPLError() if the directory path is
90 : * invalid, though NULL is returned.
91 : *
92 : * If nMaxFiles is set to a positive number, directory listing will stop after
93 : * that limit has been reached. Note that to indicate truncate, at least one
94 : * element more than the nMaxFiles limit will be returned. If CSLCount() on the
95 : * result is lesser or equal to nMaxFiles, then no truncation occurred.
96 : *
97 : * @param pszPath the relative, or absolute path of a directory to read.
98 : * UTF-8 encoded.
99 : * @param nMaxFiles maximum number of files after which to stop, or 0 for no
100 : * limit.
101 : * @return The list of entries in the directory, or NULL if the directory
102 : * doesn't exist. Filenames are returned in UTF-8 encoding.
103 : * @since GDAL 2.1
104 : */
105 :
106 53163 : char **VSIReadDirEx(const char *pszPath, int nMaxFiles)
107 : {
108 53163 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszPath);
109 :
110 53164 : return poFSHandler->ReadDirEx(pszPath, nMaxFiles);
111 : }
112 :
113 : /************************************************************************/
114 : /* VSISiblingFiles() */
115 : /************************************************************************/
116 :
117 : /**
118 : * \brief Return related filenames
119 : *
120 : * This function is essentially meant at being used by GDAL internals.
121 : *
122 : * @param pszFilename the path of a filename to inspect
123 : * UTF-8 encoded.
124 : * @return The list of entries, relative to the directory, of all sidecar
125 : * files available or NULL if the list is not known.
126 : * Filenames are returned in UTF-8 encoding.
127 : * Most implementations will return NULL, and a subsequent ReadDir will
128 : * list all files available in the file's directory. This function will be
129 : * overridden by VSI FilesystemHandlers that wish to force e.g. an empty list
130 : * to avoid opening non-existent files on slow filesystems. The return value
131 : * shall be destroyed with CSLDestroy()
132 : * @since GDAL 3.2
133 : */
134 77397 : char **VSISiblingFiles(const char *pszFilename)
135 : {
136 77397 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
137 :
138 77401 : return poFSHandler->SiblingFiles(pszFilename);
139 : }
140 :
141 : /************************************************************************/
142 : /* VSIFnMatch() */
143 : /************************************************************************/
144 :
145 538 : static bool VSIFnMatch(const char *pszPattern, const char *pszStr)
146 : {
147 538 : for (; *pszPattern && *pszStr; pszPattern++, pszStr++)
148 : {
149 532 : if (*pszPattern == '*')
150 : {
151 14 : if (pszPattern[1] == 0)
152 4 : return true;
153 46 : for (; *pszStr; ++pszStr)
154 : {
155 40 : if (VSIFnMatch(pszPattern + 1, pszStr))
156 4 : return true;
157 : }
158 6 : return false;
159 : }
160 518 : else if (*pszPattern == '?')
161 : {
162 : // match single any char
163 : }
164 514 : else if (*pszPattern == '[')
165 : {
166 : // match character classes and ranges
167 : // "[abcd]" will match a character that is a, b, c or d
168 : // "[a-z]" will match a character that is a to z
169 : // "[!abcd] will match a character that is *not* a, b, c or d
170 : // "[]]" will match character ]
171 : // "[]-]" will match character ] or -
172 : // "[!]a-]" will match a character that is *not* ], a or -
173 :
174 10 : const char *pszOpenBracket = pszPattern;
175 10 : ++pszPattern;
176 10 : const bool isNot = (*pszPattern == '!');
177 10 : if (isNot)
178 : {
179 3 : ++pszOpenBracket;
180 3 : ++pszPattern;
181 : }
182 10 : bool res = false;
183 22 : for (; *pszPattern; ++pszPattern)
184 : {
185 21 : if ((*pszPattern == ']' || *pszPattern == '-') &&
186 13 : pszPattern == pszOpenBracket + 1)
187 : {
188 3 : if (*pszStr == *pszPattern)
189 : {
190 2 : res = true;
191 : }
192 : }
193 18 : else if (*pszPattern == ']')
194 : {
195 9 : break;
196 : }
197 9 : else if (pszPattern[1] == '-' && pszPattern[2] != 0 &&
198 2 : pszPattern[2] != ']')
199 : {
200 1 : if (*pszStr >= pszPattern[0] && *pszStr <= pszPattern[2])
201 : {
202 1 : res = true;
203 : }
204 1 : pszPattern += 2;
205 : }
206 8 : else if (*pszStr == *pszPattern)
207 : {
208 2 : res = true;
209 : }
210 : }
211 10 : if (*pszPattern == 0)
212 1 : return false;
213 9 : if (!res && !isNot)
214 1 : return false;
215 8 : if (res && isNot)
216 0 : return false;
217 : }
218 504 : else if (*pszPattern != *pszStr)
219 : {
220 453 : return false;
221 : }
222 : }
223 6 : return *pszPattern == 0 && *pszStr == 0;
224 : }
225 :
226 : /************************************************************************/
227 : /* VSIGlob() */
228 : /************************************************************************/
229 :
230 : /**
231 : \brief Return a list of file and directory names matching
232 : a pattern that can contain wildcards.
233 :
234 : This function has similar behavior to the POSIX glob() function:
235 : https://man7.org/linux/man-pages/man7/glob.7.html
236 :
237 : In particular it supports the following wildcards:
238 : <ul>
239 : <li>'*': match any string</li>
240 : <li>'?': match any single character</li>
241 : <li>'[': match character class or range, with '!' immediately after '['
242 : to indicate negation.</li>
243 : </ul>
244 : Refer to to the above man page for more details.
245 :
246 : It also supports the "**" recursive wildcard, behaving similarly to Python
247 : glob.glob() with recursive=True. Be careful of the amount of memory and time
248 : required when using that recursive wildcard on directories with a large
249 : amount of files and subdirectories.
250 :
251 : Examples, given a file hierarchy:
252 : - one.tif
253 : - my_subdir/two.tif
254 : - my_subdir/subsubdir/three.tif
255 :
256 : \code{.cpp}
257 : VSIGlob("one.tif",NULL,NULL,NULL) returns ["one.tif", NULL]
258 : VSIGlob("*.tif",NULL,NULL,NULL) returns ["one.tif", NULL]
259 : VSIGlob("on?.tif",NULL,NULL,NULL) returns ["one.tif", NULL]
260 : VSIGlob("on[a-z].tif",NULL,NULL,NULL) returns ["one.tif", NULL]
261 : VSIGlob("on[ef].tif",NULL,NULL,NULL) returns ["one.tif", NULL]
262 : VSIGlob("on[!e].tif",NULL,NULL,NULL) returns NULL
263 : VSIGlob("my_subdir" "/" "*.tif",NULL,NULL,NULL) returns ["my_subdir/two.tif", NULL]
264 : VSIGlob("**" "/" "*.tif",NULL,NULL,NULL) returns ["one.tif", "my_subdir/two.tif", "my_subdir/subsubdir/three.tif", NULL]
265 : \endcode
266 :
267 : In the current implementation, matching is done based on the assumption that
268 : a character fits into a single byte, which will not work properly on
269 : non-ASCII UTF-8 filenames.
270 :
271 : VSIGlob() works with any virtual file systems supported by GDAL, including
272 : network file systems such as /vsis3/, /vsigs/, /vsiaz/, etc. But note that
273 : for those ones, the pattern is not passed to the remote server, and thus large
274 : amount of filenames can be transferred from the remote server to the host
275 : where the filtering is done.
276 :
277 : @param pszPattern the relative, or absolute path of a directory to read.
278 : UTF-8 encoded.
279 : @param papszOptions NULL-terminate list of options, or NULL. None supported
280 : currently.
281 : @param pProgressFunc Progress function, or NULL. This is only used as a way
282 : for the user to cancel operation if it takes too much time. The percentage
283 : passed to the callback is not significant (always at 0).
284 : @param pProgressData User data passed to the progress function, or NULL.
285 : @return The list of matched filenames, which must be freed with CSLDestroy().
286 : Filenames are returned in UTF-8 encoding.
287 :
288 : @since GDAL 3.11
289 : */
290 :
291 17 : char **VSIGlob(const char *pszPattern, const char *const *papszOptions,
292 : GDALProgressFunc pProgressFunc, void *pProgressData)
293 : {
294 17 : CPL_IGNORE_RET_VAL(papszOptions);
295 :
296 34 : CPLStringList aosRes;
297 34 : std::vector<std::pair<std::string, size_t>> candidates;
298 17 : candidates.emplace_back(pszPattern, 0);
299 48 : while (!candidates.empty())
300 : {
301 31 : auto [osPattern, nPosStart] = candidates.back();
302 31 : pszPattern = osPattern.c_str() + nPosStart;
303 31 : candidates.pop_back();
304 :
305 31 : std::string osPath = osPattern.substr(0, nPosStart);
306 31 : std::string osCurPath;
307 515 : for (;; ++pszPattern)
308 : {
309 546 : if (*pszPattern == 0 || *pszPattern == '/' || *pszPattern == '\\')
310 : {
311 : struct VSIDirCloser
312 : {
313 15 : void operator()(VSIDIR *dir)
314 : {
315 15 : VSICloseDir(dir);
316 15 : }
317 : };
318 :
319 104 : if (osCurPath == "**")
320 : {
321 : std::unique_ptr<VSIDIR, VSIDirCloser> psDir(
322 1 : VSIOpenDir(osPath.c_str(), -1, nullptr));
323 1 : if (!psDir)
324 0 : return nullptr;
325 : while (const VSIDIREntry *psEntry =
326 5 : VSIGetNextDirEntry(psDir.get()))
327 : {
328 4 : if (pProgressFunc &&
329 0 : !pProgressFunc(0, "", pProgressData))
330 : {
331 0 : return nullptr;
332 : }
333 : {
334 8 : std::string osCandidate(osPath);
335 4 : osCandidate += psEntry->pszName;
336 4 : nPosStart = osCandidate.size();
337 4 : if (*pszPattern)
338 : {
339 4 : osCandidate += pszPattern;
340 : }
341 4 : candidates.emplace_back(std::move(osCandidate),
342 4 : nPosStart);
343 : }
344 4 : }
345 1 : osPath.clear();
346 1 : break;
347 : }
348 103 : else if (osCurPath.find_first_of("*?[") != std::string::npos)
349 : {
350 : std::unique_ptr<VSIDIR, VSIDirCloser> psDir(
351 14 : VSIOpenDir(osPath.c_str(), 0, nullptr));
352 14 : if (!psDir)
353 0 : return nullptr;
354 : while (const VSIDIREntry *psEntry =
355 449 : VSIGetNextDirEntry(psDir.get()))
356 : {
357 435 : if (pProgressFunc &&
358 0 : !pProgressFunc(0, "", pProgressData))
359 : {
360 0 : return nullptr;
361 : }
362 435 : if (VSIFnMatch(osCurPath.c_str(), psEntry->pszName))
363 : {
364 20 : std::string osCandidate(osPath);
365 10 : osCandidate += psEntry->pszName;
366 10 : nPosStart = osCandidate.size();
367 10 : if (*pszPattern)
368 : {
369 2 : osCandidate += pszPattern;
370 : }
371 10 : candidates.emplace_back(std::move(osCandidate),
372 10 : nPosStart);
373 : }
374 435 : }
375 14 : osPath.clear();
376 14 : break;
377 : }
378 89 : else if (*pszPattern == 0)
379 : {
380 16 : osPath += osCurPath;
381 16 : break;
382 : }
383 : else
384 : {
385 73 : osPath += osCurPath;
386 73 : osPath += *pszPattern;
387 73 : osCurPath.clear();
388 73 : }
389 : }
390 : else
391 : {
392 442 : osCurPath += *pszPattern;
393 : }
394 515 : }
395 31 : if (!osPath.empty())
396 : {
397 : VSIStatBufL sStat;
398 16 : if (VSIStatL(osPath.c_str(), &sStat) == 0)
399 11 : aosRes.AddString(osPath.c_str());
400 : }
401 : }
402 :
403 17 : return aosRes.StealList();
404 : }
405 :
406 : /************************************************************************/
407 : /* VSIGetDirectorySeparator() */
408 : /************************************************************************/
409 :
410 : /** Return the directory separator for the specified path.
411 : *
412 : * Default is forward slash. The only exception currently is the Windows
413 : * file system which returns backslash, unless the specified path is of the
414 : * form "{drive_letter}:/{rest_of_the_path}".
415 : *
416 : * @since 3.9
417 : */
418 930714 : const char *VSIGetDirectorySeparator(const char *pszPath)
419 : {
420 930714 : if (STARTS_WITH(pszPath, "http://") || STARTS_WITH(pszPath, "https://"))
421 550 : return "/";
422 :
423 930164 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszPath);
424 930209 : return poFSHandler->GetDirectorySeparator(pszPath);
425 : }
426 :
427 : /************************************************************************/
428 : /* VSIReadRecursive() */
429 : /************************************************************************/
430 :
431 : /**
432 : * \brief Read names in a directory recursively.
433 : *
434 : * This function abstracts access to directory contents and subdirectories.
435 : * It returns a list of strings containing the names of files and directories
436 : * in this directory and all subdirectories. The resulting string list becomes
437 : * the responsibility of the application and should be freed with CSLDestroy()
438 : * when no longer needed.
439 : *
440 : * Note that no error is issued via CPLError() if the directory path is
441 : * invalid, though NULL is returned.
442 : *
443 : * Note: since GDAL 3.9, for recursive mode, the directory separator will no
444 : * longer be always forward slash, but will be the one returned by
445 : * VSIGetDirectorySeparator(pszPathIn), so potentially backslash on Windows
446 : * file systems.
447 : *
448 : * @param pszPathIn the relative, or absolute path of a directory to read.
449 : * UTF-8 encoded.
450 : *
451 : * @return The list of entries in the directory and subdirectories
452 : * or NULL if the directory doesn't exist. Filenames are returned in UTF-8
453 : * encoding.
454 : * @since GDAL 1.10.0
455 : *
456 : */
457 :
458 1151 : char **VSIReadDirRecursive(const char *pszPathIn)
459 : {
460 1151 : const char SEP = VSIGetDirectorySeparator(pszPathIn)[0];
461 :
462 1151 : const char *const apszOptions[] = {"NAME_AND_TYPE_ONLY=YES", nullptr};
463 1151 : VSIDIR *psDir = VSIOpenDir(pszPathIn, -1, apszOptions);
464 1151 : if (!psDir)
465 3 : return nullptr;
466 2296 : CPLStringList oFiles;
467 5003 : while (auto psEntry = VSIGetNextDirEntry(psDir))
468 : {
469 3855 : if (VSI_ISDIR(psEntry->nMode) && psEntry->pszName[0] &&
470 1474 : psEntry->pszName[strlen(psEntry->pszName) - 1] != SEP)
471 : {
472 1474 : oFiles.AddString((std::string(psEntry->pszName) + SEP).c_str());
473 : }
474 : else
475 : {
476 2381 : oFiles.AddString(psEntry->pszName);
477 : }
478 3855 : }
479 1148 : VSICloseDir(psDir);
480 :
481 1148 : return oFiles.StealList();
482 : }
483 :
484 : /************************************************************************/
485 : /* CPLReadDir() */
486 : /* */
487 : /* This is present only to provide ABI compatibility with older */
488 : /* versions. */
489 : /************************************************************************/
490 : #undef CPLReadDir
491 :
492 : CPL_C_START
493 : char CPL_DLL **CPLReadDir(const char *pszPath);
494 : CPL_C_END
495 :
496 0 : char **CPLReadDir(const char *pszPath)
497 : {
498 0 : return VSIReadDir(pszPath);
499 : }
500 :
501 : /************************************************************************/
502 : /* VSIOpenDir() */
503 : /************************************************************************/
504 :
505 : /**
506 : * \brief Open a directory to read its entries.
507 : *
508 : * This function is close to the POSIX opendir() function.
509 : *
510 : * For /vsis3/, /vsigs/, /vsioss/, /vsiaz/ and /vsiadls/, this function has an
511 : * efficient implementation, minimizing the number of network requests, when
512 : * invoked with nRecurseDepth <= 0.
513 : *
514 : * Entries are read by calling VSIGetNextDirEntry() on the handled returned by
515 : * that function, until it returns NULL. VSICloseDir() must be called once done
516 : * with the returned directory handle.
517 : *
518 : * @param pszPath the relative, or absolute path of a directory to read.
519 : * UTF-8 encoded.
520 : * @param nRecurseDepth 0 means do not recurse in subdirectories, 1 means
521 : * recurse only in the first level of subdirectories, etc. -1 means unlimited
522 : * recursion level
523 : * @param papszOptions NULL terminated list of options, or NULL. The following
524 : * options are implemented:
525 : * <ul>
526 : * <li>PREFIX=string: (GDAL >= 3.4) Filter to select filenames only starting
527 : * with the specified prefix. Implemented efficiently for /vsis3/, /vsigs/,
528 : * and /vsiaz/ (but not /vsiadls/)
529 : * </li>
530 : * <li>NAME_AND_TYPE_ONLY=YES/NO: (GDAL >= 3.4) Defaults to NO. If set to YES,
531 : * only the pszName and nMode members of VSIDIR are guaranteed to be set.
532 : * This is implemented efficiently for the Unix virtual file system.
533 : * </li>
534 : * </ul>
535 : *
536 : * @return a handle, or NULL in case of error
537 : * @since GDAL 2.4
538 : *
539 : */
540 :
541 1528 : VSIDIR *VSIOpenDir(const char *pszPath, int nRecurseDepth,
542 : const char *const *papszOptions)
543 : {
544 1528 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszPath);
545 :
546 1528 : return poFSHandler->OpenDir(pszPath, nRecurseDepth, papszOptions);
547 : }
548 :
549 : /************************************************************************/
550 : /* VSIGetNextDirEntry() */
551 : /************************************************************************/
552 :
553 : /**
554 : * \brief Return the next entry of the directory
555 : *
556 : * This function is close to the POSIX readdir() function. It actually returns
557 : * more information (file size, last modification time), which on 'real' file
558 : * systems involve one 'stat' call per file.
559 : *
560 : * For filesystems that can have both a regular file and a directory name of
561 : * the same name (typically /vsis3/), when this situation of duplicate happens,
562 : * the directory name will be suffixed by a slash character. Otherwise directory
563 : * names are not suffixed by slash.
564 : *
565 : * The returned entry remains valid until the next call to VSINextDirEntry()
566 : * or VSICloseDir() with the same handle.
567 : *
568 : * Note: since GDAL 3.9, for recursive mode, the directory separator will no
569 : * longer be always forward slash, but will be the one returned by
570 : * VSIGetDirectorySeparator(pszPathIn), so potentially backslash on Windows
571 : * file systems.
572 : *
573 : * @param dir Directory handled returned by VSIOpenDir(). Must not be NULL.
574 : *
575 : * @return a entry, or NULL if there is no more entry in the directory. This
576 : * return value must not be freed.
577 : * @since GDAL 2.4
578 : *
579 : */
580 :
581 14110 : const VSIDIREntry *VSIGetNextDirEntry(VSIDIR *dir)
582 : {
583 14110 : return dir->NextDirEntry();
584 : }
585 :
586 : /************************************************************************/
587 : /* VSICloseDir() */
588 : /************************************************************************/
589 :
590 : /**
591 : * \brief Close a directory
592 : *
593 : * This function is close to the POSIX closedir() function.
594 : *
595 : * @param dir Directory handled returned by VSIOpenDir().
596 : *
597 : * @since GDAL 2.4
598 : */
599 :
600 1496 : void VSICloseDir(VSIDIR *dir)
601 : {
602 1496 : delete dir;
603 1496 : }
604 :
605 : /************************************************************************/
606 : /* VSIMkdir() */
607 : /************************************************************************/
608 :
609 : /**
610 : * \brief Create a directory.
611 : *
612 : * Create a new directory with the indicated mode. For POSIX-style systems,
613 : * the mode is modified by the file creation mask (umask). However, some
614 : * file systems and platforms may not use umask, or they may ignore the mode
615 : * completely. So a reasonable cross-platform default mode value is 0755.
616 : *
617 : * Analog of the POSIX mkdir() function.
618 : *
619 : * @param pszPathname the path to the directory to create. UTF-8 encoded.
620 : * @param mode the permissions mode.
621 : *
622 : * @return 0 on success or -1 on an error.
623 : */
624 :
625 99473 : int VSIMkdir(const char *pszPathname, long mode)
626 :
627 : {
628 99473 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszPathname);
629 :
630 99473 : return poFSHandler->Mkdir(pszPathname, mode);
631 : }
632 :
633 : /************************************************************************/
634 : /* VSIMkdirRecursive() */
635 : /************************************************************************/
636 :
637 : /**
638 : * \brief Create a directory and all its ancestors
639 : *
640 : * @param pszPathname the path to the directory to create. UTF-8 encoded.
641 : * @param mode the permissions mode.
642 : *
643 : * @return 0 on success or -1 on an error.
644 : * @since GDAL 2.3
645 : */
646 :
647 140962 : int VSIMkdirRecursive(const char *pszPathname, long mode)
648 : {
649 140962 : if (pszPathname == nullptr || pszPathname[0] == '\0' ||
650 140961 : strncmp("/", pszPathname, 2) == 0)
651 : {
652 0 : return -1;
653 : }
654 :
655 281925 : const CPLString osPathname(pszPathname);
656 : VSIStatBufL sStat;
657 140960 : if (VSIStatL(osPathname, &sStat) == 0)
658 : {
659 50263 : return VSI_ISDIR(sStat.st_mode) ? 0 : -1;
660 : }
661 181400 : const std::string osParentPath(CPLGetPathSafe(osPathname));
662 :
663 : // Prevent crazy paths from recursing forever.
664 181400 : if (osParentPath == osPathname ||
665 90700 : osParentPath.length() >= osPathname.length())
666 : {
667 0 : return -1;
668 : }
669 :
670 90700 : if (!osParentPath.empty() && VSIStatL(osParentPath.c_str(), &sStat) != 0)
671 : {
672 45433 : if (VSIMkdirRecursive(osParentPath.c_str(), mode) != 0)
673 19 : return -1;
674 : }
675 :
676 90681 : return VSIMkdir(osPathname, mode);
677 : }
678 :
679 : /************************************************************************/
680 : /* VSIUnlink() */
681 : /************************************************************************/
682 :
683 : /**
684 : * \brief Delete a file.
685 : *
686 : * Deletes a file object from the file system.
687 : *
688 : * This method goes through the VSIFileHandler virtualization and may
689 : * work on unusual filesystems such as in memory.
690 : *
691 : * Analog of the POSIX unlink() function.
692 : *
693 : * @param pszFilename the path of the file to be deleted. UTF-8 encoded.
694 : *
695 : * @return 0 on success or -1 on an error.
696 : */
697 :
698 86513 : int VSIUnlink(const char *pszFilename)
699 :
700 : {
701 86513 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
702 :
703 86513 : return poFSHandler->Unlink(pszFilename);
704 : }
705 :
706 : /************************************************************************/
707 : /* VSIUnlinkBatch() */
708 : /************************************************************************/
709 :
710 : /**
711 : * \brief Delete several files, possibly in a batch.
712 : *
713 : * All files should belong to the same file system handler.
714 : *
715 : * This is implemented efficiently for /vsis3/ and /vsigs/ (provided for /vsigs/
716 : * that OAuth2 authentication is used).
717 : *
718 : * @param papszFiles NULL terminated list of files. UTF-8 encoded.
719 : *
720 : * @return an array of size CSLCount(papszFiles), whose values are TRUE or FALSE
721 : * depending on the success of deletion of the corresponding file. The array
722 : * should be freed with VSIFree().
723 : * NULL might be return in case of a more general error (for example,
724 : * files belonging to different file system handlers)
725 : *
726 : * @since GDAL 3.1
727 : */
728 :
729 11 : int *VSIUnlinkBatch(CSLConstList papszFiles)
730 : {
731 11 : VSIFilesystemHandler *poFSHandler = nullptr;
732 30 : for (CSLConstList papszIter = papszFiles; papszIter && *papszIter;
733 : ++papszIter)
734 : {
735 20 : auto poFSHandlerThisFile = VSIFileManager::GetHandler(*papszIter);
736 20 : if (poFSHandler == nullptr)
737 10 : poFSHandler = poFSHandlerThisFile;
738 10 : else if (poFSHandler != poFSHandlerThisFile)
739 : {
740 1 : CPLError(CE_Failure, CPLE_AppDefined,
741 : "Files belong to different file system handlers");
742 1 : poFSHandler = nullptr;
743 1 : break;
744 : }
745 : }
746 11 : if (poFSHandler == nullptr)
747 2 : return nullptr;
748 9 : return poFSHandler->UnlinkBatch(papszFiles);
749 : }
750 :
751 : /************************************************************************/
752 : /* VSIRename() */
753 : /************************************************************************/
754 :
755 : /**
756 : * \brief Rename a file.
757 : *
758 : * Renames a file object in the file system. It should be possible
759 : * to rename a file onto a new directory, but it is safest if this
760 : * function is only used to rename files that remain in the same directory.
761 : *
762 : * This function only works if the new path is located on the same VSI
763 : * virtual file system than the old path. If not, use VSIMove() instead.
764 : *
765 : * This method goes through the VSIFileHandler virtualization and may
766 : * work on unusual filesystems such as in memory or cloud object storage.
767 : * Note that for cloud object storage, renaming a directory may involve
768 : * renaming all files it contains recursively, and is thus not an atomic
769 : * operation (and could be expensive on directories with many files!)
770 : *
771 : * Analog of the POSIX rename() function.
772 : *
773 : * @param oldpath the name of the file to be renamed. UTF-8 encoded.
774 : * @param newpath the name the file should be given. UTF-8 encoded.
775 : *
776 : * @return 0 on success or -1 on an error.
777 : */
778 :
779 1464 : int VSIRename(const char *oldpath, const char *newpath)
780 :
781 : {
782 1464 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(oldpath);
783 :
784 1464 : return poFSHandler->Rename(oldpath, newpath, nullptr, nullptr);
785 : }
786 :
787 : /************************************************************************/
788 : /* VSIMove() */
789 : /************************************************************************/
790 :
791 : /**
792 : * \brief Move (or rename) a file.
793 : *
794 : * If the new path is an existing directory, the file will be moved to it.
795 : *
796 : * The function can work even if the files are not located on the same VSI
797 : * virtual file system, but it will involve copying and deletion.
798 : *
799 : * Note that for cloud object storage, moving/renaming a directory may involve
800 : * renaming all files it contains recursively, and is thus not an atomic
801 : * operation (and could be slow and expensive on directories with many files!)
802 : *
803 : * @param oldpath the path of the file to be renamed/moved. UTF-8 encoded.
804 : * @param newpath the new path the file should be given. UTF-8 encoded.
805 : * @param papszOptions Null terminated list of options, or NULL.
806 : * @param pProgressFunc Progress callback, or NULL.
807 : * @param pProgressData User data of progress callback, or NULL.
808 : *
809 : * @return 0 on success or -1 on error.
810 : * @since GDAL 3.11
811 : */
812 :
813 12 : int VSIMove(const char *oldpath, const char *newpath,
814 : const char *const *papszOptions, GDALProgressFunc pProgressFunc,
815 : void *pProgressData)
816 : {
817 :
818 12 : if (strcmp(oldpath, newpath) == 0)
819 1 : return 0;
820 :
821 11 : VSIFilesystemHandler *poOldFSHandler = VSIFileManager::GetHandler(oldpath);
822 11 : VSIFilesystemHandler *poNewFSHandler = VSIFileManager::GetHandler(newpath);
823 :
824 : VSIStatBufL sStat;
825 11 : if (VSIStatL(oldpath, &sStat) != 0)
826 : {
827 2 : CPLDebug("VSI", "%s is not a object", oldpath);
828 2 : errno = ENOENT;
829 2 : return -1;
830 : }
831 :
832 18 : std::string sNewpath(newpath);
833 : VSIStatBufL sStatNew;
834 9 : if (VSIStatL(newpath, &sStatNew) == 0 && VSI_ISDIR(sStatNew.st_mode))
835 : {
836 : sNewpath =
837 4 : CPLFormFilenameSafe(newpath, CPLGetFilename(oldpath), nullptr);
838 : }
839 :
840 9 : int ret = 0;
841 :
842 9 : if (poOldFSHandler == poNewFSHandler)
843 : {
844 2 : ret = poOldFSHandler->Rename(oldpath, sNewpath.c_str(), pProgressFunc,
845 2 : pProgressData);
846 2 : if (ret == 0 && pProgressFunc)
847 1 : ret = pProgressFunc(1.0, "", pProgressData) ? 0 : -1;
848 2 : return ret;
849 : }
850 :
851 7 : if (VSI_ISDIR(sStat.st_mode))
852 : {
853 6 : const CPLStringList aosList(VSIReadDir(oldpath));
854 3 : poNewFSHandler->Mkdir(sNewpath.c_str(), 0755);
855 3 : bool bFoundFiles = false;
856 3 : const int nListSize = aosList.size();
857 6 : for (int i = 0; ret == 0 && i < nListSize; i++)
858 : {
859 3 : if (strcmp(aosList[i], ".") != 0 && strcmp(aosList[i], "..") != 0)
860 : {
861 1 : bFoundFiles = true;
862 : const std::string osSrc =
863 2 : CPLFormFilenameSafe(oldpath, aosList[i], nullptr);
864 : const std::string osTarget =
865 2 : CPLFormFilenameSafe(sNewpath.c_str(), aosList[i], nullptr);
866 2 : void *pScaledProgress = GDALCreateScaledProgress(
867 1 : static_cast<double>(i) / nListSize,
868 1 : static_cast<double>(i + 1) / nListSize, pProgressFunc,
869 : pProgressData);
870 1 : ret = VSIMove(osSrc.c_str(), osTarget.c_str(), papszOptions,
871 : pScaledProgress ? GDALScaledProgress : nullptr,
872 : pScaledProgress);
873 1 : GDALDestroyScaledProgress(pScaledProgress);
874 : }
875 : }
876 3 : if (!bFoundFiles)
877 2 : ret = VSIStatL(sNewpath.c_str(), &sStat);
878 3 : if (ret == 0)
879 3 : ret = poOldFSHandler->Rmdir(oldpath);
880 : }
881 : else
882 : {
883 8 : ret = VSICopyFile(oldpath, sNewpath.c_str(), nullptr, sStat.st_size,
884 3 : nullptr, pProgressFunc, pProgressData) == 0 &&
885 3 : VSIUnlink(oldpath) == 0
886 7 : ? 0
887 : : -1;
888 : }
889 7 : if (ret == 0 && pProgressFunc)
890 4 : ret = pProgressFunc(1.0, "", pProgressData) ? 0 : -1;
891 7 : return ret;
892 : }
893 :
894 : /************************************************************************/
895 : /* VSICopyFile() */
896 : /************************************************************************/
897 :
898 : /**
899 : * \brief Copy a source file into a target file.
900 : *
901 : * For a /vsizip/foo.zip/bar target, the options available are those of
902 : * CPLAddFileInZip()
903 : *
904 : * The following copies are made fully on the target server, without local
905 : * download from source and upload to target:
906 : * - /vsis3/ -> /vsis3/
907 : * - /vsigs/ -> /vsigs/
908 : * - /vsiaz/ -> /vsiaz/
909 : * - /vsiadls/ -> /vsiadls/
910 : * - any of the above or /vsicurl/ -> /vsiaz/ (starting with GDAL 3.8)
911 : *
912 : * @param pszSource Source filename. UTF-8 encoded. May be NULL if fpSource is
913 : * not NULL.
914 : * @param pszTarget Target filename. UTF-8 encoded. Must not be NULL
915 : * @param fpSource File handle on the source file. May be NULL if pszSource is
916 : * not NULL.
917 : * @param nSourceSize Size of the source file. Pass -1 if unknown.
918 : * If set to -1, and progress callback is used, VSIStatL() will be used on
919 : * pszSource to retrieve the source size.
920 : * @param papszOptions Null terminated list of options, or NULL.
921 : * @param pProgressFunc Progress callback, or NULL.
922 : * @param pProgressData User data of progress callback, or NULL.
923 : *
924 : * @return 0 on success.
925 : * @since GDAL 3.7
926 : */
927 :
928 2298 : int VSICopyFile(const char *pszSource, const char *pszTarget,
929 : VSILFILE *fpSource, vsi_l_offset nSourceSize,
930 : const char *const *papszOptions, GDALProgressFunc pProgressFunc,
931 : void *pProgressData)
932 :
933 : {
934 2298 : if (!pszSource && !fpSource)
935 : {
936 1 : CPLError(CE_Failure, CPLE_AppDefined,
937 : "pszSource == nullptr && fpSource == nullptr");
938 1 : return -1;
939 : }
940 2297 : if (!pszTarget || pszTarget[0] == '\0')
941 : {
942 0 : return -1;
943 : }
944 :
945 : VSIFilesystemHandler *poFSHandlerTarget =
946 2297 : VSIFileManager::GetHandler(pszTarget);
947 2297 : return poFSHandlerTarget->CopyFile(pszSource, pszTarget, fpSource,
948 : nSourceSize, papszOptions, pProgressFunc,
949 2297 : pProgressData);
950 : }
951 :
952 : /************************************************************************/
953 : /* VSICopyFileRestartable() */
954 : /************************************************************************/
955 :
956 : /**
957 : \brief Copy a source file into a target file in a way that can (potentially)
958 : be restarted.
959 :
960 : This function provides the possibility of efficiently restarting upload of
961 : large files to cloud storage that implements upload in a chunked way,
962 : such as /vsis3/ and /vsigs/.
963 : For other destination file systems, this function may fallback to
964 : VSICopyFile() and not provide any smart restartable implementation.
965 :
966 : Example of a potential workflow:
967 :
968 : @code{.cpp}
969 : char* pszOutputPayload = NULL;
970 : int ret = VSICopyFileRestartable(pszSource, pszTarget, NULL,
971 : &pszOutputPayload, NULL, NULL, NULL);
972 : while( ret == 1 ) // add also a limiting counter to avoid potentiall endless looping
973 : {
974 : // TODO: wait for some time
975 :
976 : char* pszOutputPayloadNew = NULL;
977 : const char* pszInputPayload = pszOutputPayload;
978 : ret = VSICopyFileRestartable(pszSource, pszTarget, pszInputPayload,
979 : &pszOutputPayloadNew, NULL, NULL, NULL);
980 : VSIFree(pszOutputPayload);
981 : pszOutputPayload = pszOutputPayloadNew;
982 : }
983 : VSIFree(pszOutputPayload);
984 : @endcode
985 :
986 : @param pszSource Source filename. UTF-8 encoded. Must not be NULL
987 : @param pszTarget Target filename. UTF-8 encoded. Must not be NULL
988 : @param pszInputPayload NULL at the first invocation. When doing a retry,
989 : should be the content of *ppszOutputPayload from a
990 : previous invocation.
991 : @param[out] ppszOutputPayload Pointer to an output string that will be set to
992 : a value that can be provided as pszInputPayload
993 : for a next call to VSICopyFileRestartable().
994 : ppszOutputPayload must not be NULL.
995 : The string set in *ppszOutputPayload, if not NULL,
996 : is JSON-encoded, and can be re-used in another
997 : process instance. It must be freed with VSIFree()
998 : when no longer needed.
999 : @param papszOptions Null terminated list of options, or NULL.
1000 : Currently accepted options are:
1001 : <ul>
1002 : <li>NUM_THREADS=integer or ALL_CPUS. Number of threads to use for parallel
1003 : file copying. Only use for when /vsis3/, /vsigs/, /vsiaz/ or /vsiadls/ is in
1004 : source or target. The default is 10.
1005 : </li>
1006 : <li>CHUNK_SIZE=integer. Maximum size of chunk (in bytes) to use
1007 : to split large objects. For upload to /vsis3/, this chunk size must be set at
1008 : least to 5 MB. The default is 50 MB.
1009 : </li>
1010 : </ul>
1011 : @param pProgressFunc Progress callback, or NULL.
1012 : @param pProgressData User data of progress callback, or NULL.
1013 : @return 0 on success,
1014 : -1 on (non-restartable) failure,
1015 : 1 if VSICopyFileRestartable() can be called again in a restartable way
1016 : @since GDAL 3.10
1017 :
1018 : @see VSIAbortPendingUploads()
1019 : */
1020 :
1021 20 : int VSICopyFileRestartable(const char *pszSource, const char *pszTarget,
1022 : const char *pszInputPayload,
1023 : char **ppszOutputPayload,
1024 : const char *const *papszOptions,
1025 : GDALProgressFunc pProgressFunc, void *pProgressData)
1026 :
1027 : {
1028 20 : if (!pszSource)
1029 : {
1030 0 : return -1;
1031 : }
1032 20 : if (!pszTarget || pszTarget[0] == '\0')
1033 : {
1034 0 : return -1;
1035 : }
1036 20 : if (!ppszOutputPayload)
1037 : {
1038 0 : return -1;
1039 : }
1040 :
1041 : VSIFilesystemHandler *poFSHandlerTarget =
1042 20 : VSIFileManager::GetHandler(pszTarget);
1043 20 : return poFSHandlerTarget->CopyFileRestartable(
1044 : pszSource, pszTarget, pszInputPayload, ppszOutputPayload, papszOptions,
1045 20 : pProgressFunc, pProgressData);
1046 : }
1047 :
1048 : /************************************************************************/
1049 : /* VSISync() */
1050 : /************************************************************************/
1051 :
1052 : /**
1053 : * \brief Synchronize a source file/directory with a target file/directory.
1054 : *
1055 : * This is a analog of the 'rsync' utility. In the current implementation,
1056 : * rsync would be more efficient for local file copying, but VSISync() main
1057 : * interest is when the source or target is a remote
1058 : * file system like /vsis3/ or /vsigs/, in which case it can take into account
1059 : * the timestamps of the files (or optionally the ETag/MD5Sum) to avoid
1060 : * unneeded copy operations.
1061 : *
1062 : * This is only implemented efficiently for:
1063 : * <ul>
1064 : * <li> local filesystem <--> remote filesystem.</li>
1065 : * <li> remote filesystem <--> remote filesystem (starting with GDAL 3.1).
1066 : * Where the source and target remote filesystems are the same and one of
1067 : * /vsis3/, /vsigs/ or /vsiaz/. Or when the target is /vsiaz/ and the source
1068 : * is /vsis3/, /vsigs/, /vsiadls/ or /vsicurl/ (starting with GDAL 3.8)</li>
1069 : * </ul>
1070 : *
1071 : * Similarly to rsync behavior, if the source filename ends with a slash,
1072 : * it means that the content of the directory must be copied, but not the
1073 : * directory name. For example, assuming "/home/even/foo" contains a file "bar",
1074 : * VSISync("/home/even/foo/", "/mnt/media", ...) will create a "/mnt/media/bar"
1075 : * file. Whereas VSISync("/home/even/foo", "/mnt/media", ...) will create a
1076 : * "/mnt/media/foo" directory which contains a bar file.
1077 : *
1078 : * @param pszSource Source file or directory. UTF-8 encoded.
1079 : * @param pszTarget Target file or directory. UTF-8 encoded.
1080 : * @param papszOptions Null terminated list of options, or NULL.
1081 : * Currently accepted options are:
1082 : * <ul>
1083 : * <li>RECURSIVE=NO (the default is YES)</li>
1084 : * <li>SYNC_STRATEGY=TIMESTAMP/ETAG/OVERWRITE.
1085 : *
1086 : * Determines which criterion is used to determine if a target file must be
1087 : * replaced when it already exists and has the same file size as the source.
1088 : * Only applies for a source or target being a network filesystem.
1089 : *
1090 : * The default is TIMESTAMP (similarly to how 'aws s3 sync' works), that is
1091 : * to say that for an upload operation, a remote file is
1092 : * replaced if it has a different size or if it is older than the source.
1093 : * For a download operation, a local file is replaced if it has a different
1094 : * size or if it is newer than the remote file.
1095 : *
1096 : * The ETAG strategy assumes that the ETag metadata of the remote file is
1097 : * the MD5Sum of the file content, which is only true in the case of /vsis3/
1098 : * for files not using KMS server side encryption and uploaded in a single
1099 : * PUT operation (so smaller than 50 MB given the default used by GDAL).
1100 : * Only to be used for /vsis3/, /vsigs/ or other filesystems using a
1101 : * MD5Sum as ETAG.
1102 : *
1103 : * The OVERWRITE strategy (GDAL >= 3.2) will always overwrite the target
1104 : * file with the source one.
1105 : * </li>
1106 : * <li>NUM_THREADS=integer. (GDAL >= 3.1) Number of threads to use for parallel
1107 : * file copying. Only use for when /vsis3/, /vsigs/, /vsiaz/ or /vsiadls/ is in
1108 : * source or target. The default is 10 since GDAL 3.3</li>
1109 : * <li>CHUNK_SIZE=integer. (GDAL >= 3.1) Maximum size of chunk (in bytes) to use
1110 : * to split large objects when downloading them from /vsis3/, /vsigs/, /vsiaz/
1111 : * or /vsiadls/ to local file system, or for upload to /vsis3/, /vsiaz/ or
1112 : * /vsiadls/ from local file system. Only used if NUM_THREADS > 1. For upload to
1113 : * /vsis3/, this chunk size must be set at least to 5 MB. The default is 8 MB
1114 : * since GDAL 3.3</li> <li>x-amz-KEY=value. (GDAL >= 3.5) MIME header to pass
1115 : * during creation of a /vsis3/ object.</li> <li>x-goog-KEY=value. (GDAL >= 3.5)
1116 : * MIME header to pass during creation of a /vsigs/ object.</li>
1117 : * <li>x-ms-KEY=value. (GDAL >= 3.5) MIME header to pass during creation of a
1118 : * /vsiaz/ or /vsiadls/ object.</li>
1119 : * </ul>
1120 : * @param pProgressFunc Progress callback, or NULL.
1121 : * @param pProgressData User data of progress callback, or NULL.
1122 : * @param ppapszOutputs Unused. Should be set to NULL for now.
1123 : *
1124 : * @return TRUE on success or FALSE on an error.
1125 : * @since GDAL 2.4
1126 : */
1127 :
1128 49 : int VSISync(const char *pszSource, const char *pszTarget,
1129 : const char *const *papszOptions, GDALProgressFunc pProgressFunc,
1130 : void *pProgressData, char ***ppapszOutputs)
1131 :
1132 : {
1133 49 : if (pszSource[0] == '\0' || pszTarget[0] == '\0')
1134 : {
1135 0 : return FALSE;
1136 : }
1137 :
1138 : VSIFilesystemHandler *poFSHandlerSource =
1139 49 : VSIFileManager::GetHandler(pszSource);
1140 : VSIFilesystemHandler *poFSHandlerTarget =
1141 49 : VSIFileManager::GetHandler(pszTarget);
1142 49 : VSIFilesystemHandler *poFSHandlerLocal = VSIFileManager::GetHandler("");
1143 : VSIFilesystemHandler *poFSHandlerMem =
1144 49 : VSIFileManager::GetHandler("/vsimem/");
1145 49 : VSIFilesystemHandler *poFSHandler = poFSHandlerSource;
1146 49 : if (poFSHandlerTarget != poFSHandlerLocal &&
1147 : poFSHandlerTarget != poFSHandlerMem)
1148 : {
1149 22 : poFSHandler = poFSHandlerTarget;
1150 : }
1151 :
1152 98 : return poFSHandler->Sync(pszSource, pszTarget, papszOptions, pProgressFunc,
1153 49 : pProgressData, ppapszOutputs)
1154 49 : ? TRUE
1155 49 : : FALSE;
1156 : }
1157 :
1158 : /************************************************************************/
1159 : /* VSIMultipartUploadGetCapabilities() */
1160 : /************************************************************************/
1161 :
1162 : /**
1163 : * \brief Return capabilities for multiple part file upload.
1164 : *
1165 : * @param pszFilename Filename, or virtual file system prefix, onto which
1166 : * capabilities should apply.
1167 : * @param[out] pbNonSequentialUploadSupported If not null,
1168 : * the pointed value is set if parts can be uploaded in a non-sequential way.
1169 : * @param[out] pbParallelUploadSupported If not null,
1170 : * the pointed value is set if parts can be uploaded in a parallel way.
1171 : * (implies *pbNonSequentialUploadSupported = true)
1172 : * @param[out] pbAbortSupported If not null,
1173 : * the pointed value is set if VSIMultipartUploadAbort() is implemented.
1174 : * @param[out] pnMinPartSize If not null, the pointed value is set to the minimum
1175 : * size of parts (but the last one), in MiB.
1176 : * @param[out] pnMaxPartSize If not null, the pointed value is set to the maximum
1177 : * size of parts, in MiB.
1178 : * @param[out] pnMaxPartCount If not null, the pointed value is set to the
1179 : * maximum number of parts that can be uploaded.
1180 : *
1181 : * @return TRUE in case of success, FALSE otherwise.
1182 : *
1183 : * @since 3.10
1184 : */
1185 7 : int VSIMultipartUploadGetCapabilities(
1186 : const char *pszFilename, int *pbNonSequentialUploadSupported,
1187 : int *pbParallelUploadSupported, int *pbAbortSupported,
1188 : size_t *pnMinPartSize, size_t *pnMaxPartSize, int *pnMaxPartCount)
1189 : {
1190 7 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
1191 :
1192 14 : return poFSHandler->MultipartUploadGetCapabilities(
1193 : pbNonSequentialUploadSupported, pbParallelUploadSupported,
1194 7 : pbAbortSupported, pnMinPartSize, pnMaxPartSize, pnMaxPartCount);
1195 : }
1196 :
1197 : /************************************************************************/
1198 : /* VSIMultipartUploadStart() */
1199 : /************************************************************************/
1200 :
1201 : /**
1202 : * \brief Initiates the upload a (big) file in a piece-wise way.
1203 : *
1204 : * Using this API directly is generally not needed, but in very advanced cases,
1205 : * as VSIFOpenL(..., "wb") + VSIFWriteL(), VSISync(), VSICopyFile() or
1206 : * VSICopyFileRestartable() may be able to leverage it when needed.
1207 : *
1208 : * This is only implemented for the /vsis3/, /vsigs/, /vsiaz/, /vsiadls/ and
1209 : * /vsioss/ virtual file systems.
1210 : *
1211 : * The typical workflow is to do :
1212 : * - VSIMultipartUploadStart()
1213 : * - VSIMultipartUploadAddPart(): several times
1214 : * - VSIMultipartUploadEnd()
1215 : *
1216 : * If VSIMultipartUploadAbort() is supported by the filesystem (VSIMultipartUploadGetCapabilities()
1217 : * can be used to determine it), this function should be called to cancel an
1218 : * upload. This can be needed to avoid extra billing for some cloud storage
1219 : * providers.
1220 : *
1221 : * The following options are supported:
1222 : * <ul>
1223 : * <li>MIME headers such as Content-Type and Content-Encoding
1224 : * are supported for the /vsis3/, /vsigs/, /vsiaz/, /vsiadls/ file systems.</li>
1225 : * </ul>
1226 : *
1227 : * @param pszFilename Filename to create
1228 : * @param papszOptions NULL or null-terminated list of options.
1229 : * @return an upload ID to pass to other VSIMultipartUploadXXXXX() functions,
1230 : * and to free with CPLFree() once done, or nullptr in case of error.
1231 : *
1232 : * @since 3.10
1233 : */
1234 4 : char *VSIMultipartUploadStart(const char *pszFilename,
1235 : CSLConstList papszOptions)
1236 : {
1237 4 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
1238 :
1239 4 : return poFSHandler->MultipartUploadStart(pszFilename, papszOptions);
1240 : }
1241 :
1242 : /************************************************************************/
1243 : /* VSIMultipartUploadAddPart() */
1244 : /************************************************************************/
1245 :
1246 : /**
1247 : * \brief Uploads a new part to a multi-part uploaded file.
1248 : *
1249 : * Cf VSIMultipartUploadStart().
1250 : *
1251 : * VSIMultipartUploadGetCapabilities() returns hints on the constraints that
1252 : * apply to the upload, in terms of minimum/maximum size of each part, maximum
1253 : * number of parts, and whether non-sequential or parallel uploads are
1254 : * supported.
1255 : *
1256 : * @param pszFilename Filename to which to append the new part. Should be the
1257 : * same as the one used for VSIMultipartUploadStart()
1258 : * @param pszUploadId Value returned by VSIMultipartUploadStart()
1259 : * @param nPartNumber Part number, starting at 1.
1260 : * @param nFileOffset Offset within the file at which (starts at 0) the passed
1261 : * data starts.
1262 : * @param pData Pointer to an array of nDataLength bytes.
1263 : * @param nDataLength Size in bytes of pData.
1264 : * @param papszOptions Unused. Should be nullptr.
1265 : *
1266 : * @return a part identifier that must be passed into the apszPartIds[] array of
1267 : * VSIMultipartUploadEnd(), and to free with CPLFree() once done, or nullptr in
1268 : * case of error.
1269 : *
1270 : * @since 3.10
1271 : */
1272 5 : char *VSIMultipartUploadAddPart(const char *pszFilename,
1273 : const char *pszUploadId, int nPartNumber,
1274 : vsi_l_offset nFileOffset, const void *pData,
1275 : size_t nDataLength, CSLConstList papszOptions)
1276 : {
1277 5 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
1278 :
1279 5 : return poFSHandler->MultipartUploadAddPart(pszFilename, pszUploadId,
1280 : nPartNumber, nFileOffset, pData,
1281 5 : nDataLength, papszOptions);
1282 : }
1283 :
1284 : /************************************************************************/
1285 : /* VSIMultipartUploadEnd() */
1286 : /************************************************************************/
1287 :
1288 : /**
1289 : * \brief Completes a multi-part file upload.
1290 : *
1291 : * Cf VSIMultipartUploadStart().
1292 : *
1293 : * @param pszFilename Filename for which multipart upload should be completed.
1294 : * Should be the same as the one used for
1295 : * VSIMultipartUploadStart()
1296 : * @param pszUploadId Value returned by VSIMultipartUploadStart()
1297 : * @param nPartIdsCount Number of parts, andsize of apszPartIds
1298 : * @param apszPartIds Array of part identifiers (as returned by
1299 : * VSIMultipartUploadAddPart()), that must be ordered in
1300 : * the sequential order of parts, and of size nPartIdsCount.
1301 : * @param nTotalSize Total size of the file in bytes (must be equal to the sum
1302 : * of nDataLength passed to VSIMultipartUploadAddPart())
1303 : * @param papszOptions Unused. Should be nullptr.
1304 : *
1305 : * @return TRUE in case of success, FALSE in case of failure.
1306 : *
1307 : * @since 3.10
1308 : */
1309 5 : int VSIMultipartUploadEnd(const char *pszFilename, const char *pszUploadId,
1310 : size_t nPartIdsCount, const char *const *apszPartIds,
1311 : vsi_l_offset nTotalSize, CSLConstList papszOptions)
1312 : {
1313 5 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
1314 :
1315 10 : return poFSHandler->MultipartUploadEnd(pszFilename, pszUploadId,
1316 : nPartIdsCount, apszPartIds,
1317 5 : nTotalSize, papszOptions);
1318 : }
1319 :
1320 : /************************************************************************/
1321 : /* VSIMultipartUploadAbort() */
1322 : /************************************************************************/
1323 :
1324 : /**
1325 : * \brief Aborts a multi-part file upload.
1326 : *
1327 : * Cf VSIMultipartUploadStart().
1328 : *
1329 : * This function is not implemented for all virtual file systems.
1330 : * Use VSIMultipartUploadGetCapabilities() to determine if it is supported.
1331 : *
1332 : * This can be needed to avoid extra billing for some cloud storage providers.
1333 : *
1334 : * @param pszFilename Filename for which multipart upload should be completed.
1335 : * Should be the same as the one used for
1336 : * VSIMultipartUploadStart()
1337 : * @param pszUploadId Value returned by VSIMultipartUploadStart()
1338 : * @param papszOptions Unused. Should be nullptr.
1339 : *
1340 : * @return TRUE in case of success, FALSE in case of failure.
1341 : *
1342 : * @since 3.10
1343 : */
1344 6 : int VSIMultipartUploadAbort(const char *pszFilename, const char *pszUploadId,
1345 : CSLConstList papszOptions)
1346 : {
1347 6 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
1348 :
1349 12 : return poFSHandler->MultipartUploadAbort(pszFilename, pszUploadId,
1350 6 : papszOptions);
1351 : }
1352 :
1353 : #ifndef DOXYGEN_SKIP
1354 :
1355 : /************************************************************************/
1356 : /* MultipartUploadGetCapabilities() */
1357 : /************************************************************************/
1358 :
1359 2 : bool VSIFilesystemHandler::MultipartUploadGetCapabilities(int *, int *, int *,
1360 : size_t *, size_t *,
1361 : int *)
1362 : {
1363 2 : CPLError(
1364 : CE_Failure, CPLE_NotSupported,
1365 : "MultipartUploadGetCapabilities() not supported by this file system");
1366 2 : return false;
1367 : }
1368 :
1369 : /************************************************************************/
1370 : /* MultipartUploadStart() */
1371 : /************************************************************************/
1372 :
1373 1 : char *VSIFilesystemHandler::MultipartUploadStart(const char *, CSLConstList)
1374 : {
1375 1 : CPLError(CE_Failure, CPLE_NotSupported,
1376 : "MultipartUploadStart() not supported by this file system");
1377 1 : return nullptr;
1378 : }
1379 :
1380 : /************************************************************************/
1381 : /* MultipartUploadAddPart() */
1382 : /************************************************************************/
1383 :
1384 1 : char *VSIFilesystemHandler::MultipartUploadAddPart(const char *, const char *,
1385 : int, vsi_l_offset,
1386 : const void *, size_t,
1387 : CSLConstList)
1388 : {
1389 1 : CPLError(CE_Failure, CPLE_NotSupported,
1390 : "MultipartUploadAddPart() not supported by this file system");
1391 1 : return nullptr;
1392 : }
1393 :
1394 : /************************************************************************/
1395 : /* MultipartUploadEnd() */
1396 : /************************************************************************/
1397 :
1398 1 : bool VSIFilesystemHandler::MultipartUploadEnd(const char *, const char *,
1399 : size_t, const char *const *,
1400 : vsi_l_offset, CSLConstList)
1401 : {
1402 1 : CPLError(CE_Failure, CPLE_NotSupported,
1403 : "MultipartUploadEnd() not supported by this file system");
1404 1 : return FALSE;
1405 : }
1406 :
1407 : /************************************************************************/
1408 : /* MultipartUploadAbort() */
1409 : /************************************************************************/
1410 :
1411 1 : bool VSIFilesystemHandler::MultipartUploadAbort(const char *, const char *,
1412 : CSLConstList)
1413 : {
1414 1 : CPLError(CE_Failure, CPLE_NotSupported,
1415 : "MultipartUploadAbort() not supported by this file system");
1416 1 : return FALSE;
1417 : }
1418 :
1419 : #endif
1420 :
1421 : /************************************************************************/
1422 : /* VSIAbortPendingUploads() */
1423 : /************************************************************************/
1424 :
1425 : /**
1426 : * \brief Abort all ongoing multi-part uploads.
1427 : *
1428 : * Abort ongoing multi-part uploads on AWS S3 and Google Cloud Storage. This
1429 : * can be used in case a process doing such uploads was killed in a unclean way.
1430 : *
1431 : * This can be needed to avoid extra billing for some cloud storage providers.
1432 : *
1433 : * Without effect on other virtual file systems.
1434 : *
1435 : * VSIMultipartUploadAbort() can also be used to cancel a given upload, if the
1436 : * upload ID is known.
1437 : *
1438 : * @param pszFilename filename or prefix of a directory into which multipart
1439 : * uploads must be aborted. This can be the root directory of a bucket. UTF-8
1440 : * encoded.
1441 : *
1442 : * @return TRUE on success or FALSE on an error.
1443 : * @since GDAL 3.4
1444 : */
1445 :
1446 1 : int VSIAbortPendingUploads(const char *pszFilename)
1447 : {
1448 1 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
1449 :
1450 1 : return poFSHandler->AbortPendingUploads(pszFilename);
1451 : }
1452 :
1453 : /************************************************************************/
1454 : /* VSIRmdir() */
1455 : /************************************************************************/
1456 :
1457 : /**
1458 : * \brief Delete a directory.
1459 : *
1460 : * Deletes a directory object from the file system. On some systems
1461 : * the directory must be empty before it can be deleted.
1462 : *
1463 : * This method goes through the VSIFileHandler virtualization and may
1464 : * work on unusual filesystems such as in memory.
1465 : *
1466 : * Analog of the POSIX rmdir() function.
1467 : *
1468 : * @param pszDirname the path of the directory to be deleted. UTF-8 encoded.
1469 : *
1470 : * @return 0 on success or -1 on an error.
1471 : */
1472 :
1473 164 : int VSIRmdir(const char *pszDirname)
1474 :
1475 : {
1476 164 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszDirname);
1477 :
1478 164 : return poFSHandler->Rmdir(pszDirname);
1479 : }
1480 :
1481 : /************************************************************************/
1482 : /* VSIRmdirRecursive() */
1483 : /************************************************************************/
1484 :
1485 : /**
1486 : * \brief Delete a directory recursively
1487 : *
1488 : * Deletes a directory object and its content from the file system.
1489 : *
1490 : * Starting with GDAL 3.1, /vsis3/ has an efficient implementation of this
1491 : * function.
1492 : * Starting with GDAL 3.4, /vsigs/ has an efficient implementation of this
1493 : * function, provided that OAuth2 authentication is used.
1494 : *
1495 : * @return 0 on success or -1 on an error.
1496 : * @since GDAL 2.3
1497 : */
1498 :
1499 5058 : int VSIRmdirRecursive(const char *pszDirname)
1500 : {
1501 5058 : if (pszDirname == nullptr || pszDirname[0] == '\0' ||
1502 5058 : strncmp("/", pszDirname, 2) == 0)
1503 : {
1504 0 : return -1;
1505 : }
1506 5058 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszDirname);
1507 5058 : return poFSHandler->RmdirRecursive(pszDirname);
1508 : }
1509 :
1510 : /************************************************************************/
1511 : /* VSIStatL() */
1512 : /************************************************************************/
1513 :
1514 : /**
1515 : * \brief Get filesystem object info.
1516 : *
1517 : * Fetches status information about a filesystem object (file, directory, etc).
1518 : * The returned information is placed in the VSIStatBufL structure. For
1519 : * portability, only use the st_size (size in bytes) and st_mode (file type).
1520 : * This method is similar to VSIStat(), but will work on large files on
1521 : * systems where this requires special calls.
1522 : *
1523 : * This method goes through the VSIFileHandler virtualization and may
1524 : * work on unusual filesystems such as in memory.
1525 : *
1526 : * Analog of the POSIX stat() function.
1527 : *
1528 : * @param pszFilename the path of the filesystem object to be queried.
1529 : * UTF-8 encoded.
1530 : * @param psStatBuf the structure to load with information.
1531 : *
1532 : * @return 0 on success or -1 on an error.
1533 : */
1534 :
1535 442092 : int VSIStatL(const char *pszFilename, VSIStatBufL *psStatBuf)
1536 :
1537 : {
1538 442092 : return VSIStatExL(pszFilename, psStatBuf, 0);
1539 : }
1540 :
1541 : /************************************************************************/
1542 : /* VSIStatExL() */
1543 : /************************************************************************/
1544 :
1545 : /**
1546 : * \brief Get filesystem object info.
1547 : *
1548 : * Fetches status information about a filesystem object (file, directory, etc).
1549 : * The returned information is placed in the VSIStatBufL structure. For
1550 : * portability, only use the st_size (size in bytes) and st_mode (file type).
1551 : * This method is similar to VSIStat(), but will work on large files on
1552 : * systems where this requires special calls.
1553 : *
1554 : * This method goes through the VSIFileHandler virtualization and may
1555 : * work on unusual filesystems such as in memory.
1556 : *
1557 : * Analog of the POSIX stat() function, with an extra parameter to
1558 : * specify which information is needed, which offers a potential for
1559 : * speed optimizations on specialized and potentially slow virtual
1560 : * filesystem objects (/vsigzip/, /vsicurl/)
1561 : *
1562 : * @param pszFilename the path of the filesystem object to be queried.
1563 : * UTF-8 encoded.
1564 : * @param psStatBuf the structure to load with information.
1565 : * @param nFlags 0 to get all information, or VSI_STAT_EXISTS_FLAG,
1566 : * VSI_STAT_NATURE_FLAG, VSI_STAT_SIZE_FLAG,
1567 : * VSI_STAT_SET_ERROR_FLAG, VSI_STAT_CACHE_ONLY or a combination of those to get
1568 : * partial info.
1569 : *
1570 : * @return 0 on success or -1 on an error.
1571 : *
1572 : * @since GDAL 1.8.0
1573 : */
1574 :
1575 832479 : int VSIStatExL(const char *pszFilename, VSIStatBufL *psStatBuf, int nFlags)
1576 :
1577 : {
1578 832479 : char szAltPath[4] = {'\0'};
1579 :
1580 : // Enable to work on "C:" as if it were "C:\".
1581 832479 : if (pszFilename[0] != '\0' && pszFilename[1] == ':' &&
1582 20 : pszFilename[2] == '\0')
1583 : {
1584 0 : szAltPath[0] = pszFilename[0];
1585 0 : szAltPath[1] = pszFilename[1];
1586 0 : szAltPath[2] = '\\';
1587 0 : szAltPath[3] = '\0';
1588 :
1589 0 : pszFilename = szAltPath;
1590 : }
1591 :
1592 832479 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
1593 :
1594 832480 : if (nFlags == 0)
1595 445917 : nFlags =
1596 : VSI_STAT_EXISTS_FLAG | VSI_STAT_NATURE_FLAG | VSI_STAT_SIZE_FLAG;
1597 :
1598 1664960 : return poFSHandler->Stat(pszFilename, psStatBuf, nFlags);
1599 : }
1600 :
1601 : /************************************************************************/
1602 : /* VSIGetFileMetadata() */
1603 : /************************************************************************/
1604 :
1605 : /**
1606 : * \brief Get metadata on files.
1607 : *
1608 : * Implemented currently only for network-like filesystems, or starting
1609 : * with GDAL 3.7 for /vsizip/
1610 : *
1611 : * Starting with GDAL 3.11, calling it with pszFilename being the root of a
1612 : * /vsigs/ bucket and pszDomain == nullptr, and when authenticated through
1613 : * OAuth2, will result in returning the result of a "Buckets: get"
1614 : * operation (https://cloud.google.com/storage/docs/json_api/v1/buckets/get),
1615 : * with the keys of the top-level JSON document as keys of the key=value pairs
1616 : * returned by this function.
1617 : *
1618 : * @param pszFilename the path of the filesystem object to be queried.
1619 : * UTF-8 encoded.
1620 : * @param pszDomain Metadata domain to query. Depends on the file system.
1621 : * The following ones are supported:
1622 : * <ul>
1623 : * <li>HEADERS: to get HTTP headers for network-like filesystems (/vsicurl/,
1624 : * /vsis3/, /vsgis/, etc)</li>
1625 : * <li>TAGS:
1626 : * <ul>
1627 : * <li>/vsis3/: to get S3 Object tagging information</li>
1628 : * <li>/vsiaz/: to get blob tags. Refer to
1629 : * https://docs.microsoft.com/en-us/rest/api/storageservices/get-blob-tags
1630 : * </li>
1631 : * </ul>
1632 : * </li>
1633 : * <li>STATUS: specific to /vsiadls/: returns all system defined properties for
1634 : * a path (seems in practice to be a subset of HEADERS)</li> <li>ACL: specific
1635 : * to /vsiadls/ and /vsigs/: returns the access control list for a path. For
1636 : * /vsigs/, a single XML=xml_content string is returned. Refer to
1637 : * https://cloud.google.com/storage/docs/xml-api/get-object-acls
1638 : * </li>
1639 : * <li>METADATA: specific to /vsiaz/: to get blob metadata. Refer to
1640 : * https://docs.microsoft.com/en-us/rest/api/storageservices/get-blob-metadata.
1641 : * Note: this will be a subset of what pszDomain=HEADERS returns</li>
1642 : * <li>ZIP: specific to /vsizip/: to obtain ZIP specific metadata, in particular
1643 : * if a file is SOZIP-enabled (SOZIP_VALID=YES)</li>
1644 : * </ul>
1645 : * @param papszOptions Unused. Should be set to NULL.
1646 : *
1647 : * @return a NULL-terminated list of key=value strings, to be freed with
1648 : * CSLDestroy() or NULL in case of error / empty list.
1649 : *
1650 : * @since GDAL 3.1.0
1651 : */
1652 :
1653 91 : char **VSIGetFileMetadata(const char *pszFilename, const char *pszDomain,
1654 : CSLConstList papszOptions)
1655 : {
1656 91 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
1657 91 : return poFSHandler->GetFileMetadata(pszFilename, pszDomain, papszOptions);
1658 : }
1659 :
1660 : /************************************************************************/
1661 : /* VSISetFileMetadata() */
1662 : /************************************************************************/
1663 :
1664 : /**
1665 : * \brief Set metadata on files.
1666 : *
1667 : * Implemented currently only for /vsis3/, /vsigs/, /vsiaz/ and /vsiadls/
1668 : *
1669 : * @param pszFilename the path of the filesystem object to be set.
1670 : * UTF-8 encoded.
1671 : * @param papszMetadata NULL-terminated list of key=value strings.
1672 : * @param pszDomain Metadata domain to set. Depends on the file system.
1673 : * The following are supported:
1674 : * <ul>
1675 : * <li>HEADERS: specific to /vsis3/ and /vsigs/: to set HTTP headers, such as
1676 : * "Content-Type", or other file system specific header.
1677 : * For /vsigs/, this also includes: x-goog-meta-{key}={value}. Note that you
1678 : * should specify all metadata to be set, as existing metadata will be
1679 : * overridden.
1680 : * </li>
1681 : * <li>TAGS: Content of papszMetadata should be KEY=VALUE pairs.
1682 : * <ul>
1683 : * <li>/vsis3/: to set S3 Object tagging information</li>
1684 : * <li>/vsiaz/: to set blob tags. Refer to
1685 : * https://docs.microsoft.com/en-us/rest/api/storageservices/set-blob-tags.
1686 : * Note: storageV2 must be enabled on the account</li>
1687 : * </ul>
1688 : * </li>
1689 : * <li>PROPERTIES:
1690 : * <ul>
1691 : * <li>to /vsiaz/: to set properties. Refer to
1692 : * https://docs.microsoft.com/en-us/rest/api/storageservices/set-blob-properties.</li>
1693 : * <li>to /vsiadls/: to set properties. Refer to
1694 : * https://docs.microsoft.com/en-us/rest/api/storageservices/datalakestoragegen2/path/update
1695 : * for headers valid for action=setProperties.</li>
1696 : * </ul>
1697 : * </li>
1698 : * <li>ACL: specific to /vsiadls/ and /vsigs/: to set access control list.
1699 : * For /vsiadls/, refer to
1700 : * https://docs.microsoft.com/en-us/rest/api/storageservices/datalakestoragegen2/path/update
1701 : * for headers valid for action=setAccessControl or setAccessControlRecursive.
1702 : * In setAccessControlRecursive, x-ms-acl must be specified in papszMetadata.
1703 : * For /vsigs/, refer to
1704 : * https://cloud.google.com/storage/docs/xml-api/put-object-acls. A single
1705 : * XML=xml_content string should be specified as in papszMetadata.
1706 : * </li>
1707 : * <li>METADATA: specific to /vsiaz/: to set blob metadata. Refer to
1708 : * https://docs.microsoft.com/en-us/rest/api/storageservices/set-blob-metadata.
1709 : * Content of papszMetadata should be strings in the form
1710 : * x-ms-meta-name=value</li>
1711 : * </ul>
1712 : * @param papszOptions NULL or NULL terminated list of options.
1713 : * For /vsiadls/ and pszDomain=ACL, "RECURSIVE=TRUE" can be
1714 : * set to set the access control list recursively. When
1715 : * RECURSIVE=TRUE is set, MODE should also be set to one of
1716 : * "set", "modify" or "remove".
1717 : *
1718 : * @return TRUE in case of success.
1719 : *
1720 : * @since GDAL 3.1.0
1721 : */
1722 :
1723 17 : int VSISetFileMetadata(const char *pszFilename, CSLConstList papszMetadata,
1724 : const char *pszDomain, CSLConstList papszOptions)
1725 : {
1726 17 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
1727 34 : return poFSHandler->SetFileMetadata(pszFilename, papszMetadata, pszDomain,
1728 17 : papszOptions)
1729 17 : ? 1
1730 17 : : 0;
1731 : }
1732 :
1733 : /************************************************************************/
1734 : /* VSIIsCaseSensitiveFS() */
1735 : /************************************************************************/
1736 :
1737 : /**
1738 : * \brief Returns if the filenames of the filesystem are case sensitive.
1739 : *
1740 : * This method retrieves to which filesystem belongs the passed filename
1741 : * and return TRUE if the filenames of that filesystem are case sensitive.
1742 : *
1743 : * Currently, this will return FALSE only for Windows real filenames. Other
1744 : * VSI virtual filesystems are case sensitive.
1745 : *
1746 : * This methods avoid ugly \#ifndef _WIN32 / \#endif code, that is wrong when
1747 : * dealing with virtual filenames.
1748 : *
1749 : * @param pszFilename the path of the filesystem object to be tested.
1750 : * UTF-8 encoded.
1751 : *
1752 : * @return TRUE if the filenames of the filesystem are case sensitive.
1753 : *
1754 : * @since GDAL 1.8.0
1755 : */
1756 :
1757 27927 : int VSIIsCaseSensitiveFS(const char *pszFilename)
1758 : {
1759 27927 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
1760 :
1761 27927 : return poFSHandler->IsCaseSensitive(pszFilename);
1762 : }
1763 :
1764 : /************************************************************************/
1765 : /* VSISupportsSparseFiles() */
1766 : /************************************************************************/
1767 :
1768 : /**
1769 : * \brief Returns if the filesystem supports sparse files.
1770 : *
1771 : * Only supported on Linux (and no other Unix derivatives) and
1772 : * Windows. On Linux, the answer depends on a few hardcoded
1773 : * signatures for common filesystems. Other filesystems will be
1774 : * considered as not supporting sparse files.
1775 : *
1776 : * @param pszPath the path of the filesystem object to be tested.
1777 : * UTF-8 encoded.
1778 : *
1779 : * @return TRUE if the file system is known to support sparse files. FALSE may
1780 : * be returned both in cases where it is known to not support them,
1781 : * or when it is unknown.
1782 : *
1783 : * @since GDAL 2.2
1784 : */
1785 :
1786 2 : int VSISupportsSparseFiles(const char *pszPath)
1787 : {
1788 2 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszPath);
1789 :
1790 2 : return poFSHandler->SupportsSparseFiles(pszPath);
1791 : }
1792 :
1793 : /************************************************************************/
1794 : /* VSIIsLocal() */
1795 : /************************************************************************/
1796 :
1797 : /**
1798 : * \brief Returns if the file/filesystem is "local".
1799 : *
1800 : * The concept of local is mostly by opposition with a network / remote
1801 : * file system whose access time can be long.
1802 : *
1803 : * /vsimem/ is considered to be a local file system, although a non-persistent
1804 : * one.
1805 : *
1806 : * @param pszPath the path of the filesystem object to be tested.
1807 : * UTF-8 encoded.
1808 : *
1809 : * @return TRUE or FALSE
1810 : *
1811 : * @since GDAL 3.6
1812 : */
1813 :
1814 263 : bool VSIIsLocal(const char *pszPath)
1815 : {
1816 263 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszPath);
1817 :
1818 263 : return poFSHandler->IsLocal(pszPath);
1819 : }
1820 :
1821 : /************************************************************************/
1822 : /* VSIGetCanonicalFilename() */
1823 : /************************************************************************/
1824 :
1825 : /**
1826 : * \brief Returns the canonical filename.
1827 : *
1828 : * May be implemented by case-insensitive filesystems
1829 : * (currently Win32 and MacOSX) to return the filename with its actual case
1830 : * (i.e. the one that would be used when listing the content of the directory).
1831 : *
1832 : * @param pszPath UTF-8 encoded path
1833 : *
1834 : * @return UTF-8 encoded string, to free with VSIFree()
1835 : *
1836 : * @since GDAL 3.8
1837 : */
1838 :
1839 252 : char *VSIGetCanonicalFilename(const char *pszPath)
1840 : {
1841 252 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszPath);
1842 :
1843 252 : return CPLStrdup(poFSHandler->GetCanonicalFilename(pszPath).c_str());
1844 : }
1845 :
1846 : /************************************************************************/
1847 : /* VSISupportsSequentialWrite() */
1848 : /************************************************************************/
1849 :
1850 : /**
1851 : * \brief Returns if the filesystem supports sequential write.
1852 : *
1853 : * @param pszPath the path of the filesystem object to be tested.
1854 : * UTF-8 encoded.
1855 : * @param bAllowLocalTempFile whether the file system is allowed to use a
1856 : * local temporary file before uploading to the target location.
1857 : *
1858 : * @return TRUE or FALSE
1859 : *
1860 : * @since GDAL 3.6
1861 : */
1862 :
1863 113 : bool VSISupportsSequentialWrite(const char *pszPath, bool bAllowLocalTempFile)
1864 : {
1865 113 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszPath);
1866 :
1867 113 : return poFSHandler->SupportsSequentialWrite(pszPath, bAllowLocalTempFile);
1868 : }
1869 :
1870 : /************************************************************************/
1871 : /* VSISupportsRandomWrite() */
1872 : /************************************************************************/
1873 :
1874 : /**
1875 : * \brief Returns if the filesystem supports random write.
1876 : *
1877 : * @param pszPath the path of the filesystem object to be tested.
1878 : * UTF-8 encoded.
1879 : * @param bAllowLocalTempFile whether the file system is allowed to use a
1880 : * local temporary file before uploading to the target location.
1881 : *
1882 : * @return TRUE or FALSE
1883 : *
1884 : * @since GDAL 3.6
1885 : */
1886 :
1887 340 : bool VSISupportsRandomWrite(const char *pszPath, bool bAllowLocalTempFile)
1888 : {
1889 340 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszPath);
1890 :
1891 340 : return poFSHandler->SupportsRandomWrite(pszPath, bAllowLocalTempFile);
1892 : }
1893 :
1894 : /************************************************************************/
1895 : /* VSIHasOptimizedReadMultiRange() */
1896 : /************************************************************************/
1897 :
1898 : /**
1899 : * \brief Returns if the filesystem supports efficient multi-range reading.
1900 : *
1901 : * Currently only returns TRUE for /vsicurl/ and derived file systems.
1902 : *
1903 : * @param pszPath the path of the filesystem object to be tested.
1904 : * UTF-8 encoded.
1905 : *
1906 : * @return TRUE if the file system is known to have an efficient multi-range
1907 : * reading.
1908 : *
1909 : * @since GDAL 2.3
1910 : */
1911 :
1912 15402 : int VSIHasOptimizedReadMultiRange(const char *pszPath)
1913 : {
1914 15402 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszPath);
1915 :
1916 15404 : return poFSHandler->HasOptimizedReadMultiRange(pszPath);
1917 : }
1918 :
1919 : /************************************************************************/
1920 : /* VSIGetActualURL() */
1921 : /************************************************************************/
1922 :
1923 : /**
1924 : * \brief Returns the actual URL of a supplied filename.
1925 : *
1926 : * Currently only returns a non-NULL value for network-based virtual file
1927 : * systems. For example "/vsis3/bucket/filename" will be expanded as
1928 : * "https://bucket.s3.amazon.com/filename"
1929 : *
1930 : * Note that the lifetime of the returned string, is short, and may be
1931 : * invalidated by any following GDAL functions.
1932 : *
1933 : * @param pszFilename the path of the filesystem object. UTF-8 encoded.
1934 : *
1935 : * @return the actual URL corresponding to the supplied filename, or NULL.
1936 : * Should not be freed.
1937 : *
1938 : * @since GDAL 2.3
1939 : */
1940 :
1941 9 : const char *VSIGetActualURL(const char *pszFilename)
1942 : {
1943 9 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
1944 :
1945 9 : return poFSHandler->GetActualURL(pszFilename);
1946 : }
1947 :
1948 : /************************************************************************/
1949 : /* VSIGetSignedURL() */
1950 : /************************************************************************/
1951 :
1952 : /**
1953 : * \brief Returns a signed URL of a supplied filename.
1954 : *
1955 : * Currently only returns a non-NULL value for /vsis3/, /vsigs/, /vsiaz/ and
1956 : * /vsioss/ For example "/vsis3/bucket/filename" will be expanded as
1957 : * "https://bucket.s3.amazon.com/filename?X-Amz-Algorithm=AWS4-HMAC-SHA256..."
1958 : * Configuration options that apply for file opening (typically to provide
1959 : * credentials), and are returned by VSIGetFileSystemOptions(), are also valid
1960 : * in that context.
1961 : *
1962 : * @param pszFilename the path of the filesystem object. UTF-8 encoded.
1963 : * @param papszOptions list of options, or NULL. Depend on file system handler.
1964 : * For /vsis3/, /vsigs/, /vsiaz/ and /vsioss/, the following options are
1965 : * supported: <ul> <li>START_DATE=YYMMDDTHHMMSSZ: date and time in UTC following
1966 : * ISO 8601 standard, corresponding to the start of validity of the URL. If not
1967 : * specified, current date time.</li> <li>EXPIRATION_DELAY=number_of_seconds:
1968 : * number between 1 and 604800 (seven days) for the validity of the signed URL.
1969 : * Defaults to 3600 (one hour)</li> <li>VERB=GET/HEAD/DELETE/PUT/POST: HTTP VERB
1970 : * for which the request will be used. Default to GET.</li>
1971 : * </ul>
1972 : *
1973 : * /vsiaz/ supports additional options:
1974 : * <ul>
1975 : * <li>SIGNEDIDENTIFIER=value: to relate the given shared access signature
1976 : * to a corresponding stored access policy.</li>
1977 : * <li>SIGNEDPERMISSIONS=r|w: permissions associated with the shared access
1978 : * signature. Normally deduced from VERB.</li>
1979 : * </ul>
1980 : *
1981 : * @return a signed URL, or NULL. Should be freed with CPLFree().
1982 : * @since GDAL 2.3
1983 : */
1984 :
1985 25 : char *VSIGetSignedURL(const char *pszFilename, CSLConstList papszOptions)
1986 : {
1987 25 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
1988 :
1989 25 : return poFSHandler->GetSignedURL(pszFilename, papszOptions);
1990 : }
1991 :
1992 : /************************************************************************/
1993 : /* VSIFOpenL() */
1994 : /************************************************************************/
1995 :
1996 : /**
1997 : * \brief Open file.
1998 : *
1999 : * This function opens a file with the desired access. Large files (larger
2000 : * than 2GB) should be supported. Binary access is always implied and
2001 : * the "b" does not need to be included in the pszAccess string.
2002 : *
2003 : * Note that the "VSILFILE *" returned since GDAL 1.8.0 by this function is
2004 : * *NOT* a standard C library FILE *, and cannot be used with any functions
2005 : * other than the "VSI*L" family of functions. They aren't "real" FILE objects.
2006 : *
2007 : * On windows it is possible to define the configuration option
2008 : * GDAL_FILE_IS_UTF8 to have pszFilename treated as being in the local
2009 : * encoding instead of UTF-8, restoring the pre-1.8.0 behavior of VSIFOpenL().
2010 : *
2011 : * This method goes through the VSIFileHandler virtualization and may
2012 : * work on unusual filesystems such as in memory.
2013 : *
2014 : * Analog of the POSIX fopen() function.
2015 : *
2016 : * @param pszFilename the file to open. UTF-8 encoded.
2017 : * @param pszAccess access requested (i.e. "r", "r+", "w")
2018 : *
2019 : * @return NULL on failure, or the file handle.
2020 : */
2021 :
2022 232182 : VSILFILE *VSIFOpenL(const char *pszFilename, const char *pszAccess)
2023 :
2024 : {
2025 232182 : return VSIFOpenExL(pszFilename, pszAccess, false);
2026 : }
2027 :
2028 : /************************************************************************/
2029 : /* Open() */
2030 : /************************************************************************/
2031 :
2032 : #ifndef DOXYGEN_SKIP
2033 :
2034 : VSIVirtualHandleUniquePtr
2035 16415 : VSIFilesystemHandler::OpenStatic(const char *pszFilename, const char *pszAccess,
2036 : bool bSetError, CSLConstList papszOptions)
2037 : {
2038 16415 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
2039 :
2040 16415 : return poFSHandler->Open(pszFilename, pszAccess, bSetError, papszOptions);
2041 : }
2042 :
2043 : /************************************************************************/
2044 : /* CopyFile() */
2045 : /************************************************************************/
2046 :
2047 2273 : int VSIFilesystemHandler::CopyFile(const char *pszSource, const char *pszTarget,
2048 : VSILFILE *fpSource, vsi_l_offset nSourceSize,
2049 : CSLConstList papszOptions,
2050 : GDALProgressFunc pProgressFunc,
2051 : void *pProgressData)
2052 : {
2053 2273 : VSIVirtualHandleUniquePtr poFileHandleAutoClose;
2054 2273 : if (!fpSource)
2055 : {
2056 2256 : CPLAssert(pszSource);
2057 2256 : fpSource = VSIFOpenExL(pszSource, "rb", TRUE);
2058 2256 : if (!fpSource)
2059 : {
2060 1 : CPLError(CE_Failure, CPLE_FileIO, "Cannot open %s", pszSource);
2061 1 : return -1;
2062 : }
2063 2255 : poFileHandleAutoClose.reset(fpSource);
2064 : }
2065 2272 : if (nSourceSize == static_cast<vsi_l_offset>(-1) &&
2066 3 : pProgressFunc != nullptr && pszSource != nullptr)
2067 : {
2068 : VSIStatBufL sStat;
2069 3 : if (VSIStatL(pszSource, &sStat) == 0)
2070 : {
2071 3 : nSourceSize = sStat.st_size;
2072 : }
2073 : }
2074 :
2075 2272 : VSILFILE *fpOut = VSIFOpenEx2L(pszTarget, "wb", TRUE, papszOptions);
2076 2272 : if (!fpOut)
2077 : {
2078 3 : CPLError(CE_Failure, CPLE_FileIO, "Cannot create %s", pszTarget);
2079 3 : return -1;
2080 : }
2081 :
2082 4538 : CPLString osMsg;
2083 2269 : if (pszSource)
2084 2267 : osMsg.Printf("Copying of %s", pszSource);
2085 : else
2086 2 : pszSource = "(unknown filename)";
2087 :
2088 2269 : int ret = 0;
2089 2269 : constexpr size_t nBufferSize = 10 * 4096;
2090 2269 : std::vector<GByte> abyBuffer(nBufferSize, 0);
2091 2269 : GUIntBig nOffset = 0;
2092 : while (true)
2093 : {
2094 2571 : const size_t nRead = VSIFReadL(&abyBuffer[0], 1, nBufferSize, fpSource);
2095 2571 : if (nRead < nBufferSize && VSIFErrorL(fpSource))
2096 : {
2097 7 : CPLError(
2098 : CE_Failure, CPLE_FileIO,
2099 : "Copying of %s to %s failed: error while reading source file",
2100 : pszSource, pszTarget);
2101 7 : ret = -1;
2102 7 : break;
2103 : }
2104 2564 : if (nRead > 0)
2105 : {
2106 2563 : const size_t nWritten = VSIFWriteL(&abyBuffer[0], 1, nRead, fpOut);
2107 2563 : if (nWritten != nRead)
2108 : {
2109 20 : CPLError(CE_Failure, CPLE_FileIO,
2110 : "Copying of %s to %s failed: error while writing into "
2111 : "target file",
2112 : pszSource, pszTarget);
2113 20 : ret = -1;
2114 20 : break;
2115 : }
2116 2543 : nOffset += nRead;
2117 2592 : if (pProgressFunc &&
2118 98 : !pProgressFunc(
2119 : nSourceSize == 0 ? 1.0
2120 49 : : nSourceSize > 0 &&
2121 : nSourceSize != static_cast<vsi_l_offset>(-1)
2122 98 : ? double(nOffset) / nSourceSize
2123 : : 0.0,
2124 49 : !osMsg.empty() ? osMsg.c_str() : nullptr, pProgressData))
2125 : {
2126 1 : ret = -1;
2127 1 : break;
2128 : }
2129 : }
2130 2543 : if (nRead < nBufferSize)
2131 : {
2132 2241 : break;
2133 : }
2134 302 : }
2135 :
2136 2269 : if (nSourceSize != static_cast<vsi_l_offset>(-1) && nOffset != nSourceSize)
2137 : {
2138 2 : CPLError(CE_Failure, CPLE_FileIO,
2139 : "Copying of %s to %s failed: %" PRIu64 " bytes were copied "
2140 : "whereas %" PRIu64 " were expected",
2141 : pszSource, pszTarget, static_cast<uint64_t>(nOffset),
2142 : static_cast<uint64_t>(nSourceSize));
2143 2 : ret = -1;
2144 : }
2145 :
2146 2269 : if (VSIFCloseL(fpOut) != 0)
2147 : {
2148 1 : ret = -1;
2149 : }
2150 :
2151 2269 : if (ret != 0)
2152 29 : VSIUnlink(pszTarget);
2153 :
2154 2269 : return ret;
2155 : }
2156 :
2157 : /************************************************************************/
2158 : /* CopyFileRestartable() */
2159 : /************************************************************************/
2160 :
2161 2 : int VSIFilesystemHandler::CopyFileRestartable(
2162 : const char *pszSource, const char *pszTarget,
2163 : const char * /* pszInputPayload */, char **ppszOutputPayload,
2164 : CSLConstList papszOptions, GDALProgressFunc pProgressFunc,
2165 : void *pProgressData)
2166 : {
2167 2 : *ppszOutputPayload = nullptr;
2168 2 : return CopyFile(pszSource, pszTarget, nullptr,
2169 : static_cast<vsi_l_offset>(-1), papszOptions, pProgressFunc,
2170 2 : pProgressData);
2171 : }
2172 :
2173 : /************************************************************************/
2174 : /* Sync() */
2175 : /************************************************************************/
2176 :
2177 31 : bool VSIFilesystemHandler::Sync(const char *pszSource, const char *pszTarget,
2178 : const char *const *papszOptions,
2179 : GDALProgressFunc pProgressFunc,
2180 : void *pProgressData, char ***ppapszOutputs)
2181 : {
2182 31 : const char SOURCE_SEP = VSIGetDirectorySeparator(pszSource)[0];
2183 :
2184 31 : if (ppapszOutputs)
2185 : {
2186 0 : *ppapszOutputs = nullptr;
2187 : }
2188 :
2189 : VSIStatBufL sSource;
2190 62 : CPLString osSource(pszSource);
2191 62 : CPLString osSourceWithoutSlash(pszSource);
2192 40 : if (osSourceWithoutSlash.back() == '/' ||
2193 9 : osSourceWithoutSlash.back() == '\\')
2194 : {
2195 22 : osSourceWithoutSlash.pop_back();
2196 : }
2197 31 : if (VSIStatL(osSourceWithoutSlash, &sSource) < 0)
2198 : {
2199 2 : CPLError(CE_Failure, CPLE_FileIO, "%s does not exist", pszSource);
2200 2 : return false;
2201 : }
2202 :
2203 29 : if (VSI_ISDIR(sSource.st_mode))
2204 : {
2205 22 : std::string osTargetDir(pszTarget);
2206 11 : if (osSource.back() != '/' && osSource.back() != '\\')
2207 : {
2208 2 : osTargetDir = CPLFormFilenameSafe(
2209 1 : osTargetDir.c_str(), CPLGetFilename(pszSource), nullptr);
2210 : }
2211 :
2212 : VSIStatBufL sTarget;
2213 11 : bool ret = true;
2214 11 : if (VSIStatL(osTargetDir.c_str(), &sTarget) < 0)
2215 : {
2216 9 : if (VSIMkdirRecursive(osTargetDir.c_str(), 0755) < 0)
2217 : {
2218 1 : CPLError(CE_Failure, CPLE_FileIO, "Cannot create directory %s",
2219 : osTargetDir.c_str());
2220 1 : return false;
2221 : }
2222 : }
2223 :
2224 10 : if (!CPLFetchBool(papszOptions, "STOP_ON_DIR", false))
2225 : {
2226 20 : CPLStringList aosChildOptions(CSLDuplicate(papszOptions));
2227 10 : if (!CPLFetchBool(papszOptions, "RECURSIVE", true))
2228 : {
2229 0 : aosChildOptions.SetNameValue("RECURSIVE", nullptr);
2230 0 : aosChildOptions.AddString("STOP_ON_DIR=TRUE");
2231 : }
2232 :
2233 10 : char **papszSrcFiles = VSIReadDir(osSourceWithoutSlash);
2234 10 : int nFileCount = 0;
2235 27 : for (auto iter = papszSrcFiles; iter && *iter; ++iter)
2236 : {
2237 17 : if (strcmp(*iter, ".") != 0 && strcmp(*iter, "..") != 0)
2238 : {
2239 17 : nFileCount++;
2240 : }
2241 : }
2242 10 : int iFile = 0;
2243 10 : const int nDenom = std::max(1, nFileCount);
2244 27 : for (auto iter = papszSrcFiles; iter && *iter; ++iter, ++iFile)
2245 : {
2246 17 : if (strcmp(*iter, ".") == 0 || strcmp(*iter, "..") == 0)
2247 : {
2248 0 : continue;
2249 : }
2250 : const std::string osSubSource(CPLFormFilenameSafe(
2251 17 : osSourceWithoutSlash.c_str(), *iter, nullptr));
2252 : const std::string osSubTarget(
2253 17 : CPLFormFilenameSafe(osTargetDir.c_str(), *iter, nullptr));
2254 34 : void *pScaledProgress = GDALCreateScaledProgress(
2255 17 : double(iFile) / nDenom, double(iFile + 1) / nDenom,
2256 : pProgressFunc, pProgressData);
2257 17 : ret = Sync((osSubSource + SOURCE_SEP).c_str(),
2258 17 : osSubTarget.c_str(), aosChildOptions.List(),
2259 17 : GDALScaledProgress, pScaledProgress, nullptr);
2260 17 : GDALDestroyScaledProgress(pScaledProgress);
2261 17 : if (!ret)
2262 : {
2263 0 : break;
2264 : }
2265 : }
2266 10 : CSLDestroy(papszSrcFiles);
2267 : }
2268 10 : return ret;
2269 : }
2270 :
2271 : VSIStatBufL sTarget;
2272 36 : std::string osTarget(pszTarget);
2273 18 : if (VSIStatL(osTarget.c_str(), &sTarget) == 0)
2274 : {
2275 4 : bool bTargetIsFile = true;
2276 4 : if (VSI_ISDIR(sTarget.st_mode))
2277 : {
2278 4 : osTarget = CPLFormFilenameSafe(osTarget.c_str(),
2279 2 : CPLGetFilename(pszSource), nullptr);
2280 3 : bTargetIsFile = VSIStatL(osTarget.c_str(), &sTarget) == 0 &&
2281 1 : !CPL_TO_BOOL(VSI_ISDIR(sTarget.st_mode));
2282 : }
2283 4 : if (bTargetIsFile)
2284 : {
2285 3 : if (sSource.st_size == sTarget.st_size &&
2286 3 : sSource.st_mtime == sTarget.st_mtime && sSource.st_mtime != 0)
2287 : {
2288 2 : CPLDebug("VSI",
2289 : "%s and %s have same size and modification "
2290 : "date. Skipping copying",
2291 : osSourceWithoutSlash.c_str(), osTarget.c_str());
2292 2 : return true;
2293 : }
2294 : }
2295 : }
2296 :
2297 16 : VSILFILE *fpIn = VSIFOpenExL(osSourceWithoutSlash, "rb", TRUE);
2298 16 : if (fpIn == nullptr)
2299 : {
2300 0 : CPLError(CE_Failure, CPLE_FileIO, "Cannot open %s",
2301 : osSourceWithoutSlash.c_str());
2302 0 : return false;
2303 : }
2304 :
2305 16 : VSILFILE *fpOut = VSIFOpenExL(osTarget.c_str(), "wb", TRUE);
2306 16 : if (fpOut == nullptr)
2307 : {
2308 2 : CPLError(CE_Failure, CPLE_FileIO, "Cannot create %s", osTarget.c_str());
2309 2 : VSIFCloseL(fpIn);
2310 2 : return false;
2311 : }
2312 :
2313 14 : bool ret = true;
2314 14 : constexpr size_t nBufferSize = 10 * 4096;
2315 28 : std::vector<GByte> abyBuffer(nBufferSize, 0);
2316 14 : GUIntBig nOffset = 0;
2317 14 : CPLString osMsg;
2318 14 : osMsg.Printf("Copying of %s", osSourceWithoutSlash.c_str());
2319 : while (true)
2320 : {
2321 17 : size_t nRead = VSIFReadL(&abyBuffer[0], 1, nBufferSize, fpIn);
2322 17 : size_t nWritten = VSIFWriteL(&abyBuffer[0], 1, nRead, fpOut);
2323 17 : if (nWritten != nRead)
2324 : {
2325 0 : CPLError(CE_Failure, CPLE_FileIO, "Copying of %s to %s failed",
2326 : osSourceWithoutSlash.c_str(), osTarget.c_str());
2327 0 : ret = false;
2328 0 : break;
2329 : }
2330 17 : nOffset += nRead;
2331 17 : if (pProgressFunc && !pProgressFunc(double(nOffset) / sSource.st_size,
2332 : osMsg.c_str(), pProgressData))
2333 : {
2334 0 : ret = false;
2335 0 : break;
2336 : }
2337 17 : if (nRead < nBufferSize)
2338 : {
2339 14 : break;
2340 : }
2341 3 : }
2342 :
2343 14 : VSIFCloseL(fpIn);
2344 14 : if (VSIFCloseL(fpOut) != 0)
2345 : {
2346 0 : ret = false;
2347 : }
2348 14 : return ret;
2349 : }
2350 :
2351 : /************************************************************************/
2352 : /* VSIVirtualHandleOnlyVisibleAtCloseTime() */
2353 : /************************************************************************/
2354 :
2355 : class VSIVirtualHandleOnlyVisibleAtCloseTime final : public VSIProxyFileHandle
2356 : {
2357 : const std::string m_osTargetName;
2358 : const std::string m_osTmpName;
2359 : bool m_bAlreadyClosed = false;
2360 : bool m_bCancelCreation = false;
2361 :
2362 : public:
2363 200 : VSIVirtualHandleOnlyVisibleAtCloseTime(
2364 : VSIVirtualHandleUniquePtr &&nativeHandle,
2365 : const std::string &osTargetName, const std::string &osTmpName)
2366 200 : : VSIProxyFileHandle(std::move(nativeHandle)),
2367 200 : m_osTargetName(osTargetName), m_osTmpName(osTmpName)
2368 : {
2369 200 : }
2370 :
2371 400 : ~VSIVirtualHandleOnlyVisibleAtCloseTime() override
2372 200 : {
2373 200 : VSIVirtualHandleOnlyVisibleAtCloseTime::Close();
2374 400 : }
2375 :
2376 8 : void CancelCreation() override
2377 : {
2378 8 : VSIProxyFileHandle::CancelCreation();
2379 8 : m_bCancelCreation = true;
2380 8 : }
2381 :
2382 : int Close() override;
2383 : };
2384 :
2385 569 : int VSIVirtualHandleOnlyVisibleAtCloseTime::Close()
2386 : {
2387 569 : if (m_bAlreadyClosed)
2388 369 : return 0;
2389 200 : m_bAlreadyClosed = true;
2390 200 : int ret = VSIProxyFileHandle::Close();
2391 200 : if (ret == 0)
2392 : {
2393 200 : if (m_bCancelCreation)
2394 : {
2395 5 : ret = VSIUnlink(m_osTmpName.c_str());
2396 : VSIStatBufL sStatBuf;
2397 5 : if (ret != 0 && VSIStatL(m_osTmpName.c_str(), &sStatBuf) != 0)
2398 4 : ret = 0;
2399 : }
2400 : else
2401 : {
2402 195 : ret = VSIRename(m_osTmpName.c_str(), m_osTargetName.c_str());
2403 : }
2404 : }
2405 200 : return ret;
2406 : }
2407 :
2408 : /************************************************************************/
2409 : /* CreateOnlyVisibleAtCloseTime() */
2410 : /************************************************************************/
2411 :
2412 200 : VSIVirtualHandleUniquePtr VSIFilesystemHandler::CreateOnlyVisibleAtCloseTime(
2413 : const char *pszFilename, bool bEmulationAllowed, CSLConstList papszOptions)
2414 : {
2415 200 : if (!bEmulationAllowed)
2416 0 : return nullptr;
2417 :
2418 600 : const std::string tmpName = std::string(pszFilename).append(".tmp");
2419 : VSIVirtualHandleUniquePtr nativeHandle(
2420 400 : Open(tmpName.c_str(), "wb+", true, papszOptions));
2421 200 : if (!nativeHandle)
2422 0 : return nullptr;
2423 : return VSIVirtualHandleUniquePtr(
2424 400 : std::make_unique<VSIVirtualHandleOnlyVisibleAtCloseTime>(
2425 200 : std::move(nativeHandle), pszFilename, tmpName)
2426 200 : .release());
2427 : }
2428 :
2429 : /************************************************************************/
2430 : /* VSIDIREntry() */
2431 : /************************************************************************/
2432 :
2433 3927 : VSIDIREntry::VSIDIREntry()
2434 : : pszName(nullptr), nMode(0), nSize(0), nMTime(0), bModeKnown(false),
2435 3927 : bSizeKnown(false), bMTimeKnown(false), papszExtra(nullptr)
2436 : {
2437 3927 : }
2438 :
2439 : /************************************************************************/
2440 : /* VSIDIREntry() */
2441 : /************************************************************************/
2442 :
2443 6 : VSIDIREntry::VSIDIREntry(const VSIDIREntry &other)
2444 6 : : pszName(VSIStrdup(other.pszName)), nMode(other.nMode), nSize(other.nSize),
2445 6 : nMTime(other.nMTime), bModeKnown(other.bModeKnown),
2446 6 : bSizeKnown(other.bSizeKnown), bMTimeKnown(other.bMTimeKnown),
2447 6 : papszExtra(CSLDuplicate(other.papszExtra))
2448 : {
2449 6 : }
2450 :
2451 : /************************************************************************/
2452 : /* ~VSIDIREntry() */
2453 : /************************************************************************/
2454 :
2455 7866 : VSIDIREntry::~VSIDIREntry()
2456 : {
2457 3933 : CPLFree(pszName);
2458 3933 : CSLDestroy(papszExtra);
2459 3933 : }
2460 :
2461 : /************************************************************************/
2462 : /* ~VSIDIR() */
2463 : /************************************************************************/
2464 :
2465 3532 : VSIDIR::~VSIDIR()
2466 : {
2467 3532 : }
2468 :
2469 : /************************************************************************/
2470 : /* VSIDIRGeneric */
2471 : /************************************************************************/
2472 :
2473 : namespace
2474 : {
2475 : struct VSIDIRGeneric : public VSIDIR
2476 : {
2477 : CPLString osRootPath{};
2478 : CPLString osBasePath{};
2479 : char **papszContent = nullptr;
2480 : int nRecurseDepth = 0;
2481 : int nPos = 0;
2482 : VSIDIREntry entry{};
2483 : std::vector<VSIDIRGeneric *> aoStackSubDir{};
2484 : VSIFilesystemHandler *poFS = nullptr;
2485 : std::string m_osFilterPrefix{};
2486 :
2487 2666 : explicit VSIDIRGeneric(VSIFilesystemHandler *poFSIn) : poFS(poFSIn)
2488 : {
2489 2666 : }
2490 :
2491 : ~VSIDIRGeneric() override;
2492 :
2493 : const VSIDIREntry *NextDirEntry() override;
2494 :
2495 : VSIDIRGeneric(const VSIDIRGeneric &) = delete;
2496 : VSIDIRGeneric &operator=(const VSIDIRGeneric &) = delete;
2497 : };
2498 :
2499 : /************************************************************************/
2500 : /* ~VSIDIRGeneric() */
2501 : /************************************************************************/
2502 :
2503 7998 : VSIDIRGeneric::~VSIDIRGeneric()
2504 : {
2505 2673 : while (!aoStackSubDir.empty())
2506 : {
2507 7 : delete aoStackSubDir.back();
2508 7 : aoStackSubDir.pop_back();
2509 : }
2510 2666 : CSLDestroy(papszContent);
2511 5332 : }
2512 :
2513 : } // namespace
2514 :
2515 : /************************************************************************/
2516 : /* OpenDir() */
2517 : /************************************************************************/
2518 :
2519 2676 : VSIDIR *VSIFilesystemHandler::OpenDir(const char *pszPath, int nRecurseDepth,
2520 : const char *const *papszOptions)
2521 : {
2522 2676 : char **papszContent = VSIReadDir(pszPath);
2523 : VSIStatBufL sStatL;
2524 3202 : if (papszContent == nullptr &&
2525 526 : (VSIStatL(pszPath, &sStatL) != 0 || !VSI_ISDIR(sStatL.st_mode)))
2526 : {
2527 10 : return nullptr;
2528 : }
2529 2666 : VSIDIRGeneric *dir = new VSIDIRGeneric(this);
2530 2666 : dir->osRootPath = pszPath;
2531 5332 : if (!dir->osRootPath.empty() &&
2532 2666 : (dir->osRootPath.back() == '/' || dir->osRootPath.back() == '\\'))
2533 15 : dir->osRootPath.pop_back();
2534 2666 : dir->nRecurseDepth = nRecurseDepth;
2535 2666 : dir->papszContent = papszContent;
2536 2666 : dir->m_osFilterPrefix = CSLFetchNameValueDef(papszOptions, "PREFIX", "");
2537 2666 : return dir;
2538 : }
2539 :
2540 : /************************************************************************/
2541 : /* NextDirEntry() */
2542 : /************************************************************************/
2543 :
2544 508374 : const VSIDIREntry *VSIDIRGeneric::NextDirEntry()
2545 : {
2546 508374 : const char SEP = VSIGetDirectorySeparator(osRootPath.c_str())[0];
2547 :
2548 508375 : begin:
2549 508375 : if (VSI_ISDIR(entry.nMode) && nRecurseDepth != 0)
2550 : {
2551 1342 : CPLString osCurFile(osRootPath);
2552 1342 : if (!osCurFile.empty())
2553 1342 : osCurFile += SEP;
2554 1342 : osCurFile += entry.pszName;
2555 : auto subdir =
2556 1342 : static_cast<VSIDIRGeneric *>(poFS->VSIFilesystemHandler::OpenDir(
2557 1342 : osCurFile, nRecurseDepth - 1, nullptr));
2558 1342 : if (subdir)
2559 : {
2560 1342 : subdir->osRootPath = osRootPath;
2561 1342 : subdir->osBasePath = entry.pszName;
2562 1342 : subdir->m_osFilterPrefix = m_osFilterPrefix;
2563 1342 : aoStackSubDir.push_back(subdir);
2564 : }
2565 1342 : entry.nMode = 0;
2566 : }
2567 :
2568 509710 : while (!aoStackSubDir.empty())
2569 : {
2570 503803 : auto l_entry = aoStackSubDir.back()->NextDirEntry();
2571 503803 : if (l_entry)
2572 : {
2573 502468 : return l_entry;
2574 : }
2575 1335 : delete aoStackSubDir.back();
2576 1335 : aoStackSubDir.pop_back();
2577 : }
2578 :
2579 5907 : if (papszContent == nullptr)
2580 : {
2581 516 : return nullptr;
2582 : }
2583 :
2584 : while (true)
2585 : {
2586 5419 : if (!papszContent[nPos])
2587 : {
2588 2133 : return nullptr;
2589 : }
2590 : // Skip . and ..entries
2591 3286 : if (papszContent[nPos][0] == '.' &&
2592 115 : (papszContent[nPos][1] == '\0' ||
2593 115 : (papszContent[nPos][1] == '.' && papszContent[nPos][2] == '\0')))
2594 : {
2595 23 : nPos++;
2596 : }
2597 : else
2598 : {
2599 3263 : CPLFree(entry.pszName);
2600 3263 : CPLString osName(osBasePath);
2601 3263 : if (!osName.empty())
2602 2032 : osName += SEP;
2603 3263 : osName += papszContent[nPos];
2604 3263 : nPos++;
2605 :
2606 3263 : entry.pszName = CPLStrdup(osName);
2607 3263 : entry.nMode = 0;
2608 3263 : CPLString osCurFile(osRootPath);
2609 3263 : if (!osCurFile.empty())
2610 3263 : osCurFile += SEP;
2611 3263 : osCurFile += entry.pszName;
2612 :
2613 6516 : const auto StatFile = [&osCurFile, this]()
2614 : {
2615 : VSIStatBufL sStatL;
2616 3258 : if (VSIStatL(osCurFile, &sStatL) == 0)
2617 : {
2618 3256 : entry.nMode = sStatL.st_mode;
2619 3256 : entry.nSize = sStatL.st_size;
2620 3256 : entry.nMTime = sStatL.st_mtime;
2621 3256 : entry.bModeKnown = true;
2622 3256 : entry.bSizeKnown = true;
2623 3256 : entry.bMTimeKnown = true;
2624 : }
2625 : else
2626 : {
2627 2 : entry.nMode = 0;
2628 2 : entry.nSize = 0;
2629 2 : entry.nMTime = 0;
2630 2 : entry.bModeKnown = false;
2631 2 : entry.bSizeKnown = false;
2632 2 : entry.bMTimeKnown = false;
2633 : }
2634 3258 : };
2635 :
2636 3275 : if (!m_osFilterPrefix.empty() &&
2637 12 : m_osFilterPrefix.size() > osName.size())
2638 : {
2639 6 : if (STARTS_WITH(m_osFilterPrefix.c_str(), osName.c_str()) &&
2640 2 : m_osFilterPrefix[osName.size()] == SEP)
2641 : {
2642 1 : StatFile();
2643 1 : if (VSI_ISDIR(entry.nMode))
2644 : {
2645 1 : goto begin;
2646 : }
2647 : }
2648 3 : continue;
2649 : }
2650 3267 : if (!m_osFilterPrefix.empty() &&
2651 8 : !STARTS_WITH(osName.c_str(), m_osFilterPrefix.c_str()))
2652 : {
2653 2 : continue;
2654 : }
2655 :
2656 3257 : StatFile();
2657 :
2658 3257 : break;
2659 : }
2660 28 : }
2661 :
2662 3257 : return &(entry);
2663 : }
2664 :
2665 : /************************************************************************/
2666 : /* UnlinkBatch() */
2667 : /************************************************************************/
2668 :
2669 1 : int *VSIFilesystemHandler::UnlinkBatch(CSLConstList papszFiles)
2670 : {
2671 : int *panRet =
2672 1 : static_cast<int *>(CPLMalloc(sizeof(int) * CSLCount(papszFiles)));
2673 3 : for (int i = 0; papszFiles && papszFiles[i]; ++i)
2674 : {
2675 2 : panRet[i] = VSIUnlink(papszFiles[i]) == 0;
2676 : }
2677 1 : return panRet;
2678 : }
2679 :
2680 : /************************************************************************/
2681 : /* RmdirRecursive() */
2682 : /************************************************************************/
2683 :
2684 34 : int VSIFilesystemHandler::RmdirRecursive(const char *pszDirname)
2685 : {
2686 68 : CPLString osDirnameWithoutEndSlash(pszDirname);
2687 68 : if (!osDirnameWithoutEndSlash.empty() &&
2688 34 : (osDirnameWithoutEndSlash.back() == '/' ||
2689 34 : osDirnameWithoutEndSlash.back() == '\\'))
2690 : {
2691 0 : osDirnameWithoutEndSlash.pop_back();
2692 : }
2693 :
2694 34 : const char SEP = VSIGetDirectorySeparator(pszDirname)[0];
2695 :
2696 68 : CPLStringList aosOptions;
2697 : auto poDir =
2698 68 : std::unique_ptr<VSIDIR>(OpenDir(pszDirname, -1, aosOptions.List()));
2699 34 : if (!poDir)
2700 8 : return -1;
2701 52 : std::vector<std::string> aosDirs;
2702 : while (true)
2703 : {
2704 141 : auto entry = poDir->NextDirEntry();
2705 141 : if (!entry)
2706 26 : break;
2707 :
2708 230 : CPLString osFilename(osDirnameWithoutEndSlash + SEP + entry->pszName);
2709 115 : if ((entry->nMode & S_IFDIR))
2710 : {
2711 10 : aosDirs.push_back(std::move(osFilename));
2712 : }
2713 : else
2714 : {
2715 105 : if (VSIUnlink(osFilename) != 0)
2716 0 : return -1;
2717 : }
2718 115 : }
2719 :
2720 : // Sort in reverse order, so that inner-most directories are deleted first
2721 26 : std::sort(aosDirs.begin(), aosDirs.end(),
2722 2 : [](const std::string &a, const std::string &b) { return a > b; });
2723 :
2724 36 : for (const auto &osDir : aosDirs)
2725 : {
2726 10 : if (VSIRmdir(osDir.c_str()) != 0)
2727 0 : return -1;
2728 : }
2729 :
2730 26 : return VSIRmdir(pszDirname);
2731 : }
2732 :
2733 : /************************************************************************/
2734 : /* GetFileMetadata() */
2735 : /************************************************************************/
2736 :
2737 0 : char **VSIFilesystemHandler::GetFileMetadata(const char * /* pszFilename*/,
2738 : const char * /*pszDomain*/,
2739 : CSLConstList /*papszOptions*/)
2740 : {
2741 0 : return nullptr;
2742 : }
2743 :
2744 : /************************************************************************/
2745 : /* SetFileMetadata() */
2746 : /************************************************************************/
2747 :
2748 0 : bool VSIFilesystemHandler::SetFileMetadata(const char * /* pszFilename*/,
2749 : CSLConstList /* papszMetadata */,
2750 : const char * /* pszDomain */,
2751 : CSLConstList /* papszOptions */)
2752 : {
2753 0 : CPLError(CE_Failure, CPLE_NotSupported, "SetFileMetadata() not supported");
2754 0 : return false;
2755 : }
2756 :
2757 : #endif
2758 :
2759 : /************************************************************************/
2760 : /* VSIFOpenExL() */
2761 : /************************************************************************/
2762 :
2763 : /**
2764 : * \brief Open/create file.
2765 : *
2766 : * This function opens (or creates) a file with the desired access.
2767 : * Binary access is always implied and
2768 : * the "b" does not need to be included in the pszAccess string.
2769 : *
2770 : * Note that the "VSILFILE *" returned by this function is
2771 : * *NOT* a standard C library FILE *, and cannot be used with any functions
2772 : * other than the "VSI*L" family of functions. They aren't "real" FILE objects.
2773 : *
2774 : * On windows it is possible to define the configuration option
2775 : * GDAL_FILE_IS_UTF8 to have pszFilename treated as being in the local
2776 : * encoding instead of UTF-8, restoring the pre-1.8.0 behavior of VSIFOpenL().
2777 : *
2778 : * This method goes through the VSIFileHandler virtualization and may
2779 : * work on unusual filesystems such as in memory.
2780 : *
2781 : * Analog of the POSIX fopen() function.
2782 : *
2783 : * @param pszFilename the file to open. UTF-8 encoded.
2784 : * @param pszAccess access requested (i.e. "r", "r+", "w")
2785 : * @param bSetError flag determining whether or not this open call
2786 : * should set VSIErrors on failure.
2787 : *
2788 : * @return NULL on failure, or the file handle.
2789 : *
2790 : * @since GDAL 2.1
2791 : */
2792 :
2793 399353 : VSILFILE *VSIFOpenExL(const char *pszFilename, const char *pszAccess,
2794 : int bSetError)
2795 :
2796 : {
2797 399353 : return VSIFOpenEx2L(pszFilename, pszAccess, bSetError, nullptr);
2798 : }
2799 :
2800 : /************************************************************************/
2801 : /* VSIFOpenEx2L() */
2802 : /************************************************************************/
2803 :
2804 : /**
2805 : * \brief Open/create file.
2806 : *
2807 : * This function opens (or creates) a file with the desired access.
2808 : * Binary access is always implied and
2809 : * the "b" does not need to be included in the pszAccess string.
2810 : *
2811 : * Note that the "VSILFILE *" returned by this function is
2812 : * *NOT* a standard C library FILE *, and cannot be used with any functions
2813 : * other than the "VSI*L" family of functions. They aren't "real" FILE objects.
2814 : *
2815 : * On windows it is possible to define the configuration option
2816 : * GDAL_FILE_IS_UTF8 to have pszFilename treated as being in the local
2817 : * encoding instead of UTF-8, restoring the pre-1.8.0 behavior of VSIFOpenL().
2818 : *
2819 : * This method goes through the VSIFileHandler virtualization and may
2820 : * work on unusual filesystems such as in memory.
2821 : *
2822 : * The following options are supported:
2823 : * <ul>
2824 : * <li>MIME headers such as Content-Type and Content-Encoding
2825 : * are supported for the /vsis3/, /vsigs/, /vsiaz/, /vsiadls/ file systems.</li>
2826 : * <li>DISABLE_READDIR_ON_OPEN=YES/NO (GDAL >= 3.6) for /vsicurl/ and other
2827 : * network-based file systems. By default, directory file listing is done,
2828 : * unless YES is specified.</li>
2829 : * <li>WRITE_THROUGH=YES (GDAL >= 3.8) for the Windows regular files to
2830 : * set the FILE_FLAG_WRITE_THROUGH flag to the CreateFile() function. In that
2831 : * mode, the data is written to the system cache but is flushed to disk without
2832 : * delay.</li>
2833 : * </ul>
2834 : *
2835 : * Options specifics to /vsis3/, /vsigs/, /vsioss/ and /vsiaz/ in "w" mode:
2836 : * <ul>
2837 : * <li>CHUNK_SIZE=val in MiB. (GDAL >= 3.10) Size of a block. Default is 50 MiB.
2838 : * For /vsis3/, /vsigz/, /vsioss/, it can be up to 5000 MiB.
2839 : * For /vsiaz/, only taken into account when BLOB_TYPE=BLOCK. It can be up to 4000 MiB.
2840 : * </li>
2841 : * </ul>
2842 : *
2843 : * Options specifics to /vsiaz/ in "w" mode:
2844 : * <ul>
2845 : * <li>BLOB_TYPE=APPEND/BLOCK. (GDAL >= 3.10) Type of blob. Defaults to APPEND.
2846 : * Append blocks are limited to 195 GiB
2847 : * (however if the file size is below 4 MiB, a block blob will be created in a
2848 : * single PUT operation)
2849 : * </li>
2850 : * </ul>
2851 : *
2852 : * Analog of the POSIX fopen() function.
2853 : *
2854 : * @param pszFilename the file to open. UTF-8 encoded.
2855 : * @param pszAccess access requested (i.e. "r", "r+", "w")
2856 : * @param bSetError flag determining whether or not this open call
2857 : * should set VSIErrors on failure.
2858 : * @param papszOptions NULL or NULL-terminated list of strings. The content is
2859 : * highly file system dependent.
2860 : *
2861 : *
2862 : * @return NULL on failure, or the file handle.
2863 : *
2864 : * @since GDAL 3.3
2865 : */
2866 :
2867 428516 : VSILFILE *VSIFOpenEx2L(const char *pszFilename, const char *pszAccess,
2868 : int bSetError, CSLConstList papszOptions)
2869 :
2870 : {
2871 : // Too long filenames can cause excessive memory allocation due to
2872 : // recursion in some filesystem handlers
2873 428516 : constexpr size_t knMaxPath = 8192;
2874 428516 : if (CPLStrnlen(pszFilename, knMaxPath) == knMaxPath)
2875 5 : return nullptr;
2876 :
2877 428616 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
2878 :
2879 428620 : auto fp = poFSHandler->Open(pszFilename, pszAccess, CPL_TO_BOOL(bSetError),
2880 856587 : papszOptions);
2881 :
2882 : VSIDebug4("VSIFOpenEx2L(%s,%s,%d) = %p", pszFilename, pszAccess, bSetError,
2883 : fp.get());
2884 :
2885 428337 : return fp.release();
2886 : }
2887 :
2888 : /************************************************************************/
2889 : /* VSIFCloseL() */
2890 : /************************************************************************/
2891 :
2892 : /**
2893 : * \fn VSIVirtualHandle::Close()
2894 : * \brief Close file.
2895 : *
2896 : * This function closes the indicated file.
2897 : *
2898 : * This method goes through the VSIFileHandler virtualization and may
2899 : * work on unusual filesystems such as in memory.
2900 : *
2901 : * Analog of the POSIX fclose() function.
2902 : *
2903 : * @return 0 on success or -1 on failure.
2904 : */
2905 :
2906 : /**
2907 : * \brief Close file.
2908 : *
2909 : * This function closes the indicated file.
2910 : *
2911 : * This method goes through the VSIFileHandler virtualization and may
2912 : * work on unusual filesystems such as in memory.
2913 : *
2914 : * Analog of the POSIX fclose() function.
2915 : *
2916 : * @param fp file handle opened with VSIFOpenL(). Passing a nullptr produces
2917 : * undefined behavior.
2918 : *
2919 : * @return 0 on success or -1 on failure.
2920 : */
2921 :
2922 310859 : int VSIFCloseL(VSILFILE *fp)
2923 :
2924 : {
2925 : VSIDebug1("VSIFCloseL(%p)", fp);
2926 :
2927 310859 : const int nResult = fp->Close();
2928 :
2929 310772 : delete fp;
2930 :
2931 310633 : return nResult;
2932 : }
2933 :
2934 : /************************************************************************/
2935 : /* VSIFSeekL() */
2936 : /************************************************************************/
2937 :
2938 : /**
2939 : * \fn int VSIVirtualHandle::Seek( vsi_l_offset nOffset, int nWhence )
2940 : * \brief Seek to requested offset.
2941 : *
2942 : * Seek to the desired offset (nOffset) in the indicated file.
2943 : *
2944 : * This method goes through the VSIFileHandler virtualization and may
2945 : * work on unusual filesystems such as in memory.
2946 : *
2947 : * Analog of the POSIX fseek() call.
2948 : *
2949 : * Caution: vsi_l_offset is a unsigned type, so SEEK_CUR can only be used
2950 : * for positive seek. If negative seek is needed, use
2951 : * handle->Seek( handle->Tell() + negative_offset, SEEK_SET ).
2952 : *
2953 : * @param nOffset offset in bytes.
2954 : * @param nWhence one of SEEK_SET, SEEK_CUR or SEEK_END.
2955 : *
2956 : * @return 0 on success or -1 one failure.
2957 : */
2958 :
2959 : /**
2960 : * \brief Seek to requested offset.
2961 : *
2962 : * Seek to the desired offset (nOffset) in the indicated file.
2963 : *
2964 : * This method goes through the VSIFileHandler virtualization and may
2965 : * work on unusual filesystems such as in memory.
2966 : *
2967 : * Analog of the POSIX fseek() call.
2968 : *
2969 : * Caution: vsi_l_offset is a unsigned type, so SEEK_CUR can only be used
2970 : * for positive seek. If negative seek is needed, use
2971 : * VSIFSeekL( fp, VSIFTellL(fp) + negative_offset, SEEK_SET ).
2972 : *
2973 : * @param fp file handle opened with VSIFOpenL().
2974 : * @param nOffset offset in bytes.
2975 : * @param nWhence one of SEEK_SET, SEEK_CUR or SEEK_END.
2976 : *
2977 : * @return 0 on success or -1 one failure.
2978 : */
2979 :
2980 9097880 : int VSIFSeekL(VSILFILE *fp, vsi_l_offset nOffset, int nWhence)
2981 :
2982 : {
2983 9097880 : return fp->Seek(nOffset, nWhence);
2984 : }
2985 :
2986 : /************************************************************************/
2987 : /* VSIFTellL() */
2988 : /************************************************************************/
2989 :
2990 : /**
2991 : * \fn VSIVirtualHandle::Tell()
2992 : * \brief Tell current file offset.
2993 : *
2994 : * Returns the current file read/write offset in bytes from the beginning of
2995 : * the file.
2996 : *
2997 : * This method goes through the VSIFileHandler virtualization and may
2998 : * work on unusual filesystems such as in memory.
2999 : *
3000 : * Analog of the POSIX ftell() call.
3001 : *
3002 : * @return file offset in bytes.
3003 : */
3004 :
3005 : /**
3006 : * \brief Tell current file offset.
3007 : *
3008 : * Returns the current file read/write offset in bytes from the beginning of
3009 : * the file.
3010 : *
3011 : * This method goes through the VSIFileHandler virtualization and may
3012 : * work on unusual filesystems such as in memory.
3013 : *
3014 : * Analog of the POSIX ftell() call.
3015 : *
3016 : * @param fp file handle opened with VSIFOpenL().
3017 : *
3018 : * @return file offset in bytes.
3019 : */
3020 :
3021 6289140 : vsi_l_offset VSIFTellL(VSILFILE *fp)
3022 :
3023 : {
3024 6289140 : return fp->Tell();
3025 : }
3026 :
3027 : /************************************************************************/
3028 : /* VSIRewindL() */
3029 : /************************************************************************/
3030 :
3031 : /**
3032 : * \brief Rewind the file pointer to the beginning of the file.
3033 : *
3034 : * This is equivalent to VSIFSeekL( fp, 0, SEEK_SET )
3035 : *
3036 : * Analog of the POSIX rewind() call.
3037 : *
3038 : * @param fp file handle opened with VSIFOpenL().
3039 : */
3040 :
3041 87917 : void VSIRewindL(VSILFILE *fp)
3042 :
3043 : {
3044 87917 : CPL_IGNORE_RET_VAL(VSIFSeekL(fp, 0, SEEK_SET));
3045 87925 : }
3046 :
3047 : /************************************************************************/
3048 : /* VSIFFlushL() */
3049 : /************************************************************************/
3050 :
3051 : /**
3052 : * \fn VSIVirtualHandle::Flush()
3053 : * \brief Flush pending writes to disk.
3054 : *
3055 : * For files in write or update mode and on filesystem types where it is
3056 : * applicable, all pending output on the file is flushed to the physical disk.
3057 : *
3058 : * This method goes through the VSIFileHandler virtualization and may
3059 : * work on unusual filesystems such as in memory.
3060 : *
3061 : * Analog of the POSIX fflush() call.
3062 : *
3063 : * On Windows regular files, this method does nothing, unless the
3064 : * VSI_FLUSH configuration option is set to YES (and only when the file has
3065 : * *not* been opened with the WRITE_THROUGH option).
3066 : *
3067 : * @return 0 on success or -1 on error.
3068 : */
3069 :
3070 : /**
3071 : * \brief Flush pending writes to disk.
3072 : *
3073 : * For files in write or update mode and on filesystem types where it is
3074 : * applicable, all pending output on the file is flushed to the physical disk.
3075 : *
3076 : * This method goes through the VSIFileHandler virtualization and may
3077 : * work on unusual filesystems such as in memory.
3078 : *
3079 : * Analog of the POSIX fflush() call.
3080 : *
3081 : * On Windows regular files, this method does nothing, unless the
3082 : * VSI_FLUSH configuration option is set to YES (and only when the file has
3083 : * *not* been opened with the WRITE_THROUGH option).
3084 : *
3085 : * @param fp file handle opened with VSIFOpenL().
3086 : *
3087 : * @return 0 on success or -1 on error.
3088 : */
3089 :
3090 90944 : int VSIFFlushL(VSILFILE *fp)
3091 :
3092 : {
3093 90944 : return fp->Flush();
3094 : }
3095 :
3096 : /************************************************************************/
3097 : /* VSIFReadL() */
3098 : /************************************************************************/
3099 :
3100 : /**
3101 : * \fn VSIVirtualHandle::Read( void *pBuffer, size_t nSize, size_t nCount )
3102 : * \brief Read bytes from file.
3103 : *
3104 : * Reads nCount objects of nSize bytes from the indicated file at the
3105 : * current offset into the indicated buffer.
3106 : *
3107 : * This method goes through the VSIFileHandler virtualization and may
3108 : * work on unusual filesystems such as in memory.
3109 : *
3110 : * Analog of the POSIX fread() call.
3111 : *
3112 : * @param pBuffer the buffer into which the data should be read (at least
3113 : * nCount * nSize bytes in size.
3114 : * @param nSize size of objects to read in bytes.
3115 : * @param nCount number of objects to read.
3116 : *
3117 : * @return number of objects successfully read. If that number is less than
3118 : * nCount, VSIFEofL() or VSIFErrorL() can be used to determine the reason for
3119 : * the short read.
3120 : */
3121 :
3122 : /**
3123 : * \brief Read bytes from file.
3124 : *
3125 : * Reads nCount objects of nSize bytes from the indicated file at the
3126 : * current offset into the indicated buffer.
3127 : *
3128 : * This method goes through the VSIFileHandler virtualization and may
3129 : * work on unusual filesystems such as in memory.
3130 : *
3131 : * Analog of the POSIX fread() call.
3132 : *
3133 : * @param pBuffer the buffer into which the data should be read (at least
3134 : * nCount * nSize bytes in size.
3135 : * @param nSize size of objects to read in bytes.
3136 : * @param nCount number of objects to read.
3137 : * @param fp file handle opened with VSIFOpenL().
3138 : *
3139 : * @return number of objects successfully read. If that number is less than
3140 : * nCount, VSIFEofL() or VSIFErrorL() can be used to determine the reason for
3141 : * the short read.
3142 : */
3143 :
3144 16302800 : size_t VSIFReadL(void *pBuffer, size_t nSize, size_t nCount, VSILFILE *fp)
3145 :
3146 : {
3147 16302800 : return fp->Read(pBuffer, nSize, nCount);
3148 : }
3149 :
3150 : /************************************************************************/
3151 : /* VSIFReadMultiRangeL() */
3152 : /************************************************************************/
3153 :
3154 : /**
3155 : * \fn VSIVirtualHandle::ReadMultiRange( int nRanges, void ** ppData,
3156 : * const vsi_l_offset* panOffsets,
3157 : * const size_t* panSizes )
3158 : * \brief Read several ranges of bytes from file.
3159 : *
3160 : * Reads nRanges objects of panSizes[i] bytes from the indicated file at the
3161 : * offset panOffsets[i] into the buffer ppData[i].
3162 : *
3163 : * Ranges must be sorted in ascending start offset, and must not overlap each
3164 : * other.
3165 : *
3166 : * This method goes through the VSIFileHandler virtualization and may
3167 : * work on unusual filesystems such as in memory or /vsicurl/.
3168 : *
3169 : * @param nRanges number of ranges to read.
3170 : * @param ppData array of nRanges buffer into which the data should be read
3171 : * (ppData[i] must be at list panSizes[i] bytes).
3172 : * @param panOffsets array of nRanges offsets at which the data should be read.
3173 : * @param panSizes array of nRanges sizes of objects to read (in bytes).
3174 : *
3175 : * @return 0 in case of success, -1 otherwise.
3176 : * @since GDAL 1.9.0
3177 : */
3178 :
3179 : /**
3180 : * \brief Read several ranges of bytes from file.
3181 : *
3182 : * Reads nRanges objects of panSizes[i] bytes from the indicated file at the
3183 : * offset panOffsets[i] into the buffer ppData[i].
3184 : *
3185 : * Ranges must be sorted in ascending start offset, and must not overlap each
3186 : * other.
3187 : *
3188 : * This method goes through the VSIFileHandler virtualization and may
3189 : * work on unusual filesystems such as in memory or /vsicurl/.
3190 : *
3191 : * @param nRanges number of ranges to read.
3192 : * @param ppData array of nRanges buffer into which the data should be read
3193 : * (ppData[i] must be at list panSizes[i] bytes).
3194 : * @param panOffsets array of nRanges offsets at which the data should be read.
3195 : * @param panSizes array of nRanges sizes of objects to read (in bytes).
3196 : * @param fp file handle opened with VSIFOpenL().
3197 : *
3198 : * @return 0 in case of success, -1 otherwise.
3199 : * @since GDAL 1.9.0
3200 : */
3201 :
3202 763 : int VSIFReadMultiRangeL(int nRanges, void **ppData,
3203 : const vsi_l_offset *panOffsets, const size_t *panSizes,
3204 : VSILFILE *fp)
3205 : {
3206 763 : return fp->ReadMultiRange(nRanges, ppData, panOffsets, panSizes);
3207 : }
3208 :
3209 : /************************************************************************/
3210 : /* VSIFWriteL() */
3211 : /************************************************************************/
3212 :
3213 : /**
3214 : * \fn VSIVirtualHandle::Write( const void *pBuffer,
3215 : * size_t nSize, size_t nCount )
3216 : * \brief Write bytes to file.
3217 : *
3218 : * Writes nCount objects of nSize bytes to the indicated file at the
3219 : * current offset into the indicated buffer.
3220 : *
3221 : * This method goes through the VSIFileHandler virtualization and may
3222 : * work on unusual filesystems such as in memory.
3223 : *
3224 : * Analog of the POSIX fwrite() call.
3225 : *
3226 : * @param pBuffer the buffer from which the data should be written (at least
3227 : * nCount * nSize bytes in size.
3228 : * @param nSize size of objects to write in bytes.
3229 : * @param nCount number of objects to write.
3230 : *
3231 : * @return number of objects successfully written.
3232 : */
3233 :
3234 : /**
3235 : * \brief Write bytes to file.
3236 : *
3237 : * Writes nCount objects of nSize bytes to the indicated file at the
3238 : * current offset into the indicated buffer.
3239 : *
3240 : * This method goes through the VSIFileHandler virtualization and may
3241 : * work on unusual filesystems such as in memory.
3242 : *
3243 : * Analog of the POSIX fwrite() call.
3244 : *
3245 : * @param pBuffer the buffer from which the data should be written (at least
3246 : * nCount * nSize bytes in size.
3247 : * @param nSize size of objects to write in bytes.
3248 : * @param nCount number of objects to write.
3249 : * @param fp file handle opened with VSIFOpenL().
3250 : *
3251 : * @return number of objects successfully written.
3252 : */
3253 :
3254 4270050 : size_t VSIFWriteL(const void *pBuffer, size_t nSize, size_t nCount,
3255 : VSILFILE *fp)
3256 :
3257 : {
3258 4270050 : return fp->Write(pBuffer, nSize, nCount);
3259 : }
3260 :
3261 : /************************************************************************/
3262 : /* VSIFEofL() */
3263 : /************************************************************************/
3264 :
3265 : /**
3266 : * \fn VSIVirtualHandle::Eof()
3267 : * \brief Test for end of file.
3268 : *
3269 : * Returns TRUE (non-zero) if an end-of-file condition occurred during the
3270 : * previous read operation. The end-of-file flag is cleared by a successful
3271 : * VSIFSeekL() call, or a call to VSIFClearErrL().
3272 : *
3273 : * This method goes through the VSIFileHandler virtualization and may
3274 : * work on unusual filesystems such as in memory.
3275 : *
3276 : * Analog of the POSIX feof() call.
3277 : *
3278 : * @return TRUE if at EOF, else FALSE.
3279 : */
3280 :
3281 : /**
3282 : * \brief Test for end of file.
3283 : *
3284 : * Returns TRUE (non-zero) if an end-of-file condition occurred during the
3285 : * previous read operation. The end-of-file flag is cleared by a successful
3286 : * VSIFSeekL() call, or a call to VSIFClearErrL().
3287 : *
3288 : * This method goes through the VSIFileHandler virtualization and may
3289 : * work on unusual filesystems such as in memory.
3290 : *
3291 : * Analog of the POSIX feof() call.
3292 : *
3293 : * @param fp file handle opened with VSIFOpenL().
3294 : *
3295 : * @return TRUE if at EOF, else FALSE.
3296 : */
3297 :
3298 281875 : int VSIFEofL(VSILFILE *fp)
3299 :
3300 : {
3301 281875 : return fp->Eof();
3302 : }
3303 :
3304 : /************************************************************************/
3305 : /* VSIFErrorL() */
3306 : /************************************************************************/
3307 :
3308 : /**
3309 : * \fn VSIVirtualHandle::Error()
3310 : * \brief Test the error indicator.
3311 : *
3312 : * Returns TRUE (non-zero) if an error condition occurred during the
3313 : * previous read operation. The error indicator is cleared by a call to
3314 : * VSIFClearErrL(). Note that a end-of-file situation, reported by VSIFEofL(),
3315 : * is *not* an error reported by VSIFErrorL().
3316 : *
3317 : * This method goes through the VSIFileHandler virtualization and may
3318 : * work on unusual filesystems such as in memory.
3319 : *
3320 : * Analog of the POSIX ferror() call.
3321 : *
3322 : * @return TRUE if the error indicator is set, else FALSE.
3323 : * @since 3.10
3324 : */
3325 :
3326 : /**
3327 : * \brief Test the error indicator.
3328 : *
3329 : * Returns TRUE (non-zero) if an error condition occurred during the
3330 : * previous read operation. The error indicator is cleared by a call to
3331 : * VSIFClearErrL(). Note that a end-of-file situation, reported by VSIFEofL(),
3332 : * is *not* an error reported by VSIFErrorL().
3333 : *
3334 : * This method goes through the VSIFileHandler virtualization and may
3335 : * work on unusual filesystems such as in memory.
3336 : *
3337 : * Analog of the POSIX feof() call.
3338 : *
3339 : * @param fp file handle opened with VSIFOpenL().
3340 : *
3341 : * @return TRUE if the error indicator is set, else FALSE.
3342 : * @since 3.10
3343 : */
3344 :
3345 90945 : int VSIFErrorL(VSILFILE *fp)
3346 :
3347 : {
3348 90945 : return fp->Error();
3349 : }
3350 :
3351 : /************************************************************************/
3352 : /* VSIFClearErrL() */
3353 : /************************************************************************/
3354 :
3355 : /**
3356 : * \fn VSIVirtualHandle::ClearErr()
3357 : * \brief Reset the error and end-of-file indicators.
3358 : *
3359 : * This method goes through the VSIFileHandler virtualization and may
3360 : * work on unusual filesystems such as in memory.
3361 : *
3362 : * Analog of the POSIX clearerr() call.
3363 : *
3364 : * @since 3.10
3365 : */
3366 :
3367 : /**
3368 : * \brief Reset the error and end-of-file indicators.
3369 : *
3370 : * This method goes through the VSIFileHandler virtualization and may
3371 : * work on unusual filesystems such as in memory.
3372 : *
3373 : * Analog of the POSIX clearerr() call.
3374 : *
3375 : * @param fp file handle opened with VSIFOpenL().
3376 : *
3377 : * @since 3.10
3378 : */
3379 :
3380 31281 : void VSIFClearErrL(VSILFILE *fp)
3381 :
3382 : {
3383 31281 : fp->ClearErr();
3384 31281 : }
3385 :
3386 : /************************************************************************/
3387 : /* VSIFTruncateL() */
3388 : /************************************************************************/
3389 :
3390 : /**
3391 : * \fn VSIVirtualHandle::Truncate( vsi_l_offset nNewSize )
3392 : * \brief Truncate/expand the file to the specified size
3393 :
3394 : * This method goes through the VSIFileHandler virtualization and may
3395 : * work on unusual filesystems such as in memory.
3396 : *
3397 : * Analog of the POSIX ftruncate() call.
3398 : *
3399 : * @param nNewSize new size in bytes.
3400 : *
3401 : * @return 0 on success
3402 : * @since GDAL 1.9.0
3403 : */
3404 :
3405 : /**
3406 : * \brief Truncate/expand the file to the specified size
3407 :
3408 : * This method goes through the VSIFileHandler virtualization and may
3409 : * work on unusual filesystems such as in memory.
3410 : *
3411 : * Analog of the POSIX ftruncate() call.
3412 : *
3413 : * @param fp file handle opened with VSIFOpenL().
3414 : * @param nNewSize new size in bytes.
3415 : *
3416 : * @return 0 on success
3417 : * @since GDAL 1.9.0
3418 : */
3419 :
3420 1305 : int VSIFTruncateL(VSILFILE *fp, vsi_l_offset nNewSize)
3421 :
3422 : {
3423 1305 : return fp->Truncate(nNewSize);
3424 : }
3425 :
3426 : /************************************************************************/
3427 : /* VSIFPrintfL() */
3428 : /************************************************************************/
3429 :
3430 : /**
3431 : * \brief Formatted write to file.
3432 : *
3433 : * Provides fprintf() style formatted output to a VSI*L file. This formats
3434 : * an internal buffer which is written using VSIFWriteL().
3435 : *
3436 : * Analog of the POSIX fprintf() call.
3437 : *
3438 : * @param fp file handle opened with VSIFOpenL().
3439 : * @param pszFormat the printf() style format string.
3440 : *
3441 : * @return the number of bytes written or -1 on an error.
3442 : */
3443 :
3444 79234 : int VSIFPrintfL(VSILFILE *fp, CPL_FORMAT_STRING(const char *pszFormat), ...)
3445 :
3446 : {
3447 : va_list args;
3448 :
3449 79234 : va_start(args, pszFormat);
3450 79234 : CPLString osResult;
3451 79234 : osResult.vPrintf(pszFormat, args);
3452 79234 : va_end(args);
3453 :
3454 : return static_cast<int>(
3455 158468 : VSIFWriteL(osResult.c_str(), 1, osResult.length(), fp));
3456 : }
3457 :
3458 : /************************************************************************/
3459 : /* VSIVirtualHandle::Printf() */
3460 : /************************************************************************/
3461 :
3462 : /**
3463 : * \brief Formatted write to file.
3464 : *
3465 : * Provides fprintf() style formatted output to a VSI*L file. This formats
3466 : * an internal buffer which is written using VSIFWriteL().
3467 : *
3468 : * Analog of the POSIX fprintf() call.
3469 : *
3470 : * @param pszFormat the printf() style format string.
3471 : *
3472 : * @return the number of bytes written or -1 on an error.
3473 : */
3474 :
3475 2601 : int VSIVirtualHandle::Printf(CPL_FORMAT_STRING(const char *pszFormat), ...)
3476 : {
3477 : va_list args;
3478 :
3479 2601 : va_start(args, pszFormat);
3480 2601 : CPLString osResult;
3481 2601 : osResult.vPrintf(pszFormat, args);
3482 2601 : va_end(args);
3483 :
3484 5202 : return static_cast<int>(Write(osResult.c_str(), 1, osResult.length()));
3485 : }
3486 :
3487 : /************************************************************************/
3488 : /* VSIFPutcL() */
3489 : /************************************************************************/
3490 :
3491 : // TODO: should we put in conformance with POSIX regarding the return
3492 : // value. As of today (2015-08-29), no code in GDAL sources actually
3493 : // check the return value.
3494 :
3495 : /**
3496 : * \brief Write a single byte to the file
3497 : *
3498 : * Writes the character nChar, cast to an unsigned char, to file.
3499 : *
3500 : * Almost an analog of the POSIX fputc() call, except that it returns
3501 : * the number of character written (1 or 0), and not the (cast)
3502 : * character itself or EOF.
3503 : *
3504 : * @param nChar character to write.
3505 : * @param fp file handle opened with VSIFOpenL().
3506 : *
3507 : * @return 1 in case of success, 0 on error.
3508 : */
3509 :
3510 476 : int VSIFPutcL(int nChar, VSILFILE *fp)
3511 :
3512 : {
3513 476 : const unsigned char cChar = static_cast<unsigned char>(nChar);
3514 476 : return static_cast<int>(VSIFWriteL(&cChar, 1, 1, fp));
3515 : }
3516 :
3517 : /************************************************************************/
3518 : /* VSIFGetRangeStatusL() */
3519 : /************************************************************************/
3520 :
3521 : /**
3522 : * \fn VSIVirtualHandle::GetRangeStatus( vsi_l_offset nOffset,
3523 : * vsi_l_offset nLength )
3524 : * \brief Return if a given file range contains data or holes filled with zeroes
3525 : *
3526 : * This uses the filesystem capabilities of querying which regions of
3527 : * a sparse file are allocated or not. This is currently only
3528 : * implemented for Linux (and no other Unix derivatives) and Windows.
3529 : *
3530 : * Note: A return of VSI_RANGE_STATUS_DATA doesn't exclude that the
3531 : * extent is filled with zeroes! It must be interpreted as "may
3532 : * contain non-zero data".
3533 : *
3534 : * @param nOffset offset of the start of the extent.
3535 : * @param nLength extent length.
3536 : *
3537 : * @return extent status: VSI_RANGE_STATUS_UNKNOWN, VSI_RANGE_STATUS_DATA or
3538 : * VSI_RANGE_STATUS_HOLE
3539 : * @since GDAL 2.2
3540 : */
3541 :
3542 : /**
3543 : * \brief Return if a given file range contains data or holes filled with zeroes
3544 : *
3545 : * This uses the filesystem capabilities of querying which regions of
3546 : * a sparse file are allocated or not. This is currently only
3547 : * implemented for Linux (and no other Unix derivatives) and Windows.
3548 : *
3549 : * Note: A return of VSI_RANGE_STATUS_DATA doesn't exclude that the
3550 : * extent is filled with zeroes! It must be interpreted as "may
3551 : * contain non-zero data".
3552 : *
3553 : * @param fp file handle opened with VSIFOpenL().
3554 : * @param nOffset offset of the start of the extent.
3555 : * @param nLength extent length.
3556 : *
3557 : * @return extent status: VSI_RANGE_STATUS_UNKNOWN, VSI_RANGE_STATUS_DATA or
3558 : * VSI_RANGE_STATUS_HOLE
3559 : * @since GDAL 2.2
3560 : */
3561 :
3562 680 : VSIRangeStatus VSIFGetRangeStatusL(VSILFILE *fp, vsi_l_offset nOffset,
3563 : vsi_l_offset nLength)
3564 : {
3565 680 : return fp->GetRangeStatus(nOffset, nLength);
3566 : }
3567 :
3568 : /************************************************************************/
3569 : /* VSIIngestFile() */
3570 : /************************************************************************/
3571 :
3572 : /**
3573 : * \brief Ingest a file into memory.
3574 : *
3575 : * Read the whole content of a file into a memory buffer.
3576 : *
3577 : * Either fp or pszFilename can be NULL, but not both at the same time.
3578 : *
3579 : * If fp is passed non-NULL, it is the responsibility of the caller to
3580 : * close it.
3581 : *
3582 : * If non-NULL, the returned buffer is guaranteed to be NUL-terminated.
3583 : *
3584 : * @param fp file handle opened with VSIFOpenL().
3585 : * @param pszFilename filename.
3586 : * @param ppabyRet pointer to the target buffer. *ppabyRet must be freed with
3587 : * VSIFree()
3588 : * @param pnSize pointer to variable to store the file size. May be NULL.
3589 : * @param nMaxSize maximum size of file allowed. If no limit, set to a negative
3590 : * value.
3591 : *
3592 : * @return TRUE in case of success.
3593 : *
3594 : * @since GDAL 1.11
3595 : */
3596 :
3597 13192 : int VSIIngestFile(VSILFILE *fp, const char *pszFilename, GByte **ppabyRet,
3598 : vsi_l_offset *pnSize, GIntBig nMaxSize)
3599 : {
3600 13192 : if (fp == nullptr && pszFilename == nullptr)
3601 0 : return FALSE;
3602 13192 : if (ppabyRet == nullptr)
3603 0 : return FALSE;
3604 :
3605 13192 : *ppabyRet = nullptr;
3606 13192 : if (pnSize != nullptr)
3607 6310 : *pnSize = 0;
3608 :
3609 13192 : bool bFreeFP = false;
3610 13192 : if (nullptr == fp)
3611 : {
3612 11950 : fp = VSIFOpenL(pszFilename, "rb");
3613 11950 : if (nullptr == fp)
3614 : {
3615 919 : CPLError(CE_Failure, CPLE_FileIO, "Cannot open file '%s'",
3616 : pszFilename);
3617 919 : return FALSE;
3618 : }
3619 11031 : bFreeFP = true;
3620 : }
3621 : else
3622 : {
3623 1242 : if (VSIFSeekL(fp, 0, SEEK_SET) != 0)
3624 0 : return FALSE;
3625 : }
3626 :
3627 12273 : vsi_l_offset nDataLen = 0;
3628 :
3629 12273 : if (pszFilename == nullptr || strcmp(pszFilename, "/vsistdin/") == 0)
3630 : {
3631 100 : vsi_l_offset nDataAlloc = 0;
3632 100 : if (VSIFSeekL(fp, 0, SEEK_SET) != 0)
3633 : {
3634 0 : if (bFreeFP)
3635 0 : CPL_IGNORE_RET_VAL(VSIFCloseL(fp));
3636 0 : return FALSE;
3637 : }
3638 : while (true)
3639 : {
3640 507 : if (nDataLen + 8192 + 1 > nDataAlloc)
3641 : {
3642 233 : nDataAlloc = (nDataAlloc * 4) / 3 + 8192 + 1;
3643 : if (nDataAlloc >
3644 : static_cast<vsi_l_offset>(static_cast<size_t>(nDataAlloc)))
3645 : {
3646 : CPLError(CE_Failure, CPLE_AppDefined,
3647 : "Input file too large to be opened");
3648 : VSIFree(*ppabyRet);
3649 : *ppabyRet = nullptr;
3650 : if (bFreeFP)
3651 : CPL_IGNORE_RET_VAL(VSIFCloseL(fp));
3652 : return FALSE;
3653 : }
3654 : GByte *pabyNew = static_cast<GByte *>(
3655 233 : VSIRealloc(*ppabyRet, static_cast<size_t>(nDataAlloc)));
3656 233 : if (pabyNew == nullptr)
3657 : {
3658 0 : CPLError(CE_Failure, CPLE_OutOfMemory,
3659 : "Cannot allocate " CPL_FRMT_GIB " bytes",
3660 : nDataAlloc);
3661 0 : VSIFree(*ppabyRet);
3662 0 : *ppabyRet = nullptr;
3663 0 : if (bFreeFP)
3664 0 : CPL_IGNORE_RET_VAL(VSIFCloseL(fp));
3665 0 : return FALSE;
3666 : }
3667 233 : *ppabyRet = pabyNew;
3668 : }
3669 : const int nRead =
3670 507 : static_cast<int>(VSIFReadL(*ppabyRet + nDataLen, 1, 8192, fp));
3671 507 : nDataLen += nRead;
3672 :
3673 507 : if (nMaxSize >= 0 && nDataLen > static_cast<vsi_l_offset>(nMaxSize))
3674 : {
3675 0 : CPLError(CE_Failure, CPLE_AppDefined,
3676 : "Input file too large to be opened");
3677 0 : VSIFree(*ppabyRet);
3678 0 : *ppabyRet = nullptr;
3679 0 : if (pnSize != nullptr)
3680 0 : *pnSize = 0;
3681 0 : if (bFreeFP)
3682 0 : CPL_IGNORE_RET_VAL(VSIFCloseL(fp));
3683 0 : return FALSE;
3684 : }
3685 :
3686 507 : if (pnSize != nullptr)
3687 245 : *pnSize += nRead;
3688 507 : (*ppabyRet)[nDataLen] = '\0';
3689 507 : if (nRead == 0)
3690 100 : break;
3691 507 : }
3692 : }
3693 : else
3694 : {
3695 12173 : if (VSIFSeekL(fp, 0, SEEK_END) != 0)
3696 : {
3697 0 : if (bFreeFP)
3698 0 : CPL_IGNORE_RET_VAL(VSIFCloseL(fp));
3699 0 : return FALSE;
3700 : }
3701 12173 : nDataLen = VSIFTellL(fp);
3702 :
3703 : // With "large" VSI I/O API we can read data chunks larger than
3704 : // VSIMalloc could allocate. Catch it here.
3705 12173 : if (nDataLen !=
3706 : static_cast<vsi_l_offset>(static_cast<size_t>(nDataLen)) ||
3707 : nDataLen + 1 < nDataLen
3708 : // opening a directory returns nDataLen = INT_MAX (on 32bit) or
3709 : // INT64_MAX (on 64bit)
3710 19081 : || nDataLen + 1 > std::numeric_limits<size_t>::max() / 2 ||
3711 6908 : (nMaxSize >= 0 && nDataLen > static_cast<vsi_l_offset>(nMaxSize)))
3712 : {
3713 0 : CPLError(CE_Failure, CPLE_AppDefined,
3714 : "Input file too large to be opened");
3715 0 : if (bFreeFP)
3716 0 : CPL_IGNORE_RET_VAL(VSIFCloseL(fp));
3717 0 : return FALSE;
3718 : }
3719 :
3720 12173 : if (VSIFSeekL(fp, 0, SEEK_SET) != 0)
3721 : {
3722 0 : if (bFreeFP)
3723 0 : CPL_IGNORE_RET_VAL(VSIFCloseL(fp));
3724 0 : return FALSE;
3725 : }
3726 :
3727 12173 : *ppabyRet =
3728 12173 : static_cast<GByte *>(VSIMalloc(static_cast<size_t>(nDataLen + 1)));
3729 12173 : if (nullptr == *ppabyRet)
3730 : {
3731 0 : CPLError(CE_Failure, CPLE_OutOfMemory,
3732 : "Cannot allocate " CPL_FRMT_GIB " bytes", nDataLen + 1);
3733 0 : if (bFreeFP)
3734 0 : CPL_IGNORE_RET_VAL(VSIFCloseL(fp));
3735 0 : return FALSE;
3736 : }
3737 :
3738 12173 : (*ppabyRet)[nDataLen] = '\0';
3739 12173 : if (nDataLen !=
3740 12173 : VSIFReadL(*ppabyRet, 1, static_cast<size_t>(nDataLen), fp))
3741 : {
3742 0 : CPLError(CE_Failure, CPLE_FileIO,
3743 : "Cannot read " CPL_FRMT_GIB " bytes", nDataLen);
3744 0 : VSIFree(*ppabyRet);
3745 0 : *ppabyRet = nullptr;
3746 0 : if (bFreeFP)
3747 0 : CPL_IGNORE_RET_VAL(VSIFCloseL(fp));
3748 0 : return FALSE;
3749 : }
3750 12173 : if (pnSize != nullptr)
3751 5402 : *pnSize = nDataLen;
3752 : }
3753 12273 : if (bFreeFP)
3754 11031 : CPL_IGNORE_RET_VAL(VSIFCloseL(fp));
3755 12273 : return TRUE;
3756 : }
3757 :
3758 : /************************************************************************/
3759 : /* VSIOverwriteFile() */
3760 : /************************************************************************/
3761 :
3762 : /**
3763 : * \brief Overwrite an existing file with content from another one
3764 : *
3765 : * @param fpTarget file handle opened with VSIFOpenL() with "rb+" flag.
3766 : * @param pszSourceFilename source filename
3767 : *
3768 : * @return TRUE in case of success.
3769 : *
3770 : * @since GDAL 3.1
3771 : */
3772 :
3773 4 : int VSIOverwriteFile(VSILFILE *fpTarget, const char *pszSourceFilename)
3774 : {
3775 4 : VSILFILE *fpSource = VSIFOpenL(pszSourceFilename, "rb");
3776 4 : if (fpSource == nullptr)
3777 : {
3778 0 : CPLError(CE_Failure, CPLE_FileIO, "Cannot open %s", pszSourceFilename);
3779 0 : return false;
3780 : }
3781 :
3782 4 : const size_t nBufferSize = 4096;
3783 4 : void *pBuffer = CPLMalloc(nBufferSize);
3784 4 : VSIFSeekL(fpTarget, 0, SEEK_SET);
3785 4 : bool bRet = true;
3786 : while (true)
3787 : {
3788 4 : size_t nRead = VSIFReadL(pBuffer, 1, nBufferSize, fpSource);
3789 4 : size_t nWritten = VSIFWriteL(pBuffer, 1, nRead, fpTarget);
3790 4 : if (nWritten != nRead)
3791 : {
3792 0 : bRet = false;
3793 0 : break;
3794 : }
3795 4 : if (nRead < nBufferSize)
3796 4 : break;
3797 0 : }
3798 :
3799 4 : if (bRet)
3800 : {
3801 4 : bRet = VSIFTruncateL(fpTarget, VSIFTellL(fpTarget)) == 0;
3802 4 : if (!bRet)
3803 : {
3804 0 : CPLError(CE_Failure, CPLE_FileIO, "Truncation failed");
3805 : }
3806 : }
3807 :
3808 4 : CPLFree(pBuffer);
3809 4 : VSIFCloseL(fpSource);
3810 4 : return bRet;
3811 : }
3812 :
3813 : /************************************************************************/
3814 : /* VSIFGetNativeFileDescriptorL() */
3815 : /************************************************************************/
3816 :
3817 : /**
3818 : * \fn VSIVirtualHandle::GetNativeFileDescriptor()
3819 : * \brief Returns the "native" file descriptor for the virtual handle.
3820 : *
3821 : * This will only return a non-NULL value for "real" files handled by the
3822 : * operating system (to be opposed to GDAL virtual file systems).
3823 : *
3824 : * On POSIX systems, this will be a integer value ("fd") cast as a void*.
3825 : * On Windows systems, this will be the HANDLE.
3826 : *
3827 : * @return the native file descriptor, or NULL.
3828 : */
3829 :
3830 : /**
3831 : * \brief Returns the "native" file descriptor for the virtual handle.
3832 : *
3833 : * This will only return a non-NULL value for "real" files handled by the
3834 : * operating system (to be opposed to GDAL virtual file systems).
3835 : *
3836 : * On POSIX systems, this will be a integer value ("fd") cast as a void*.
3837 : * On Windows systems, this will be the HANDLE.
3838 : *
3839 : * @param fp file handle opened with VSIFOpenL().
3840 : *
3841 : * @return the native file descriptor, or NULL.
3842 : */
3843 :
3844 64 : void *VSIFGetNativeFileDescriptorL(VSILFILE *fp)
3845 : {
3846 64 : return fp->GetNativeFileDescriptor();
3847 : }
3848 :
3849 : /************************************************************************/
3850 : /* VSIGetDiskFreeSpace() */
3851 : /************************************************************************/
3852 :
3853 : /**
3854 : * \brief Return free disk space available on the filesystem.
3855 : *
3856 : * This function returns the free disk space available on the filesystem.
3857 : *
3858 : * @param pszDirname a directory of the filesystem to query.
3859 : * @return The free space in bytes. Or -1 in case of error.
3860 : * @since GDAL 2.1
3861 : */
3862 :
3863 74 : GIntBig VSIGetDiskFreeSpace(const char *pszDirname)
3864 : {
3865 74 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszDirname);
3866 :
3867 74 : return poFSHandler->GetDiskFreeSpace(pszDirname);
3868 : }
3869 :
3870 : /************************************************************************/
3871 : /* VSIGetFileSystemsPrefixes() */
3872 : /************************************************************************/
3873 :
3874 : /**
3875 : * \brief Return the list of prefixes for virtual file system handlers
3876 : * currently registered.
3877 : *
3878 : * Typically: "", "/vsimem/", "/vsicurl/", etc
3879 : *
3880 : * @return a NULL terminated list of prefixes. Must be freed with CSLDestroy()
3881 : * @since GDAL 2.3
3882 : */
3883 :
3884 9 : char **VSIGetFileSystemsPrefixes(void)
3885 : {
3886 9 : return VSIFileManager::GetPrefixes();
3887 : }
3888 :
3889 : /************************************************************************/
3890 : /* VSIGetFileSystemOptions() */
3891 : /************************************************************************/
3892 :
3893 : /**
3894 : * \brief Return the list of options associated with a virtual file system
3895 : * handler as a serialized XML string.
3896 : *
3897 : * Those options may be set as configuration options with CPLSetConfigOption().
3898 : *
3899 : * @param pszFilename a filename, or prefix of a virtual file system handler.
3900 : * @return a string, which must not be freed, or NULL if no options is declared.
3901 : * @since GDAL 2.3
3902 : */
3903 :
3904 36 : const char *VSIGetFileSystemOptions(const char *pszFilename)
3905 : {
3906 36 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
3907 :
3908 36 : return poFSHandler->GetOptions();
3909 : }
3910 :
3911 : /************************************************************************/
3912 : /* VSISetPathSpecificOption() */
3913 : /************************************************************************/
3914 :
3915 : static std::mutex oMutexPathSpecificOptions;
3916 :
3917 : // key is a path prefix
3918 : // value is a map of key, value pair
3919 : static std::map<std::string, std::map<std::string, std::string>>
3920 : oMapPathSpecificOptions;
3921 :
3922 : /**
3923 : * \brief Set a credential (or more generally an option related to a
3924 : * virtual file system) for a given path prefix.
3925 : * @deprecated in GDAL 3.6 for the better named VSISetPathSpecificOption()
3926 : * @see VSISetPathSpecificOption()
3927 : */
3928 0 : void VSISetCredential(const char *pszPathPrefix, const char *pszKey,
3929 : const char *pszValue)
3930 : {
3931 0 : VSISetPathSpecificOption(pszPathPrefix, pszKey, pszValue);
3932 0 : }
3933 :
3934 : /**
3935 : * \brief Set a path specific option for a given path prefix.
3936 : *
3937 : * Such option is typically, but not limited to, a credential setting for a
3938 : * virtual file system.
3939 : *
3940 : * That option may also be set as a configuration option with
3941 : * CPLSetConfigOption(), but this function allows to specify them with a
3942 : * granularity at the level of a file path, which makes it easier if using the
3943 : * same virtual file system but with different credentials (e.g. different
3944 : * credentials for bucket "/vsis3/foo" and "/vsis3/bar")
3945 : *
3946 : * This is supported for the following virtual file systems:
3947 : * /vsis3/, /vsigs/, /vsiaz/, /vsioss/, /vsiwebhdfs, /vsiswift.
3948 : * Note: setting them for a path starting with /vsiXXX/ will also apply for
3949 : * /vsiXXX_streaming/ requests.
3950 : *
3951 : * Note that no particular care is taken to store them in RAM in a secure way.
3952 : * So they might accidentally hit persistent storage if swapping occurs, or
3953 : * someone with access to the memory allocated by the process may be able to
3954 : * read them.
3955 : *
3956 : * @param pszPathPrefix a path prefix of a virtual file system handler.
3957 : * Typically of the form "/vsiXXX/bucket". Must NOT be
3958 : * NULL. Should not include trailing slashes.
3959 : * @param pszKey Option name. Must NOT be NULL.
3960 : * @param pszValue Option value. May be NULL to erase it.
3961 : *
3962 : * @since GDAL 3.6
3963 : */
3964 :
3965 90 : void VSISetPathSpecificOption(const char *pszPathPrefix, const char *pszKey,
3966 : const char *pszValue)
3967 : {
3968 180 : std::lock_guard<std::mutex> oLock(oMutexPathSpecificOptions);
3969 90 : auto oIter = oMapPathSpecificOptions.find(pszPathPrefix);
3970 180 : CPLString osKey(pszKey);
3971 90 : osKey.toupper();
3972 90 : if (oIter == oMapPathSpecificOptions.end())
3973 : {
3974 24 : if (pszValue != nullptr)
3975 24 : oMapPathSpecificOptions[pszPathPrefix][osKey] = pszValue;
3976 : }
3977 66 : else if (pszValue != nullptr)
3978 58 : oIter->second[osKey] = pszValue;
3979 : else
3980 8 : oIter->second.erase(osKey);
3981 90 : }
3982 :
3983 : /************************************************************************/
3984 : /* VSIClearPathSpecificOptions() */
3985 : /************************************************************************/
3986 :
3987 : /**
3988 : * \brief Clear path specific options set with VSISetPathSpecificOption()
3989 : * @deprecated in GDAL 3.6 for the better named VSIClearPathSpecificOptions()
3990 : * @see VSIClearPathSpecificOptions()
3991 : */
3992 0 : void VSIClearCredentials(const char *pszPathPrefix)
3993 : {
3994 0 : return VSIClearPathSpecificOptions(pszPathPrefix);
3995 : }
3996 :
3997 : /**
3998 : * \brief Clear path specific options set with VSISetPathSpecificOption()
3999 : *
4000 : * Note that no particular care is taken to remove them from RAM in a secure
4001 : * way.
4002 : *
4003 : * @param pszPathPrefix If set to NULL, all path specific options are cleared.
4004 : * If set to not-NULL, only those set with
4005 : * VSISetPathSpecificOption(pszPathPrefix, ...) will be
4006 : * cleared.
4007 : *
4008 : * @since GDAL 3.6
4009 : */
4010 19 : void VSIClearPathSpecificOptions(const char *pszPathPrefix)
4011 : {
4012 38 : std::lock_guard<std::mutex> oLock(oMutexPathSpecificOptions);
4013 19 : if (pszPathPrefix == nullptr)
4014 : {
4015 3 : oMapPathSpecificOptions.clear();
4016 : }
4017 : else
4018 : {
4019 16 : oMapPathSpecificOptions.erase(pszPathPrefix);
4020 : }
4021 19 : }
4022 :
4023 : /************************************************************************/
4024 : /* VSIGetPathSpecificOption() */
4025 : /************************************************************************/
4026 :
4027 : /**
4028 : * \brief Get the value of a credential (or more generally an option related to
4029 : * a virtual file system) for a given path.
4030 : * @deprecated in GDAL 3.6 for the better named VSIGetPathSpecificOption()
4031 : * @see VSIGetPathSpecificOption()
4032 : */
4033 0 : const char *VSIGetCredential(const char *pszPath, const char *pszKey,
4034 : const char *pszDefault)
4035 : {
4036 0 : return VSIGetPathSpecificOption(pszPath, pszKey, pszDefault);
4037 : }
4038 :
4039 : /**
4040 : * \brief Get the value a path specific option.
4041 : *
4042 : * Such option is typically, but not limited to, a credential setting for a
4043 : * virtual file system.
4044 : *
4045 : * If no match occurs, CPLGetConfigOption(pszKey, pszDefault) is returned.
4046 : *
4047 : * Mostly to be used by virtual file system implementations.
4048 : *
4049 : * @since GDAL 3.6
4050 : * @see VSISetPathSpecificOption()
4051 : */
4052 148043 : const char *VSIGetPathSpecificOption(const char *pszPath, const char *pszKey,
4053 : const char *pszDefault)
4054 : {
4055 : {
4056 148043 : std::lock_guard<std::mutex> oLock(oMutexPathSpecificOptions);
4057 186017 : for (auto it = oMapPathSpecificOptions.rbegin();
4058 223989 : it != oMapPathSpecificOptions.rend(); ++it)
4059 : {
4060 38241 : if (STARTS_WITH(pszPath, it->first.c_str()))
4061 : {
4062 3748 : auto oIter = it->second.find(CPLString(pszKey).toupper());
4063 3748 : if (oIter != it->second.end())
4064 269 : return oIter->second.c_str();
4065 : }
4066 : }
4067 : }
4068 147776 : return CPLGetConfigOption(pszKey, pszDefault);
4069 : }
4070 :
4071 : /************************************************************************/
4072 : /* VSIDuplicateFileSystemHandler() */
4073 : /************************************************************************/
4074 :
4075 : /**
4076 : * \brief Duplicate an existing file system handler.
4077 : *
4078 : * A number of virtual file system for remote object stores use protocols
4079 : * identical or close to popular ones (typically AWS S3), but with slightly
4080 : * different settings (at the very least the endpoint).
4081 : *
4082 : * This functions allows to duplicate the source virtual file system handler
4083 : * as a new one with a different prefix (when the source virtual file system
4084 : * handler supports the duplication operation).
4085 : *
4086 : * VSISetPathSpecificOption() will typically be called afterwards to change
4087 : * configurable settings on the cloned file system handler (e.g. AWS_S3_ENDPOINT
4088 : * for a clone of /vsis3/).
4089 : *
4090 : * @since GDAL 3.7
4091 : */
4092 4 : bool VSIDuplicateFileSystemHandler(const char *pszSourceFSName,
4093 : const char *pszNewFSName)
4094 : {
4095 : VSIFilesystemHandler *poTargetFSHandler =
4096 4 : VSIFileManager::GetHandler(pszNewFSName);
4097 4 : if (poTargetFSHandler != VSIFileManager::GetHandler("/"))
4098 : {
4099 1 : CPLError(CE_Failure, CPLE_AppDefined,
4100 : "%s is already a known virtual file system", pszNewFSName);
4101 1 : return false;
4102 : }
4103 :
4104 : VSIFilesystemHandler *poSourceFSHandler =
4105 3 : VSIFileManager::GetHandler(pszSourceFSName);
4106 3 : if (!poSourceFSHandler)
4107 : {
4108 0 : CPLError(CE_Failure, CPLE_AppDefined,
4109 : "%s is not a known virtual file system", pszSourceFSName);
4110 0 : return false;
4111 : }
4112 :
4113 3 : poTargetFSHandler = poSourceFSHandler->Duplicate(pszNewFSName);
4114 3 : if (!poTargetFSHandler)
4115 2 : return false;
4116 :
4117 1 : VSIFileManager::InstallHandler(pszNewFSName, poTargetFSHandler);
4118 1 : return true;
4119 : }
4120 :
4121 : /************************************************************************/
4122 : /* ==================================================================== */
4123 : /* VSIFileManager() */
4124 : /* ==================================================================== */
4125 : /************************************************************************/
4126 :
4127 : #ifndef DOXYGEN_SKIP
4128 :
4129 : /*
4130 : ** Notes on Multithreading:
4131 : **
4132 : ** The VSIFileManager maintains a list of file type handlers (mem, large
4133 : ** file, etc). It should be thread safe.
4134 : **/
4135 :
4136 : /************************************************************************/
4137 : /* VSIFileManager() */
4138 : /************************************************************************/
4139 :
4140 1757 : VSIFileManager::VSIFileManager() : poDefaultHandler(nullptr)
4141 : {
4142 1757 : }
4143 :
4144 : /************************************************************************/
4145 : /* ~VSIFileManager() */
4146 : /************************************************************************/
4147 :
4148 1121 : VSIFileManager::~VSIFileManager()
4149 : {
4150 2242 : std::set<VSIFilesystemHandler *> oSetAlreadyDeleted;
4151 34753 : for (std::map<std::string, VSIFilesystemHandler *>::const_iterator iter =
4152 1121 : oHandlers.begin();
4153 70627 : iter != oHandlers.end(); ++iter)
4154 : {
4155 34753 : if (oSetAlreadyDeleted.find(iter->second) == oSetAlreadyDeleted.end())
4156 : {
4157 31390 : oSetAlreadyDeleted.insert(iter->second);
4158 31390 : delete iter->second;
4159 : }
4160 : }
4161 :
4162 1121 : delete poDefaultHandler;
4163 1121 : }
4164 :
4165 : /************************************************************************/
4166 : /* Get() */
4167 : /************************************************************************/
4168 :
4169 : static VSIFileManager *poManager = nullptr;
4170 : static CPLMutex *hVSIFileManagerMutex = nullptr;
4171 :
4172 2766200 : VSIFileManager *VSIFileManager::Get()
4173 : {
4174 5532530 : CPLMutexHolder oHolder(&hVSIFileManagerMutex);
4175 2766330 : if (poManager != nullptr)
4176 : {
4177 2764570 : return poManager;
4178 : }
4179 :
4180 1757 : poManager = new VSIFileManager;
4181 1757 : VSIInstallLargeFileHandler();
4182 1757 : VSIInstallSubFileHandler();
4183 1757 : VSIInstallMemFileHandler();
4184 : #ifdef HAVE_LIBZ
4185 1757 : VSIInstallGZipFileHandler();
4186 1757 : VSIInstallZipFileHandler();
4187 : #endif
4188 : #ifdef HAVE_LIBARCHIVE
4189 : VSIInstall7zFileHandler();
4190 : VSIInstallRarFileHandler();
4191 : #endif
4192 : #ifdef HAVE_CURL
4193 1757 : VSIInstallCurlFileHandler();
4194 1757 : VSIInstallCurlStreamingFileHandler();
4195 1757 : VSIInstallS3FileHandler();
4196 1757 : VSIInstallS3StreamingFileHandler();
4197 1757 : VSIInstallGSFileHandler();
4198 1757 : VSIInstallGSStreamingFileHandler();
4199 1757 : VSIInstallAzureFileHandler();
4200 1757 : VSIInstallAzureStreamingFileHandler();
4201 1757 : VSIInstallADLSFileHandler();
4202 1757 : VSIInstallOSSFileHandler();
4203 1757 : VSIInstallOSSStreamingFileHandler();
4204 1757 : VSIInstallSwiftFileHandler();
4205 1757 : VSIInstallSwiftStreamingFileHandler();
4206 1757 : VSIInstallWebHdfsHandler();
4207 : #endif
4208 1757 : VSIInstallStdinHandler();
4209 1757 : VSIInstallHdfsHandler();
4210 1757 : VSIInstallStdoutHandler();
4211 1757 : VSIInstallSparseFileHandler();
4212 1757 : VSIInstallTarFileHandler();
4213 1757 : VSIInstallCachedFileHandler();
4214 1757 : VSIInstallCryptFileHandler();
4215 :
4216 1757 : return poManager;
4217 : }
4218 :
4219 : /************************************************************************/
4220 : /* GetPrefixes() */
4221 : /************************************************************************/
4222 :
4223 683 : char **VSIFileManager::GetPrefixes()
4224 : {
4225 1366 : CPLMutexHolder oHolder(&hVSIFileManagerMutex);
4226 1366 : CPLStringList aosList;
4227 21856 : for (const auto &oIter : Get()->oHandlers)
4228 : {
4229 21173 : if (oIter.first != "/vsicurl?")
4230 : {
4231 20490 : aosList.AddString(oIter.first.c_str());
4232 : }
4233 : }
4234 1366 : return aosList.StealList();
4235 : }
4236 :
4237 : /************************************************************************/
4238 : /* GetHandler() */
4239 : /************************************************************************/
4240 :
4241 2709340 : VSIFilesystemHandler *VSIFileManager::GetHandler(const char *pszPath)
4242 :
4243 : {
4244 2709340 : VSIFileManager *poThis = Get();
4245 2709470 : const size_t nPathLen = strlen(pszPath);
4246 :
4247 60051700 : for (std::map<std::string, VSIFilesystemHandler *>::const_iterator iter =
4248 2709470 : poThis->oHandlers.begin();
4249 122809000 : iter != poThis->oHandlers.end(); ++iter)
4250 : {
4251 61898500 : const char *pszIterKey = iter->first.c_str();
4252 61900700 : const size_t nIterKeyLen = iter->first.size();
4253 61907300 : if (strncmp(pszPath, pszIterKey, nIterKeyLen) == 0)
4254 1859020 : return iter->second;
4255 :
4256 : // "/vsimem\foo" should be handled as "/vsimem/foo".
4257 60158300 : if (nIterKeyLen && nPathLen > nIterKeyLen &&
4258 53058200 : pszIterKey[nIterKeyLen - 1] == '/' &&
4259 46859900 : pszPath[nIterKeyLen - 1] == '\\' &&
4260 30 : strncmp(pszPath, pszIterKey, nIterKeyLen - 1) == 0)
4261 0 : return iter->second;
4262 :
4263 : // /vsimem should be treated as a match for /vsimem/.
4264 60158300 : if (nPathLen + 1 == nIterKeyLen &&
4265 571275 : strncmp(pszPath, pszIterKey, nPathLen) == 0)
4266 110039 : return iter->second;
4267 : }
4268 :
4269 850382 : return poThis->poDefaultHandler;
4270 : }
4271 :
4272 : /************************************************************************/
4273 : /* InstallHandler() */
4274 : /************************************************************************/
4275 :
4276 56175 : void VSIFileManager::InstallHandler(const std::string &osPrefix,
4277 : VSIFilesystemHandler *poHandler)
4278 :
4279 : {
4280 56175 : if (osPrefix == "")
4281 1757 : Get()->poDefaultHandler = poHandler;
4282 : else
4283 54418 : Get()->oHandlers[osPrefix] = poHandler;
4284 56175 : }
4285 :
4286 : /************************************************************************/
4287 : /* RemoveHandler() */
4288 : /************************************************************************/
4289 :
4290 3 : void VSIFileManager::RemoveHandler(const std::string &osPrefix)
4291 : {
4292 3 : if (osPrefix == "")
4293 0 : Get()->poDefaultHandler = nullptr;
4294 : else
4295 3 : Get()->oHandlers.erase(osPrefix);
4296 3 : }
4297 :
4298 : /************************************************************************/
4299 : /* VSICleanupFileManager() */
4300 : /************************************************************************/
4301 :
4302 1121 : void VSICleanupFileManager()
4303 :
4304 : {
4305 1121 : if (poManager)
4306 : {
4307 1121 : delete poManager;
4308 1121 : poManager = nullptr;
4309 : }
4310 :
4311 1121 : if (hVSIFileManagerMutex != nullptr)
4312 : {
4313 1121 : CPLDestroyMutex(hVSIFileManagerMutex);
4314 1121 : hVSIFileManagerMutex = nullptr;
4315 : }
4316 :
4317 : #ifdef HAVE_CURL
4318 1121 : VSICURLDestroyCacheFileProp();
4319 : #endif
4320 1121 : }
4321 :
4322 : /************************************************************************/
4323 : /* Truncate() */
4324 : /************************************************************************/
4325 :
4326 2 : int VSIVirtualHandle::Truncate(vsi_l_offset nNewSize)
4327 : {
4328 2 : const vsi_l_offset nOriginalPos = Tell();
4329 2 : if (Seek(0, SEEK_END) == 0 && nNewSize >= Tell())
4330 : {
4331 : // Fill with zeroes
4332 2 : std::vector<GByte> aoBytes(4096, 0);
4333 1 : vsi_l_offset nCurOffset = nOriginalPos;
4334 3 : while (nCurOffset < nNewSize)
4335 : {
4336 2 : constexpr vsi_l_offset nMaxOffset = 4096;
4337 : const int nSize =
4338 2 : static_cast<int>(std::min(nMaxOffset, nNewSize - nCurOffset));
4339 2 : if (Write(&aoBytes[0], nSize, 1) != 1)
4340 : {
4341 0 : Seek(nOriginalPos, SEEK_SET);
4342 0 : return -1;
4343 : }
4344 2 : nCurOffset += nSize;
4345 : }
4346 1 : return Seek(nOriginalPos, SEEK_SET) == 0 ? 0 : -1;
4347 : }
4348 :
4349 1 : CPLDebug("VSI", "Truncation is not supported in generic implementation "
4350 : "of Truncate()");
4351 1 : Seek(nOriginalPos, SEEK_SET);
4352 1 : return -1;
4353 : }
4354 :
4355 : /************************************************************************/
4356 : /* ReadMultiRange() */
4357 : /************************************************************************/
4358 :
4359 762 : int VSIVirtualHandle::ReadMultiRange(int nRanges, void **ppData,
4360 : const vsi_l_offset *panOffsets,
4361 : const size_t *panSizes)
4362 : {
4363 762 : int nRet = 0;
4364 762 : const vsi_l_offset nCurOffset = Tell();
4365 58777 : for (int i = 0; i < nRanges; i++)
4366 : {
4367 58034 : if (Seek(panOffsets[i], SEEK_SET) < 0)
4368 : {
4369 0 : nRet = -1;
4370 0 : break;
4371 : }
4372 :
4373 58034 : const size_t nRead = Read(ppData[i], 1, panSizes[i]);
4374 58034 : if (panSizes[i] != nRead)
4375 : {
4376 19 : nRet = -1;
4377 19 : break;
4378 : }
4379 : }
4380 :
4381 762 : Seek(nCurOffset, SEEK_SET);
4382 :
4383 762 : return nRet;
4384 : }
4385 :
4386 : #endif // #ifndef DOXYGEN_SKIP
4387 :
4388 : /************************************************************************/
4389 : /* HasPRead() */
4390 : /************************************************************************/
4391 :
4392 : /** Returns whether this file handle supports the PRead() method.
4393 : *
4394 : * @since GDAL 3.6
4395 : */
4396 0 : bool VSIVirtualHandle::HasPRead() const
4397 : {
4398 0 : return false;
4399 : }
4400 :
4401 : /************************************************************************/
4402 : /* PRead() */
4403 : /************************************************************************/
4404 :
4405 : /** Do a parallel-compatible read operation.
4406 : *
4407 : * This methods reads into pBuffer up to nSize bytes starting at offset nOffset
4408 : * in the file. The current file offset is not affected by this method.
4409 : *
4410 : * The implementation is thread-safe: several threads can issue PRead()
4411 : * concurrently on the same VSIVirtualHandle object.
4412 : *
4413 : * This method has the same semantics as pread() Linux operation. It is only
4414 : * available if HasPRead() returns true.
4415 : *
4416 : * @param pBuffer output buffer (must be at least nSize bytes large).
4417 : * @param nSize number of bytes to read in the file.
4418 : * @param nOffset file offset from which to read.
4419 : * @return number of bytes read.
4420 : * @since GDAL 3.6
4421 : */
4422 0 : size_t VSIVirtualHandle::PRead(CPL_UNUSED void *pBuffer,
4423 : CPL_UNUSED size_t nSize,
4424 : CPL_UNUSED vsi_l_offset nOffset) const
4425 : {
4426 0 : return 0;
4427 : }
4428 :
4429 : #ifndef DOXYGEN_SKIP
4430 : /************************************************************************/
4431 : /* VSIProxyFileHandle::CancelCreation() */
4432 : /************************************************************************/
4433 :
4434 8 : void VSIProxyFileHandle::CancelCreation()
4435 : {
4436 8 : m_nativeHandle->CancelCreation();
4437 8 : }
4438 : #endif
4439 :
4440 : /************************************************************************/
4441 : /* VSIURIToVSIPath() */
4442 : /************************************************************************/
4443 :
4444 : /** Return a VSI compatible path from a URI / URL
4445 : *
4446 : * Substitute URI / URLs starting with s3://, gs://, etc. by their VSI
4447 : * prefix equivalent. If no known substitution is found, the input string is
4448 : * returned unmodified.
4449 : *
4450 : * @since GDAL 3.12
4451 : */
4452 675 : std::string VSIURIToVSIPath(const std::string &osURI)
4453 : {
4454 : static const struct
4455 : {
4456 : const char *pszFSSpecPrefix;
4457 : const char *pszVSIPrefix;
4458 : } substitutions[] = {
4459 : {"s3://", "/vsis3/"},
4460 : {"gs://", "/vsigs/"},
4461 : {"gcs://", "/vsigs/"},
4462 : {"az://", "/vsiaz/"},
4463 : {"azure://", "/vsiaz/"},
4464 : {"http://", "/vsicurl/http://"},
4465 : {"https://", "/vsicurl/https://"},
4466 : {"file://", ""},
4467 : };
4468 :
4469 6064 : for (const auto &substitution : substitutions)
4470 : {
4471 5397 : if (STARTS_WITH(osURI.c_str(), substitution.pszFSSpecPrefix))
4472 : {
4473 19 : return std::string(substitution.pszVSIPrefix)
4474 9 : .append(osURI.c_str() + strlen(substitution.pszFSSpecPrefix));
4475 : }
4476 : }
4477 :
4478 667 : return osURI;
4479 : }
|
