Line data Source code
1 : /******************************************************************************
2 : *
3 : * Project: VSI Virtual File System
4 : * Purpose: Implementation VSI*L File API and other file system access
5 : * methods going through file virtualization.
6 : * Author: Frank Warmerdam, warmerdam@pobox.com
7 : *
8 : ******************************************************************************
9 : * Copyright (c) 2005, Frank Warmerdam <warmerdam@pobox.com>
10 : * Copyright (c) 2008-2014, Even Rouault <even dot rouault at spatialys.com>
11 : *
12 : * SPDX-License-Identifier: MIT
13 : ****************************************************************************/
14 :
15 : #include "cpl_port.h"
16 : #include "cpl_vsi.h"
17 :
18 : #include <cassert>
19 : #include <cinttypes>
20 : #include <cstdarg>
21 : #include <cstddef>
22 : #include <cstring>
23 :
24 : #include <fcntl.h>
25 :
26 : #include <algorithm>
27 : #include <limits>
28 : #include <map>
29 : #include <memory>
30 : #include <mutex>
31 : #include <set>
32 : #include <string>
33 : #include <utility>
34 : #include <vector>
35 :
36 : #include "cpl_conv.h"
37 : #include "cpl_error.h"
38 : #include "cpl_multiproc.h"
39 : #include "cpl_string.h"
40 : #include "cpl_vsi_virtual.h"
41 : #include "cpl_vsil_curl_class.h"
42 :
43 : // To avoid aliasing to GetDiskFreeSpace to GetDiskFreeSpaceA on Windows
44 : #ifdef GetDiskFreeSpace
45 : #undef GetDiskFreeSpace
46 : #endif
47 :
48 : /************************************************************************/
49 : /* VSIReadDir() */
50 : /************************************************************************/
51 :
52 : /**
53 : * \brief Read names in a directory.
54 : *
55 : * This function abstracts access to directory contains. It returns a
56 : * list of strings containing the names of files, and directories in this
57 : * directory. The resulting string list becomes the responsibility of the
58 : * application and should be freed with CSLDestroy() when no longer needed.
59 : *
60 : * Note that no error is issued via CPLError() if the directory path is
61 : * invalid, though NULL is returned.
62 : *
63 : * This function used to be known as CPLReadDir(), but the old name is now
64 : * deprecated.
65 : *
66 : * @param pszPath the relative, or absolute path of a directory to read.
67 : * UTF-8 encoded.
68 : * @return The list of entries in the directory, or NULL if the directory
69 : * doesn't exist. Filenames are returned in UTF-8 encoding.
70 : */
71 :
72 15665 : char **VSIReadDir(const char *pszPath)
73 : {
74 15665 : return VSIReadDirEx(pszPath, 0);
75 : }
76 :
77 : /************************************************************************/
78 : /* VSIReadDirEx() */
79 : /************************************************************************/
80 :
81 : /**
82 : * \brief Read names in a directory.
83 : *
84 : * This function abstracts access to directory contains. It returns a
85 : * list of strings containing the names of files, and directories in this
86 : * directory. The resulting string list becomes the responsibility of the
87 : * application and should be freed with CSLDestroy() when no longer needed.
88 : *
89 : * Note that no error is issued via CPLError() if the directory path is
90 : * invalid, though NULL is returned.
91 : *
92 : * If nMaxFiles is set to a positive number, directory listing will stop after
93 : * that limit has been reached. Note that to indicate truncate, at least one
94 : * element more than the nMaxFiles limit will be returned. If CSLCount() on the
95 : * result is lesser or equal to nMaxFiles, then no truncation occurred.
96 : *
97 : * @param pszPath the relative, or absolute path of a directory to read.
98 : * UTF-8 encoded.
99 : * @param nMaxFiles maximum number of files after which to stop, or 0 for no
100 : * limit.
101 : * @return The list of entries in the directory, or NULL if the directory
102 : * doesn't exist. Filenames are returned in UTF-8 encoding.
103 : */
104 :
105 56981 : char **VSIReadDirEx(const char *pszPath, int nMaxFiles)
106 : {
107 56981 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszPath);
108 :
109 56981 : return poFSHandler->ReadDirEx(pszPath, nMaxFiles);
110 : }
111 :
112 : /************************************************************************/
113 : /* VSISiblingFiles() */
114 : /************************************************************************/
115 :
116 : /**
117 : * \brief Return related filenames
118 : *
119 : * This function is essentially meant at being used by GDAL internals.
120 : *
121 : * @param pszFilename the path of a filename to inspect
122 : * UTF-8 encoded.
123 : * @return The list of entries, relative to the directory, of all sidecar
124 : * files available or NULL if the list is not known.
125 : * Filenames are returned in UTF-8 encoding.
126 : * Most implementations will return NULL, and a subsequent ReadDir will
127 : * list all files available in the file's directory. This function will be
128 : * overridden by VSI FilesystemHandlers that wish to force e.g. an empty list
129 : * to avoid opening non-existent files on slow filesystems. The return value
130 : * shall be destroyed with CSLDestroy()
131 : * @since GDAL 3.2
132 : */
133 75827 : char **VSISiblingFiles(const char *pszFilename)
134 : {
135 75827 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
136 :
137 75831 : return poFSHandler->SiblingFiles(pszFilename);
138 : }
139 :
140 : /************************************************************************/
141 : /* VSIFnMatch() */
142 : /************************************************************************/
143 :
144 705 : static bool VSIFnMatch(const char *pszPattern, const char *pszStr)
145 : {
146 705 : for (; *pszPattern && *pszStr; pszPattern++, pszStr++)
147 : {
148 686 : if (*pszPattern == '*')
149 : {
150 27 : if (pszPattern[1] == 0)
151 4 : return true;
152 78 : for (; *pszStr; ++pszStr)
153 : {
154 71 : if (VSIFnMatch(pszPattern + 1, pszStr))
155 16 : return true;
156 : }
157 7 : return false;
158 : }
159 659 : else if (*pszPattern == '?')
160 : {
161 : // match single any char
162 : }
163 655 : else if (*pszPattern == '[')
164 : {
165 : // match character classes and ranges
166 : // "[abcd]" will match a character that is a, b, c or d
167 : // "[a-z]" will match a character that is a to z
168 : // "[!abcd] will match a character that is *not* a, b, c or d
169 : // "[]]" will match character ]
170 : // "[]-]" will match character ] or -
171 : // "[!]a-]" will match a character that is *not* ], a or -
172 :
173 10 : const char *pszOpenBracket = pszPattern;
174 10 : ++pszPattern;
175 10 : const bool isNot = (*pszPattern == '!');
176 10 : if (isNot)
177 : {
178 3 : ++pszOpenBracket;
179 3 : ++pszPattern;
180 : }
181 10 : bool res = false;
182 22 : for (; *pszPattern; ++pszPattern)
183 : {
184 21 : if ((*pszPattern == ']' || *pszPattern == '-') &&
185 13 : pszPattern == pszOpenBracket + 1)
186 : {
187 3 : if (*pszStr == *pszPattern)
188 : {
189 2 : res = true;
190 : }
191 : }
192 18 : else if (*pszPattern == ']')
193 : {
194 9 : break;
195 : }
196 9 : else if (pszPattern[1] == '-' && pszPattern[2] != 0 &&
197 2 : pszPattern[2] != ']')
198 : {
199 1 : if (*pszStr >= pszPattern[0] && *pszStr <= pszPattern[2])
200 : {
201 1 : res = true;
202 : }
203 1 : pszPattern += 2;
204 : }
205 8 : else if (*pszStr == *pszPattern)
206 : {
207 2 : res = true;
208 : }
209 : }
210 10 : if (*pszPattern == 0)
211 1 : return false;
212 9 : if (!res && !isNot)
213 1 : return false;
214 8 : if (res && isNot)
215 0 : return false;
216 : }
217 645 : else if (*pszPattern != *pszStr)
218 : {
219 499 : return false;
220 : }
221 : }
222 19 : return *pszPattern == 0 && *pszStr == 0;
223 : }
224 :
225 : /************************************************************************/
226 : /* VSIGlob() */
227 : /************************************************************************/
228 :
229 : /**
230 : \brief Return a list of file and directory names matching
231 : a pattern that can contain wildcards.
232 :
233 : This function has similar behavior to the POSIX glob() function:
234 : https://man7.org/linux/man-pages/man7/glob.7.html
235 :
236 : In particular it supports the following wildcards:
237 : <ul>
238 : <li>'*': match any string</li>
239 : <li>'?': match any single character</li>
240 : <li>'[': match character class or range, with '!' immediately after '['
241 : to indicate negation.</li>
242 : </ul>
243 : Refer to to the above man page for more details.
244 :
245 : It also supports the "**" recursive wildcard, behaving similarly to Python
246 : glob.glob() with recursive=True. Be careful of the amount of memory and time
247 : required when using that recursive wildcard on directories with a large
248 : amount of files and subdirectories.
249 :
250 : Examples, given a file hierarchy:
251 : - one.tif
252 : - my_subdir/two.tif
253 : - my_subdir/subsubdir/three.tif
254 :
255 : \code{.cpp}
256 : VSIGlob("one.tif",NULL,NULL,NULL) returns ["one.tif", NULL]
257 : VSIGlob("*.tif",NULL,NULL,NULL) returns ["one.tif", NULL]
258 : VSIGlob("on?.tif",NULL,NULL,NULL) returns ["one.tif", NULL]
259 : VSIGlob("on[a-z].tif",NULL,NULL,NULL) returns ["one.tif", NULL]
260 : VSIGlob("on[ef].tif",NULL,NULL,NULL) returns ["one.tif", NULL]
261 : VSIGlob("on[!e].tif",NULL,NULL,NULL) returns NULL
262 : VSIGlob("my_subdir" "/" "*.tif",NULL,NULL,NULL) returns ["my_subdir/two.tif", NULL]
263 : VSIGlob("**" "/" "*.tif",NULL,NULL,NULL) returns ["one.tif", "my_subdir/two.tif", "my_subdir/subsubdir/three.tif", NULL]
264 : \endcode
265 :
266 : In the current implementation, matching is done based on the assumption that
267 : a character fits into a single byte, which will not work properly on
268 : non-ASCII UTF-8 filenames.
269 :
270 : VSIGlob() works with any virtual file systems supported by GDAL, including
271 : network file systems such as /vsis3/, /vsigs/, /vsiaz/, etc. But note that
272 : for those ones, the pattern is not passed to the remote server, and thus large
273 : amount of filenames can be transferred from the remote server to the host
274 : where the filtering is done.
275 :
276 : @param pszPattern the relative, or absolute path of a directory to read.
277 : UTF-8 encoded.
278 : @param papszOptions NULL-terminate list of options, or NULL. None supported
279 : currently.
280 : @param pProgressFunc Progress function, or NULL. This is only used as a way
281 : for the user to cancel operation if it takes too much time. The percentage
282 : passed to the callback is not significant (always at 0).
283 : @param pProgressData User data passed to the progress function, or NULL.
284 : @return The list of matched filenames, which must be freed with CSLDestroy().
285 : Filenames are returned in UTF-8 encoding.
286 :
287 : @since GDAL 3.11
288 : */
289 :
290 23 : char **VSIGlob(const char *pszPattern, const char *const *papszOptions,
291 : GDALProgressFunc pProgressFunc, void *pProgressData)
292 : {
293 23 : CPL_IGNORE_RET_VAL(papszOptions);
294 :
295 46 : CPLStringList aosRes;
296 46 : std::vector<std::pair<std::string, size_t>> candidates;
297 23 : candidates.emplace_back(pszPattern, 0);
298 72 : while (!candidates.empty())
299 : {
300 49 : auto [osPattern, nPosStart] = candidates.back();
301 49 : pszPattern = osPattern.c_str() + nPosStart;
302 49 : candidates.pop_back();
303 :
304 49 : std::string osPath = osPattern.substr(0, nPosStart);
305 49 : std::string osCurPath;
306 884 : for (;; ++pszPattern)
307 : {
308 933 : if (*pszPattern == 0 || *pszPattern == '/' || *pszPattern == '\\')
309 : {
310 : struct VSIDirCloser
311 : {
312 21 : void operator()(VSIDIR *dir)
313 : {
314 21 : VSICloseDir(dir);
315 21 : }
316 : };
317 :
318 147 : if (osCurPath == "**")
319 : {
320 : std::unique_ptr<VSIDIR, VSIDirCloser> psDir(
321 1 : VSIOpenDir(osPath.c_str(), -1, nullptr));
322 1 : if (!psDir)
323 0 : return nullptr;
324 : while (const VSIDIREntry *psEntry =
325 5 : VSIGetNextDirEntry(psDir.get()))
326 : {
327 4 : if (pProgressFunc &&
328 0 : !pProgressFunc(0, "", pProgressData))
329 : {
330 0 : return nullptr;
331 : }
332 : {
333 8 : std::string osCandidate(osPath);
334 4 : osCandidate += psEntry->pszName;
335 4 : nPosStart = osCandidate.size();
336 4 : if (*pszPattern)
337 : {
338 4 : osCandidate += pszPattern;
339 : }
340 4 : candidates.emplace_back(std::move(osCandidate),
341 4 : nPosStart);
342 : }
343 4 : }
344 1 : osPath.clear();
345 1 : break;
346 : }
347 146 : else if (osCurPath.find_first_of("*?[") != std::string::npos)
348 : {
349 20 : if (osPath.empty())
350 1 : osPath = ".";
351 : std::unique_ptr<VSIDIR, VSIDirCloser> psDir(
352 20 : VSIOpenDir(osPath.c_str(), 0, nullptr));
353 20 : if (!psDir)
354 0 : return nullptr;
355 : while (const VSIDIREntry *psEntry =
356 496 : VSIGetNextDirEntry(psDir.get()))
357 : {
358 487 : if (pProgressFunc &&
359 11 : !pProgressFunc(0, "", pProgressData))
360 : {
361 0 : return nullptr;
362 : }
363 476 : if (VSIFnMatch(osCurPath.c_str(), psEntry->pszName))
364 : {
365 44 : std::string osCandidate;
366 22 : if (osPath != ".")
367 21 : osCandidate = osPath;
368 22 : osCandidate += psEntry->pszName;
369 22 : nPosStart = osCandidate.size();
370 22 : if (*pszPattern)
371 : {
372 2 : osCandidate += pszPattern;
373 : }
374 22 : candidates.emplace_back(std::move(osCandidate),
375 22 : nPosStart);
376 : }
377 476 : }
378 20 : osPath.clear();
379 20 : break;
380 : }
381 126 : else if (*pszPattern == 0)
382 : {
383 28 : osPath += osCurPath;
384 28 : break;
385 : }
386 : else
387 : {
388 98 : osPath += osCurPath;
389 98 : osPath += *pszPattern;
390 98 : osCurPath.clear();
391 98 : }
392 : }
393 : else
394 : {
395 786 : osCurPath += *pszPattern;
396 : }
397 884 : }
398 49 : if (!osPath.empty())
399 : {
400 : VSIStatBufL sStat;
401 28 : if (VSIStatL(osPath.c_str(), &sStat) == 0)
402 23 : aosRes.AddString(osPath.c_str());
403 : }
404 : }
405 :
406 23 : return aosRes.StealList();
407 : }
408 :
409 : /************************************************************************/
410 : /* VSIGetDirectorySeparator() */
411 : /************************************************************************/
412 :
413 : /** Return the directory separator for the specified path.
414 : *
415 : * Default is forward slash. The only exception currently is the Windows
416 : * file system which returns backslash, unless the specified path is of the
417 : * form "{drive_letter}:/{rest_of_the_path}".
418 : *
419 : * @since 3.9
420 : */
421 933089 : const char *VSIGetDirectorySeparator(const char *pszPath)
422 : {
423 933089 : if (STARTS_WITH(pszPath, "http://") || STARTS_WITH(pszPath, "https://"))
424 663 : return "/";
425 :
426 932426 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszPath);
427 932495 : return poFSHandler->GetDirectorySeparator(pszPath);
428 : }
429 :
430 : /************************************************************************/
431 : /* VSIReadRecursive() */
432 : /************************************************************************/
433 :
434 : /**
435 : * \brief Read names in a directory recursively.
436 : *
437 : * This function abstracts access to directory contents and subdirectories.
438 : * It returns a list of strings containing the names of files and directories
439 : * in this directory and all subdirectories. The resulting string list becomes
440 : * the responsibility of the application and should be freed with CSLDestroy()
441 : * when no longer needed.
442 : *
443 : * Note that no error is issued via CPLError() if the directory path is
444 : * invalid, though NULL is returned.
445 : *
446 : * Note: since GDAL 3.9, for recursive mode, the directory separator will no
447 : * longer be always forward slash, but will be the one returned by
448 : * VSIGetDirectorySeparator(pszPathIn), so potentially backslash on Windows
449 : * file systems.
450 : *
451 : * @param pszPathIn the relative, or absolute path of a directory to read.
452 : * UTF-8 encoded.
453 : *
454 : * @return The list of entries in the directory and subdirectories
455 : * or NULL if the directory doesn't exist. Filenames are returned in UTF-8
456 : * encoding.
457 : *
458 : */
459 :
460 1151 : char **VSIReadDirRecursive(const char *pszPathIn)
461 : {
462 1151 : const char SEP = VSIGetDirectorySeparator(pszPathIn)[0];
463 :
464 1151 : const char *const apszOptions[] = {"NAME_AND_TYPE_ONLY=YES", nullptr};
465 1151 : VSIDIR *psDir = VSIOpenDir(pszPathIn, -1, apszOptions);
466 1151 : if (!psDir)
467 3 : return nullptr;
468 2296 : CPLStringList oFiles;
469 5007 : while (auto psEntry = VSIGetNextDirEntry(psDir))
470 : {
471 3859 : if (VSI_ISDIR(psEntry->nMode) && psEntry->pszName[0] &&
472 1474 : psEntry->pszName[strlen(psEntry->pszName) - 1] != SEP)
473 : {
474 1474 : oFiles.AddString((std::string(psEntry->pszName) + SEP).c_str());
475 : }
476 : else
477 : {
478 2385 : oFiles.AddString(psEntry->pszName);
479 : }
480 3859 : }
481 1148 : VSICloseDir(psDir);
482 :
483 1148 : return oFiles.StealList();
484 : }
485 :
486 : /************************************************************************/
487 : /* CPLReadDir() */
488 : /* */
489 : /* This is present only to provide ABI compatibility with older */
490 : /* versions. */
491 : /************************************************************************/
492 : #undef CPLReadDir
493 :
494 : CPL_C_START
495 : char CPL_DLL **CPLReadDir(const char *pszPath);
496 : CPL_C_END
497 :
498 0 : char **CPLReadDir(const char *pszPath)
499 : {
500 0 : return VSIReadDir(pszPath);
501 : }
502 :
503 : /************************************************************************/
504 : /* VSIOpenDir() */
505 : /************************************************************************/
506 :
507 : /**
508 : * \brief Open a directory to read its entries.
509 : *
510 : * This function is close to the POSIX opendir() function.
511 : *
512 : * For /vsis3/, /vsigs/, /vsioss/, /vsiaz/ and /vsiadls/, this function has an
513 : * efficient implementation, minimizing the number of network requests, when
514 : * invoked with nRecurseDepth <= 0.
515 : *
516 : * Entries are read by calling VSIGetNextDirEntry() on the handled returned by
517 : * that function, until it returns NULL. VSICloseDir() must be called once done
518 : * with the returned directory handle.
519 : *
520 : * @param pszPath the relative, or absolute path of a directory to read.
521 : * UTF-8 encoded.
522 : * @param nRecurseDepth 0 means do not recurse in subdirectories, 1 means
523 : * recurse only in the first level of subdirectories, etc. -1 means unlimited
524 : * recursion level
525 : * @param papszOptions NULL terminated list of options, or NULL. The following
526 : * options are implemented:
527 : * <ul>
528 : * <li>PREFIX=string: (GDAL >= 3.4) Filter to select filenames only starting
529 : * with the specified prefix. Implemented efficiently for /vsis3/, /vsigs/,
530 : * and /vsiaz/ (but not /vsiadls/)
531 : * </li>
532 : * <li>NAME_AND_TYPE_ONLY=YES/NO: (GDAL >= 3.4) Defaults to NO. If set to YES,
533 : * only the pszName and nMode members of VSIDIR are guaranteed to be set.
534 : * This is implemented efficiently for the Unix virtual file system.
535 : * </li>
536 : * </ul>
537 : *
538 : * @return a handle, or NULL in case of error
539 : *
540 : */
541 :
542 1541 : VSIDIR *VSIOpenDir(const char *pszPath, int nRecurseDepth,
543 : const char *const *papszOptions)
544 : {
545 1541 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszPath);
546 :
547 1541 : return poFSHandler->OpenDir(pszPath, nRecurseDepth, papszOptions);
548 : }
549 :
550 : /************************************************************************/
551 : /* VSIGetNextDirEntry() */
552 : /************************************************************************/
553 :
554 : /**
555 : * \brief Return the next entry of the directory
556 : *
557 : * This function is close to the POSIX readdir() function. It actually returns
558 : * more information (file size, last modification time), which on 'real' file
559 : * systems involve one 'stat' call per file.
560 : *
561 : * For filesystems that can have both a regular file and a directory name of
562 : * the same name (typically /vsis3/), when this situation of duplicate happens,
563 : * the directory name will be suffixed by a slash character. Otherwise directory
564 : * names are not suffixed by slash.
565 : *
566 : * The returned entry remains valid until the next call to VSINextDirEntry()
567 : * or VSICloseDir() with the same handle.
568 : *
569 : * Note: since GDAL 3.9, for recursive mode, the directory separator will no
570 : * longer be always forward slash, but will be the one returned by
571 : * VSIGetDirectorySeparator(pszPathIn), so potentially backslash on Windows
572 : * file systems.
573 : *
574 : * @param dir Directory handled returned by VSIOpenDir(). Must not be NULL.
575 : *
576 : * @return a entry, or NULL if there is no more entry in the directory. This
577 : * return value must not be freed.
578 : *
579 : */
580 :
581 14284 : const VSIDIREntry *VSIGetNextDirEntry(VSIDIR *dir)
582 : {
583 14284 : return dir->NextDirEntry();
584 : }
585 :
586 : /************************************************************************/
587 : /* VSICloseDir() */
588 : /************************************************************************/
589 :
590 : /**
591 : * \brief Close a directory
592 : *
593 : * This function is close to the POSIX closedir() function.
594 : *
595 : * @param dir Directory handled returned by VSIOpenDir().
596 : *
597 : */
598 :
599 1509 : void VSICloseDir(VSIDIR *dir)
600 : {
601 1509 : delete dir;
602 1509 : }
603 :
604 : /************************************************************************/
605 : /* VSIMkdir() */
606 : /************************************************************************/
607 :
608 : /**
609 : * \brief Create a directory.
610 : *
611 : * Create a new directory with the indicated mode. For POSIX-style systems,
612 : * the mode is modified by the file creation mask (umask). However, some
613 : * file systems and platforms may not use umask, or they may ignore the mode
614 : * completely. So a reasonable cross-platform default mode value is 0755.
615 : *
616 : * Analog of the POSIX mkdir() function.
617 : *
618 : * @param pszPathname the path to the directory to create. UTF-8 encoded.
619 : * @param mode the permissions mode.
620 : *
621 : * @return 0 on success or -1 on an error.
622 : */
623 :
624 99674 : int VSIMkdir(const char *pszPathname, long mode)
625 :
626 : {
627 99674 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszPathname);
628 :
629 99674 : return poFSHandler->Mkdir(pszPathname, mode);
630 : }
631 :
632 : /************************************************************************/
633 : /* VSIMkdirRecursive() */
634 : /************************************************************************/
635 :
636 : /**
637 : * \brief Create a directory and all its ancestors
638 : *
639 : * @param pszPathname the path to the directory to create. UTF-8 encoded.
640 : * @param mode the permissions mode.
641 : *
642 : * @return 0 on success or -1 on an error.
643 : */
644 :
645 140995 : int VSIMkdirRecursive(const char *pszPathname, long mode)
646 : {
647 140995 : if (pszPathname == nullptr || pszPathname[0] == '\0' ||
648 140967 : strncmp("/", pszPathname, 2) == 0)
649 : {
650 41 : return -1;
651 : }
652 :
653 281956 : const CPLString osPathname(pszPathname);
654 : VSIStatBufL sStat;
655 140972 : if (VSIStatL(osPathname, &sStat) == 0)
656 : {
657 50234 : return VSI_ISDIR(sStat.st_mode) ? 0 : -1;
658 : }
659 181536 : const std::string osParentPath(CPLGetPathSafe(osPathname));
660 :
661 : // Prevent crazy paths from recursing forever.
662 181536 : if (osParentPath == osPathname ||
663 90768 : osParentPath.length() >= osPathname.length())
664 : {
665 0 : return -1;
666 : }
667 :
668 90768 : if (!osParentPath.empty() && VSIStatL(osParentPath.c_str(), &sStat) != 0)
669 : {
670 45493 : if (VSIMkdirRecursive(osParentPath.c_str(), mode) != 0)
671 20 : return -1;
672 : }
673 :
674 90747 : return VSIMkdir(osPathname, mode);
675 : }
676 :
677 : /************************************************************************/
678 : /* VSIUnlink() */
679 : /************************************************************************/
680 :
681 : /**
682 : * \brief Delete a file.
683 : *
684 : * Deletes a file object from the file system.
685 : *
686 : * This method goes through the VSIFileHandler virtualization and may
687 : * work on unusual filesystems such as in memory.
688 : *
689 : * Analog of the POSIX unlink() function.
690 : *
691 : * @param pszFilename the path of the file to be deleted. UTF-8 encoded.
692 : *
693 : * @return 0 on success or -1 on an error.
694 : */
695 :
696 86248 : int VSIUnlink(const char *pszFilename)
697 :
698 : {
699 86248 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
700 :
701 86248 : return poFSHandler->Unlink(pszFilename);
702 : }
703 :
704 : /************************************************************************/
705 : /* VSIUnlinkBatch() */
706 : /************************************************************************/
707 :
708 : /**
709 : * \brief Delete several files, possibly in a batch.
710 : *
711 : * All files should belong to the same file system handler.
712 : *
713 : * This is implemented efficiently for /vsis3/ and /vsigs/ (provided for /vsigs/
714 : * that OAuth2 authentication is used).
715 : *
716 : * @param papszFiles NULL terminated list of files. UTF-8 encoded.
717 : *
718 : * @return an array of size CSLCount(papszFiles), whose values are TRUE or FALSE
719 : * depending on the success of deletion of the corresponding file. The array
720 : * should be freed with VSIFree().
721 : * NULL might be return in case of a more general error (for example,
722 : * files belonging to different file system handlers)
723 : *
724 : * @since GDAL 3.1
725 : */
726 :
727 11 : int *VSIUnlinkBatch(CSLConstList papszFiles)
728 : {
729 11 : VSIFilesystemHandler *poFSHandler = nullptr;
730 30 : for (CSLConstList papszIter = papszFiles; papszIter && *papszIter;
731 : ++papszIter)
732 : {
733 20 : auto poFSHandlerThisFile = VSIFileManager::GetHandler(*papszIter);
734 20 : if (poFSHandler == nullptr)
735 10 : poFSHandler = poFSHandlerThisFile;
736 10 : else if (poFSHandler != poFSHandlerThisFile)
737 : {
738 1 : CPLError(CE_Failure, CPLE_AppDefined,
739 : "Files belong to different file system handlers");
740 1 : poFSHandler = nullptr;
741 1 : break;
742 : }
743 : }
744 11 : if (poFSHandler == nullptr)
745 2 : return nullptr;
746 9 : return poFSHandler->UnlinkBatch(papszFiles);
747 : }
748 :
749 : /************************************************************************/
750 : /* VSIRename() */
751 : /************************************************************************/
752 :
753 : /**
754 : * \brief Rename a file.
755 : *
756 : * Renames a file object in the file system. It should be possible
757 : * to rename a file onto a new directory, but it is safest if this
758 : * function is only used to rename files that remain in the same directory.
759 : *
760 : * This function only works if the new path is located on the same VSI
761 : * virtual file system than the old path. If not, use VSIMove() instead.
762 : *
763 : * This method goes through the VSIFileHandler virtualization and may
764 : * work on unusual filesystems such as in memory or cloud object storage.
765 : * Note that for cloud object storage, renaming a directory may involve
766 : * renaming all files it contains recursively, and is thus not an atomic
767 : * operation (and could be expensive on directories with many files!)
768 : *
769 : * Analog of the POSIX rename() function.
770 : *
771 : * @param oldpath the name of the file to be renamed. UTF-8 encoded.
772 : * @param newpath the name the file should be given. UTF-8 encoded.
773 : *
774 : * @return 0 on success or -1 on an error.
775 : */
776 :
777 1474 : int VSIRename(const char *oldpath, const char *newpath)
778 :
779 : {
780 1474 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(oldpath);
781 :
782 1474 : return poFSHandler->Rename(oldpath, newpath, nullptr, nullptr);
783 : }
784 :
785 : /************************************************************************/
786 : /* VSIMove() */
787 : /************************************************************************/
788 :
789 : /**
790 : * \brief Move (or rename) a file.
791 : *
792 : * If the new path is an existing directory, the file will be moved to it.
793 : *
794 : * The function can work even if the files are not located on the same VSI
795 : * virtual file system, but it will involve copying and deletion.
796 : *
797 : * Note that for cloud object storage, moving/renaming a directory may involve
798 : * renaming all files it contains recursively, and is thus not an atomic
799 : * operation (and could be slow and expensive on directories with many files!)
800 : *
801 : * @param oldpath the path of the file to be renamed/moved. UTF-8 encoded.
802 : * @param newpath the new path the file should be given. UTF-8 encoded.
803 : * @param papszOptions Null terminated list of options, or NULL.
804 : * @param pProgressFunc Progress callback, or NULL.
805 : * @param pProgressData User data of progress callback, or NULL.
806 : *
807 : * @return 0 on success or -1 on error.
808 : * @since GDAL 3.11
809 : */
810 :
811 12 : int VSIMove(const char *oldpath, const char *newpath,
812 : const char *const *papszOptions, GDALProgressFunc pProgressFunc,
813 : void *pProgressData)
814 : {
815 :
816 12 : if (strcmp(oldpath, newpath) == 0)
817 1 : return 0;
818 :
819 11 : VSIFilesystemHandler *poOldFSHandler = VSIFileManager::GetHandler(oldpath);
820 11 : VSIFilesystemHandler *poNewFSHandler = VSIFileManager::GetHandler(newpath);
821 :
822 : VSIStatBufL sStat;
823 11 : if (VSIStatL(oldpath, &sStat) != 0)
824 : {
825 2 : CPLDebug("VSI", "%s is not a object", oldpath);
826 2 : errno = ENOENT;
827 2 : return -1;
828 : }
829 :
830 18 : std::string sNewpath(newpath);
831 : VSIStatBufL sStatNew;
832 9 : if (VSIStatL(newpath, &sStatNew) == 0 && VSI_ISDIR(sStatNew.st_mode))
833 : {
834 : sNewpath =
835 4 : CPLFormFilenameSafe(newpath, CPLGetFilename(oldpath), nullptr);
836 : }
837 :
838 9 : int ret = 0;
839 :
840 9 : if (poOldFSHandler == poNewFSHandler)
841 : {
842 2 : ret = poOldFSHandler->Rename(oldpath, sNewpath.c_str(), pProgressFunc,
843 2 : pProgressData);
844 2 : if (ret == 0 && pProgressFunc)
845 1 : ret = pProgressFunc(1.0, "", pProgressData) ? 0 : -1;
846 2 : return ret;
847 : }
848 :
849 7 : if (VSI_ISDIR(sStat.st_mode))
850 : {
851 6 : const CPLStringList aosList(VSIReadDir(oldpath));
852 3 : poNewFSHandler->Mkdir(sNewpath.c_str(), 0755);
853 3 : bool bFoundFiles = false;
854 3 : const int nListSize = aosList.size();
855 6 : for (int i = 0; ret == 0 && i < nListSize; i++)
856 : {
857 3 : if (strcmp(aosList[i], ".") != 0 && strcmp(aosList[i], "..") != 0)
858 : {
859 1 : bFoundFiles = true;
860 : const std::string osSrc =
861 2 : CPLFormFilenameSafe(oldpath, aosList[i], nullptr);
862 : const std::string osTarget =
863 2 : CPLFormFilenameSafe(sNewpath.c_str(), aosList[i], nullptr);
864 2 : void *pScaledProgress = GDALCreateScaledProgress(
865 1 : static_cast<double>(i) / nListSize,
866 1 : static_cast<double>(i + 1) / nListSize, pProgressFunc,
867 : pProgressData);
868 1 : ret = VSIMove(osSrc.c_str(), osTarget.c_str(), papszOptions,
869 : pScaledProgress ? GDALScaledProgress : nullptr,
870 : pScaledProgress);
871 1 : GDALDestroyScaledProgress(pScaledProgress);
872 : }
873 : }
874 3 : if (!bFoundFiles)
875 2 : ret = VSIStatL(sNewpath.c_str(), &sStat);
876 3 : if (ret == 0)
877 3 : ret = poOldFSHandler->Rmdir(oldpath);
878 : }
879 : else
880 : {
881 8 : ret = VSICopyFile(oldpath, sNewpath.c_str(), nullptr, sStat.st_size,
882 3 : nullptr, pProgressFunc, pProgressData) == 0 &&
883 3 : VSIUnlink(oldpath) == 0
884 7 : ? 0
885 : : -1;
886 : }
887 7 : if (ret == 0 && pProgressFunc)
888 4 : ret = pProgressFunc(1.0, "", pProgressData) ? 0 : -1;
889 7 : return ret;
890 : }
891 :
892 : /************************************************************************/
893 : /* VSICopyFile() */
894 : /************************************************************************/
895 :
896 : /**
897 : * \brief Copy a source file into a target file.
898 : *
899 : * For a /vsizip/foo.zip/bar target, the options available are those of
900 : * CPLAddFileInZip()
901 : *
902 : * The following copies are made fully on the target server, without local
903 : * download from source and upload to target:
904 : * - /vsis3/ -> /vsis3/
905 : * - /vsigs/ -> /vsigs/
906 : * - /vsiaz/ -> /vsiaz/
907 : * - /vsiadls/ -> /vsiadls/
908 : * - any of the above or /vsicurl/ -> /vsiaz/ (starting with GDAL 3.8)
909 : *
910 : * @param pszSource Source filename. UTF-8 encoded. May be NULL if fpSource is
911 : * not NULL.
912 : * @param pszTarget Target filename. UTF-8 encoded. Must not be NULL
913 : * @param fpSource File handle on the source file. May be NULL if pszSource is
914 : * not NULL.
915 : * @param nSourceSize Size of the source file. Pass -1 if unknown.
916 : * If set to -1, and progress callback is used, VSIStatL() will be used on
917 : * pszSource to retrieve the source size.
918 : * @param papszOptions Null terminated list of options, or NULL.
919 : * @param pProgressFunc Progress callback, or NULL.
920 : * @param pProgressData User data of progress callback, or NULL.
921 : *
922 : * @return 0 on success.
923 : * @since GDAL 3.7
924 : */
925 :
926 2299 : int VSICopyFile(const char *pszSource, const char *pszTarget,
927 : VSILFILE *fpSource, vsi_l_offset nSourceSize,
928 : const char *const *papszOptions, GDALProgressFunc pProgressFunc,
929 : void *pProgressData)
930 :
931 : {
932 2299 : if (!pszSource && !fpSource)
933 : {
934 1 : CPLError(CE_Failure, CPLE_AppDefined,
935 : "pszSource == nullptr && fpSource == nullptr");
936 1 : return -1;
937 : }
938 2298 : if (!pszTarget || pszTarget[0] == '\0')
939 : {
940 0 : return -1;
941 : }
942 :
943 : VSIFilesystemHandler *poFSHandlerTarget =
944 2298 : VSIFileManager::GetHandler(pszTarget);
945 2298 : return poFSHandlerTarget->CopyFile(pszSource, pszTarget, fpSource,
946 : nSourceSize, papszOptions, pProgressFunc,
947 2298 : pProgressData);
948 : }
949 :
950 : /************************************************************************/
951 : /* VSICopyFileRestartable() */
952 : /************************************************************************/
953 :
954 : /**
955 : \brief Copy a source file into a target file in a way that can (potentially)
956 : be restarted.
957 :
958 : This function provides the possibility of efficiently restarting upload of
959 : large files to cloud storage that implements upload in a chunked way,
960 : such as /vsis3/ and /vsigs/.
961 : For other destination file systems, this function may fallback to
962 : VSICopyFile() and not provide any smart restartable implementation.
963 :
964 : Example of a potential workflow:
965 :
966 : @code{.cpp}
967 : char* pszOutputPayload = NULL;
968 : int ret = VSICopyFileRestartable(pszSource, pszTarget, NULL,
969 : &pszOutputPayload, NULL, NULL, NULL);
970 : while( ret == 1 ) // add also a limiting counter to avoid potentiall endless looping
971 : {
972 : // TODO: wait for some time
973 :
974 : char* pszOutputPayloadNew = NULL;
975 : const char* pszInputPayload = pszOutputPayload;
976 : ret = VSICopyFileRestartable(pszSource, pszTarget, pszInputPayload,
977 : &pszOutputPayloadNew, NULL, NULL, NULL);
978 : VSIFree(pszOutputPayload);
979 : pszOutputPayload = pszOutputPayloadNew;
980 : }
981 : VSIFree(pszOutputPayload);
982 : @endcode
983 :
984 : @param pszSource Source filename. UTF-8 encoded. Must not be NULL
985 : @param pszTarget Target filename. UTF-8 encoded. Must not be NULL
986 : @param pszInputPayload NULL at the first invocation. When doing a retry,
987 : should be the content of *ppszOutputPayload from a
988 : previous invocation.
989 : @param[out] ppszOutputPayload Pointer to an output string that will be set to
990 : a value that can be provided as pszInputPayload
991 : for a next call to VSICopyFileRestartable().
992 : ppszOutputPayload must not be NULL.
993 : The string set in *ppszOutputPayload, if not NULL,
994 : is JSON-encoded, and can be re-used in another
995 : process instance. It must be freed with VSIFree()
996 : when no longer needed.
997 : @param papszOptions Null terminated list of options, or NULL.
998 : Currently accepted options are:
999 : <ul>
1000 : <li>NUM_THREADS=integer or ALL_CPUS. Number of threads to use for parallel
1001 : file copying. Only use for when /vsis3/, /vsigs/, /vsiaz/ or /vsiadls/ is in
1002 : source or target. The default is 10.
1003 : </li>
1004 : <li>CHUNK_SIZE=integer. Maximum size of chunk (in bytes) to use
1005 : to split large objects. For upload to /vsis3/, this chunk size must be set at
1006 : least to 5 MB. The default is 50 MB.
1007 : </li>
1008 : </ul>
1009 : @param pProgressFunc Progress callback, or NULL.
1010 : @param pProgressData User data of progress callback, or NULL.
1011 : @return 0 on success,
1012 : -1 on (non-restartable) failure,
1013 : 1 if VSICopyFileRestartable() can be called again in a restartable way
1014 : @since GDAL 3.10
1015 :
1016 : @see VSIAbortPendingUploads()
1017 : */
1018 :
1019 20 : int VSICopyFileRestartable(const char *pszSource, const char *pszTarget,
1020 : const char *pszInputPayload,
1021 : char **ppszOutputPayload,
1022 : const char *const *papszOptions,
1023 : GDALProgressFunc pProgressFunc, void *pProgressData)
1024 :
1025 : {
1026 20 : if (!pszSource)
1027 : {
1028 0 : return -1;
1029 : }
1030 20 : if (!pszTarget || pszTarget[0] == '\0')
1031 : {
1032 0 : return -1;
1033 : }
1034 20 : if (!ppszOutputPayload)
1035 : {
1036 0 : return -1;
1037 : }
1038 :
1039 : VSIFilesystemHandler *poFSHandlerTarget =
1040 20 : VSIFileManager::GetHandler(pszTarget);
1041 20 : return poFSHandlerTarget->CopyFileRestartable(
1042 : pszSource, pszTarget, pszInputPayload, ppszOutputPayload, papszOptions,
1043 20 : pProgressFunc, pProgressData);
1044 : }
1045 :
1046 : /************************************************************************/
1047 : /* VSISync() */
1048 : /************************************************************************/
1049 :
1050 : /**
1051 : * \brief Synchronize a source file/directory with a target file/directory.
1052 : *
1053 : * This is a analog of the 'rsync' utility. In the current implementation,
1054 : * rsync would be more efficient for local file copying, but VSISync() main
1055 : * interest is when the source or target is a remote
1056 : * file system like /vsis3/ or /vsigs/, in which case it can take into account
1057 : * the timestamps of the files (or optionally the ETag/MD5Sum) to avoid
1058 : * unneeded copy operations.
1059 : *
1060 : * This is only implemented efficiently for:
1061 : * <ul>
1062 : * <li> local filesystem <--> remote filesystem.</li>
1063 : * <li> remote filesystem <--> remote filesystem (starting with GDAL 3.1).
1064 : * Where the source and target remote filesystems are the same and one of
1065 : * /vsis3/, /vsigs/ or /vsiaz/. Or when the target is /vsiaz/ and the source
1066 : * is /vsis3/, /vsigs/, /vsiadls/ or /vsicurl/ (starting with GDAL 3.8)</li>
1067 : * </ul>
1068 : *
1069 : * Similarly to rsync behavior, if the source filename ends with a slash,
1070 : * it means that the content of the directory must be copied, but not the
1071 : * directory name. For example, assuming "/home/even/foo" contains a file "bar",
1072 : * VSISync("/home/even/foo/", "/mnt/media", ...) will create a "/mnt/media/bar"
1073 : * file. Whereas VSISync("/home/even/foo", "/mnt/media", ...) will create a
1074 : * "/mnt/media/foo" directory which contains a bar file.
1075 : *
1076 : * @param pszSource Source file or directory. UTF-8 encoded.
1077 : * @param pszTarget Target file or directory. UTF-8 encoded.
1078 : * @param papszOptions Null terminated list of options, or NULL.
1079 : * Currently accepted options are:
1080 : * <ul>
1081 : * <li>RECURSIVE=NO (the default is YES)</li>
1082 : * <li>SYNC_STRATEGY=TIMESTAMP/ETAG/OVERWRITE.
1083 : *
1084 : * Determines which criterion is used to determine if a target file must be
1085 : * replaced when it already exists and has the same file size as the source.
1086 : * Only applies for a source or target being a network filesystem.
1087 : *
1088 : * The default is TIMESTAMP (similarly to how 'aws s3 sync' works), that is
1089 : * to say that for an upload operation, a remote file is
1090 : * replaced if it has a different size or if it is older than the source.
1091 : * For a download operation, a local file is replaced if it has a different
1092 : * size or if it is newer than the remote file.
1093 : *
1094 : * The ETAG strategy assumes that the ETag metadata of the remote file is
1095 : * the MD5Sum of the file content, which is only true in the case of /vsis3/
1096 : * for files not using KMS server side encryption and uploaded in a single
1097 : * PUT operation (so smaller than 50 MB given the default used by GDAL).
1098 : * Only to be used for /vsis3/, /vsigs/ or other filesystems using a
1099 : * MD5Sum as ETAG.
1100 : *
1101 : * The OVERWRITE strategy (GDAL >= 3.2) will always overwrite the target
1102 : * file with the source one.
1103 : * </li>
1104 : * <li>NUM_THREADS=integer. (GDAL >= 3.1) Number of threads to use for parallel
1105 : * file copying. Only use for when /vsis3/, /vsigs/, /vsiaz/ or /vsiadls/ is in
1106 : * source or target. The default is 10 since GDAL 3.3</li>
1107 : * <li>CHUNK_SIZE=integer. (GDAL >= 3.1) Maximum size of chunk (in bytes) to use
1108 : * to split large objects when downloading them from /vsis3/, /vsigs/, /vsiaz/
1109 : * or /vsiadls/ to local file system, or for upload to /vsis3/, /vsiaz/ or
1110 : * /vsiadls/ from local file system. Only used if NUM_THREADS > 1. For upload to
1111 : * /vsis3/, this chunk size must be set at least to 5 MB. The default is 8 MB
1112 : * since GDAL 3.3</li> <li>x-amz-KEY=value. (GDAL >= 3.5) MIME header to pass
1113 : * during creation of a /vsis3/ object.</li> <li>x-goog-KEY=value. (GDAL >= 3.5)
1114 : * MIME header to pass during creation of a /vsigs/ object.</li>
1115 : * <li>x-ms-KEY=value. (GDAL >= 3.5) MIME header to pass during creation of a
1116 : * /vsiaz/ or /vsiadls/ object.</li>
1117 : * </ul>
1118 : * @param pProgressFunc Progress callback, or NULL.
1119 : * @param pProgressData User data of progress callback, or NULL.
1120 : * @param ppapszOutputs Unused. Should be set to NULL for now.
1121 : *
1122 : * @return TRUE on success or FALSE on an error.
1123 : */
1124 :
1125 49 : int VSISync(const char *pszSource, const char *pszTarget,
1126 : const char *const *papszOptions, GDALProgressFunc pProgressFunc,
1127 : void *pProgressData, char ***ppapszOutputs)
1128 :
1129 : {
1130 49 : if (pszSource[0] == '\0' || pszTarget[0] == '\0')
1131 : {
1132 0 : return FALSE;
1133 : }
1134 :
1135 : VSIFilesystemHandler *poFSHandlerSource =
1136 49 : VSIFileManager::GetHandler(pszSource);
1137 : VSIFilesystemHandler *poFSHandlerTarget =
1138 49 : VSIFileManager::GetHandler(pszTarget);
1139 49 : VSIFilesystemHandler *poFSHandlerLocal = VSIFileManager::GetHandler("");
1140 : VSIFilesystemHandler *poFSHandlerMem =
1141 49 : VSIFileManager::GetHandler("/vsimem/");
1142 49 : VSIFilesystemHandler *poFSHandler = poFSHandlerSource;
1143 49 : if (poFSHandlerTarget != poFSHandlerLocal &&
1144 : poFSHandlerTarget != poFSHandlerMem)
1145 : {
1146 22 : poFSHandler = poFSHandlerTarget;
1147 : }
1148 :
1149 98 : return poFSHandler->Sync(pszSource, pszTarget, papszOptions, pProgressFunc,
1150 49 : pProgressData, ppapszOutputs)
1151 49 : ? TRUE
1152 49 : : FALSE;
1153 : }
1154 :
1155 : /************************************************************************/
1156 : /* VSIMultipartUploadGetCapabilities() */
1157 : /************************************************************************/
1158 :
1159 : /**
1160 : * \brief Return capabilities for multiple part file upload.
1161 : *
1162 : * @param pszFilename Filename, or virtual file system prefix, onto which
1163 : * capabilities should apply.
1164 : * @param[out] pbNonSequentialUploadSupported If not null,
1165 : * the pointed value is set if parts can be uploaded in a non-sequential way.
1166 : * @param[out] pbParallelUploadSupported If not null,
1167 : * the pointed value is set if parts can be uploaded in a parallel way.
1168 : * (implies *pbNonSequentialUploadSupported = true)
1169 : * @param[out] pbAbortSupported If not null,
1170 : * the pointed value is set if VSIMultipartUploadAbort() is implemented.
1171 : * @param[out] pnMinPartSize If not null, the pointed value is set to the minimum
1172 : * size of parts (but the last one), in MiB.
1173 : * @param[out] pnMaxPartSize If not null, the pointed value is set to the maximum
1174 : * size of parts, in MiB.
1175 : * @param[out] pnMaxPartCount If not null, the pointed value is set to the
1176 : * maximum number of parts that can be uploaded.
1177 : *
1178 : * @return TRUE in case of success, FALSE otherwise.
1179 : *
1180 : * @since 3.10
1181 : */
1182 7 : int VSIMultipartUploadGetCapabilities(
1183 : const char *pszFilename, int *pbNonSequentialUploadSupported,
1184 : int *pbParallelUploadSupported, int *pbAbortSupported,
1185 : size_t *pnMinPartSize, size_t *pnMaxPartSize, int *pnMaxPartCount)
1186 : {
1187 7 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
1188 :
1189 14 : return poFSHandler->MultipartUploadGetCapabilities(
1190 : pbNonSequentialUploadSupported, pbParallelUploadSupported,
1191 7 : pbAbortSupported, pnMinPartSize, pnMaxPartSize, pnMaxPartCount);
1192 : }
1193 :
1194 : /************************************************************************/
1195 : /* VSIMultipartUploadStart() */
1196 : /************************************************************************/
1197 :
1198 : /**
1199 : * \brief Initiates the upload a (big) file in a piece-wise way.
1200 : *
1201 : * Using this API directly is generally not needed, but in very advanced cases,
1202 : * as VSIFOpenL(..., "wb") + VSIFWriteL(), VSISync(), VSICopyFile() or
1203 : * VSICopyFileRestartable() may be able to leverage it when needed.
1204 : *
1205 : * This is only implemented for the /vsis3/, /vsigs/, /vsiaz/, /vsiadls/ and
1206 : * /vsioss/ virtual file systems.
1207 : *
1208 : * The typical workflow is to do :
1209 : * - VSIMultipartUploadStart()
1210 : * - VSIMultipartUploadAddPart(): several times
1211 : * - VSIMultipartUploadEnd()
1212 : *
1213 : * If VSIMultipartUploadAbort() is supported by the filesystem (VSIMultipartUploadGetCapabilities()
1214 : * can be used to determine it), this function should be called to cancel an
1215 : * upload. This can be needed to avoid extra billing for some cloud storage
1216 : * providers.
1217 : *
1218 : * The following options are supported:
1219 : * <ul>
1220 : * <li>MIME headers such as Content-Type and Content-Encoding
1221 : * are supported for the /vsis3/, /vsigs/, /vsiaz/, /vsiadls/ file systems.</li>
1222 : * </ul>
1223 : *
1224 : * @param pszFilename Filename to create
1225 : * @param papszOptions NULL or null-terminated list of options.
1226 : * @return an upload ID to pass to other VSIMultipartUploadXXXXX() functions,
1227 : * and to free with CPLFree() once done, or nullptr in case of error.
1228 : *
1229 : * @since 3.10
1230 : */
1231 4 : char *VSIMultipartUploadStart(const char *pszFilename,
1232 : CSLConstList papszOptions)
1233 : {
1234 4 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
1235 :
1236 4 : return poFSHandler->MultipartUploadStart(pszFilename, papszOptions);
1237 : }
1238 :
1239 : /************************************************************************/
1240 : /* VSIMultipartUploadAddPart() */
1241 : /************************************************************************/
1242 :
1243 : /**
1244 : * \brief Uploads a new part to a multi-part uploaded file.
1245 : *
1246 : * Cf VSIMultipartUploadStart().
1247 : *
1248 : * VSIMultipartUploadGetCapabilities() returns hints on the constraints that
1249 : * apply to the upload, in terms of minimum/maximum size of each part, maximum
1250 : * number of parts, and whether non-sequential or parallel uploads are
1251 : * supported.
1252 : *
1253 : * @param pszFilename Filename to which to append the new part. Should be the
1254 : * same as the one used for VSIMultipartUploadStart()
1255 : * @param pszUploadId Value returned by VSIMultipartUploadStart()
1256 : * @param nPartNumber Part number, starting at 1.
1257 : * @param nFileOffset Offset within the file at which (starts at 0) the passed
1258 : * data starts.
1259 : * @param pData Pointer to an array of nDataLength bytes.
1260 : * @param nDataLength Size in bytes of pData.
1261 : * @param papszOptions Unused. Should be nullptr.
1262 : *
1263 : * @return a part identifier that must be passed into the apszPartIds[] array of
1264 : * VSIMultipartUploadEnd(), and to free with CPLFree() once done, or nullptr in
1265 : * case of error.
1266 : *
1267 : * @since 3.10
1268 : */
1269 5 : char *VSIMultipartUploadAddPart(const char *pszFilename,
1270 : const char *pszUploadId, int nPartNumber,
1271 : vsi_l_offset nFileOffset, const void *pData,
1272 : size_t nDataLength, CSLConstList papszOptions)
1273 : {
1274 5 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
1275 :
1276 5 : return poFSHandler->MultipartUploadAddPart(pszFilename, pszUploadId,
1277 : nPartNumber, nFileOffset, pData,
1278 5 : nDataLength, papszOptions);
1279 : }
1280 :
1281 : /************************************************************************/
1282 : /* VSIMultipartUploadEnd() */
1283 : /************************************************************************/
1284 :
1285 : /**
1286 : * \brief Completes a multi-part file upload.
1287 : *
1288 : * Cf VSIMultipartUploadStart().
1289 : *
1290 : * @param pszFilename Filename for which multipart upload should be completed.
1291 : * Should be the same as the one used for
1292 : * VSIMultipartUploadStart()
1293 : * @param pszUploadId Value returned by VSIMultipartUploadStart()
1294 : * @param nPartIdsCount Number of parts, andsize of apszPartIds
1295 : * @param apszPartIds Array of part identifiers (as returned by
1296 : * VSIMultipartUploadAddPart()), that must be ordered in
1297 : * the sequential order of parts, and of size nPartIdsCount.
1298 : * @param nTotalSize Total size of the file in bytes (must be equal to the sum
1299 : * of nDataLength passed to VSIMultipartUploadAddPart())
1300 : * @param papszOptions Unused. Should be nullptr.
1301 : *
1302 : * @return TRUE in case of success, FALSE in case of failure.
1303 : *
1304 : * @since 3.10
1305 : */
1306 5 : int VSIMultipartUploadEnd(const char *pszFilename, const char *pszUploadId,
1307 : size_t nPartIdsCount, const char *const *apszPartIds,
1308 : vsi_l_offset nTotalSize, CSLConstList papszOptions)
1309 : {
1310 5 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
1311 :
1312 10 : return poFSHandler->MultipartUploadEnd(pszFilename, pszUploadId,
1313 : nPartIdsCount, apszPartIds,
1314 5 : nTotalSize, papszOptions);
1315 : }
1316 :
1317 : /************************************************************************/
1318 : /* VSIMultipartUploadAbort() */
1319 : /************************************************************************/
1320 :
1321 : /**
1322 : * \brief Aborts a multi-part file upload.
1323 : *
1324 : * Cf VSIMultipartUploadStart().
1325 : *
1326 : * This function is not implemented for all virtual file systems.
1327 : * Use VSIMultipartUploadGetCapabilities() to determine if it is supported.
1328 : *
1329 : * This can be needed to avoid extra billing for some cloud storage providers.
1330 : *
1331 : * @param pszFilename Filename for which multipart upload should be completed.
1332 : * Should be the same as the one used for
1333 : * VSIMultipartUploadStart()
1334 : * @param pszUploadId Value returned by VSIMultipartUploadStart()
1335 : * @param papszOptions Unused. Should be nullptr.
1336 : *
1337 : * @return TRUE in case of success, FALSE in case of failure.
1338 : *
1339 : * @since 3.10
1340 : */
1341 6 : int VSIMultipartUploadAbort(const char *pszFilename, const char *pszUploadId,
1342 : CSLConstList papszOptions)
1343 : {
1344 6 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
1345 :
1346 12 : return poFSHandler->MultipartUploadAbort(pszFilename, pszUploadId,
1347 6 : papszOptions);
1348 : }
1349 :
1350 : #ifndef DOXYGEN_SKIP
1351 :
1352 : /************************************************************************/
1353 : /* MultipartUploadGetCapabilities() */
1354 : /************************************************************************/
1355 :
1356 2 : bool VSIFilesystemHandler::MultipartUploadGetCapabilities(int *, int *, int *,
1357 : size_t *, size_t *,
1358 : int *)
1359 : {
1360 2 : CPLError(
1361 : CE_Failure, CPLE_NotSupported,
1362 : "MultipartUploadGetCapabilities() not supported by this file system");
1363 2 : return false;
1364 : }
1365 :
1366 : /************************************************************************/
1367 : /* MultipartUploadStart() */
1368 : /************************************************************************/
1369 :
1370 1 : char *VSIFilesystemHandler::MultipartUploadStart(const char *, CSLConstList)
1371 : {
1372 1 : CPLError(CE_Failure, CPLE_NotSupported,
1373 : "MultipartUploadStart() not supported by this file system");
1374 1 : return nullptr;
1375 : }
1376 :
1377 : /************************************************************************/
1378 : /* MultipartUploadAddPart() */
1379 : /************************************************************************/
1380 :
1381 1 : char *VSIFilesystemHandler::MultipartUploadAddPart(const char *, const char *,
1382 : int, vsi_l_offset,
1383 : const void *, size_t,
1384 : CSLConstList)
1385 : {
1386 1 : CPLError(CE_Failure, CPLE_NotSupported,
1387 : "MultipartUploadAddPart() not supported by this file system");
1388 1 : return nullptr;
1389 : }
1390 :
1391 : /************************************************************************/
1392 : /* MultipartUploadEnd() */
1393 : /************************************************************************/
1394 :
1395 1 : bool VSIFilesystemHandler::MultipartUploadEnd(const char *, const char *,
1396 : size_t, const char *const *,
1397 : vsi_l_offset, CSLConstList)
1398 : {
1399 1 : CPLError(CE_Failure, CPLE_NotSupported,
1400 : "MultipartUploadEnd() not supported by this file system");
1401 1 : return FALSE;
1402 : }
1403 :
1404 : /************************************************************************/
1405 : /* MultipartUploadAbort() */
1406 : /************************************************************************/
1407 :
1408 1 : bool VSIFilesystemHandler::MultipartUploadAbort(const char *, const char *,
1409 : CSLConstList)
1410 : {
1411 1 : CPLError(CE_Failure, CPLE_NotSupported,
1412 : "MultipartUploadAbort() not supported by this file system");
1413 1 : return FALSE;
1414 : }
1415 :
1416 : #endif
1417 :
1418 : /************************************************************************/
1419 : /* VSIAbortPendingUploads() */
1420 : /************************************************************************/
1421 :
1422 : /**
1423 : * \brief Abort all ongoing multi-part uploads.
1424 : *
1425 : * Abort ongoing multi-part uploads on AWS S3 and Google Cloud Storage. This
1426 : * can be used in case a process doing such uploads was killed in a unclean way.
1427 : *
1428 : * This can be needed to avoid extra billing for some cloud storage providers.
1429 : *
1430 : * Without effect on other virtual file systems.
1431 : *
1432 : * VSIMultipartUploadAbort() can also be used to cancel a given upload, if the
1433 : * upload ID is known.
1434 : *
1435 : * @param pszFilename filename or prefix of a directory into which multipart
1436 : * uploads must be aborted. This can be the root directory of a bucket. UTF-8
1437 : * encoded.
1438 : *
1439 : * @return TRUE on success or FALSE on an error.
1440 : * @since GDAL 3.4
1441 : */
1442 :
1443 1 : int VSIAbortPendingUploads(const char *pszFilename)
1444 : {
1445 1 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
1446 :
1447 1 : return poFSHandler->AbortPendingUploads(pszFilename);
1448 : }
1449 :
1450 : /************************************************************************/
1451 : /* VSIRmdir() */
1452 : /************************************************************************/
1453 :
1454 : /**
1455 : * \brief Delete a directory.
1456 : *
1457 : * Deletes a directory object from the file system. On some systems
1458 : * the directory must be empty before it can be deleted.
1459 : *
1460 : * This method goes through the VSIFileHandler virtualization and may
1461 : * work on unusual filesystems such as in memory.
1462 : *
1463 : * Analog of the POSIX rmdir() function.
1464 : *
1465 : * @param pszDirname the path of the directory to be deleted. UTF-8 encoded.
1466 : *
1467 : * @return 0 on success or -1 on an error.
1468 : */
1469 :
1470 164 : int VSIRmdir(const char *pszDirname)
1471 :
1472 : {
1473 164 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszDirname);
1474 :
1475 164 : return poFSHandler->Rmdir(pszDirname);
1476 : }
1477 :
1478 : /************************************************************************/
1479 : /* VSIRmdirRecursive() */
1480 : /************************************************************************/
1481 :
1482 : /**
1483 : * \brief Delete a directory recursively
1484 : *
1485 : * Deletes a directory object and its content from the file system.
1486 : *
1487 : * Starting with GDAL 3.1, /vsis3/ has an efficient implementation of this
1488 : * function.
1489 : * Starting with GDAL 3.4, /vsigs/ has an efficient implementation of this
1490 : * function, provided that OAuth2 authentication is used.
1491 : *
1492 : * @return 0 on success or -1 on an error.
1493 : */
1494 :
1495 5188 : int VSIRmdirRecursive(const char *pszDirname)
1496 : {
1497 5188 : if (pszDirname == nullptr || pszDirname[0] == '\0' ||
1498 5188 : strncmp("/", pszDirname, 2) == 0)
1499 : {
1500 0 : return -1;
1501 : }
1502 5188 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszDirname);
1503 5188 : return poFSHandler->RmdirRecursive(pszDirname);
1504 : }
1505 :
1506 : /************************************************************************/
1507 : /* VSIStatL() */
1508 : /************************************************************************/
1509 :
1510 : /**
1511 : * \brief Get filesystem object info.
1512 : *
1513 : * Fetches status information about a filesystem object (file, directory, etc).
1514 : * The returned information is placed in the VSIStatBufL structure. For
1515 : * portability, only use the st_size (size in bytes) and st_mode (file type).
1516 : * This method is similar to VSIStat(), but will work on large files on
1517 : * systems where this requires special calls.
1518 : *
1519 : * This method goes through the VSIFileHandler virtualization and may
1520 : * work on unusual filesystems such as in memory.
1521 : *
1522 : * Analog of the POSIX stat() function.
1523 : *
1524 : * @param pszFilename the path of the filesystem object to be queried.
1525 : * UTF-8 encoded.
1526 : * @param psStatBuf the structure to load with information.
1527 : *
1528 : * @return 0 on success or -1 on an error.
1529 : */
1530 :
1531 443398 : int VSIStatL(const char *pszFilename, VSIStatBufL *psStatBuf)
1532 :
1533 : {
1534 443398 : return VSIStatExL(pszFilename, psStatBuf, 0);
1535 : }
1536 :
1537 : /************************************************************************/
1538 : /* VSIStatExL() */
1539 : /************************************************************************/
1540 :
1541 : /**
1542 : * \brief Get filesystem object info.
1543 : *
1544 : * Fetches status information about a filesystem object (file, directory, etc).
1545 : * The returned information is placed in the VSIStatBufL structure. For
1546 : * portability, only use the st_size (size in bytes) and st_mode (file type).
1547 : * This method is similar to VSIStat(), but will work on large files on
1548 : * systems where this requires special calls.
1549 : *
1550 : * This method goes through the VSIFileHandler virtualization and may
1551 : * work on unusual filesystems such as in memory.
1552 : *
1553 : * Analog of the POSIX stat() function, with an extra parameter to
1554 : * specify which information is needed, which offers a potential for
1555 : * speed optimizations on specialized and potentially slow virtual
1556 : * filesystem objects (/vsigzip/, /vsicurl/)
1557 : *
1558 : * @param pszFilename the path of the filesystem object to be queried.
1559 : * UTF-8 encoded.
1560 : * @param psStatBuf the structure to load with information.
1561 : * @param nFlags 0 to get all information, or VSI_STAT_EXISTS_FLAG,
1562 : * VSI_STAT_NATURE_FLAG, VSI_STAT_SIZE_FLAG,
1563 : * VSI_STAT_SET_ERROR_FLAG, VSI_STAT_CACHE_ONLY or a combination of those to get
1564 : * partial info.
1565 : *
1566 : * @return 0 on success or -1 on an error.
1567 : *
1568 : */
1569 :
1570 834801 : int VSIStatExL(const char *pszFilename, VSIStatBufL *psStatBuf, int nFlags)
1571 :
1572 : {
1573 834801 : char szAltPath[4] = {'\0'};
1574 :
1575 : // Enable to work on "C:" as if it were "C:\".
1576 834801 : if (pszFilename[0] != '\0' && pszFilename[1] == ':' &&
1577 20 : pszFilename[2] == '\0')
1578 : {
1579 0 : szAltPath[0] = pszFilename[0];
1580 0 : szAltPath[1] = pszFilename[1];
1581 0 : szAltPath[2] = '\\';
1582 0 : szAltPath[3] = '\0';
1583 :
1584 0 : pszFilename = szAltPath;
1585 : }
1586 :
1587 834801 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
1588 :
1589 834813 : if (nFlags == 0)
1590 447224 : nFlags =
1591 : VSI_STAT_EXISTS_FLAG | VSI_STAT_NATURE_FLAG | VSI_STAT_SIZE_FLAG;
1592 :
1593 1669630 : return poFSHandler->Stat(pszFilename, psStatBuf, nFlags);
1594 : }
1595 :
1596 : /************************************************************************/
1597 : /* VSIGetFileMetadata() */
1598 : /************************************************************************/
1599 :
1600 : /**
1601 : * \brief Get metadata on files.
1602 : *
1603 : * Implemented currently only for network-like filesystems, or starting
1604 : * with GDAL 3.7 for /vsizip/
1605 : *
1606 : * Starting with GDAL 3.11, calling it with pszFilename being the root of a
1607 : * /vsigs/ bucket and pszDomain == nullptr, and when authenticated through
1608 : * OAuth2, will result in returning the result of a "Buckets: get"
1609 : * operation (https://cloud.google.com/storage/docs/json_api/v1/buckets/get),
1610 : * with the keys of the top-level JSON document as keys of the key=value pairs
1611 : * returned by this function.
1612 : *
1613 : * @param pszFilename the path of the filesystem object to be queried.
1614 : * UTF-8 encoded.
1615 : * @param pszDomain Metadata domain to query. Depends on the file system.
1616 : * The following ones are supported:
1617 : * <ul>
1618 : * <li>HEADERS: to get HTTP headers for network-like filesystems (/vsicurl/,
1619 : * /vsis3/, /vsgis/, etc)</li>
1620 : * <li>TAGS:
1621 : * <ul>
1622 : * <li>/vsis3/: to get S3 Object tagging information</li>
1623 : * <li>/vsiaz/: to get blob tags. Refer to
1624 : * https://docs.microsoft.com/en-us/rest/api/storageservices/get-blob-tags
1625 : * </li>
1626 : * </ul>
1627 : * </li>
1628 : * <li>STATUS: specific to /vsiadls/: returns all system defined properties for
1629 : * a path (seems in practice to be a subset of HEADERS)</li> <li>ACL: specific
1630 : * to /vsiadls/ and /vsigs/: returns the access control list for a path. For
1631 : * /vsigs/, a single XML=xml_content string is returned. Refer to
1632 : * https://cloud.google.com/storage/docs/xml-api/get-object-acls
1633 : * </li>
1634 : * <li>METADATA: specific to /vsiaz/: to get blob metadata. Refer to
1635 : * https://docs.microsoft.com/en-us/rest/api/storageservices/get-blob-metadata.
1636 : * Note: this will be a subset of what pszDomain=HEADERS returns</li>
1637 : * <li>ZIP: specific to /vsizip/: to obtain ZIP specific metadata, in particular
1638 : * if a file is SOZIP-enabled (SOZIP_VALID=YES)</li>
1639 : * </ul>
1640 : * @param papszOptions Unused. Should be set to NULL.
1641 : *
1642 : * @return a NULL-terminated list of key=value strings, to be freed with
1643 : * CSLDestroy() or NULL in case of error / empty list.
1644 : *
1645 : * @since GDAL 3.1.0
1646 : */
1647 :
1648 131 : char **VSIGetFileMetadata(const char *pszFilename, const char *pszDomain,
1649 : CSLConstList papszOptions)
1650 : {
1651 131 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
1652 131 : return poFSHandler->GetFileMetadata(pszFilename, pszDomain, papszOptions);
1653 : }
1654 :
1655 : /************************************************************************/
1656 : /* VSISetFileMetadata() */
1657 : /************************************************************************/
1658 :
1659 : /**
1660 : * \brief Set metadata on files.
1661 : *
1662 : * Implemented currently only for /vsis3/, /vsigs/, /vsiaz/ and /vsiadls/
1663 : *
1664 : * @param pszFilename the path of the filesystem object to be set.
1665 : * UTF-8 encoded.
1666 : * @param papszMetadata NULL-terminated list of key=value strings.
1667 : * @param pszDomain Metadata domain to set. Depends on the file system.
1668 : * The following are supported:
1669 : * <ul>
1670 : * <li>HEADERS: specific to /vsis3/ and /vsigs/: to set HTTP headers, such as
1671 : * "Content-Type", or other file system specific header.
1672 : * For /vsigs/, this also includes: x-goog-meta-{key}={value}. Note that you
1673 : * should specify all metadata to be set, as existing metadata will be
1674 : * overridden.
1675 : * </li>
1676 : * <li>TAGS: Content of papszMetadata should be KEY=VALUE pairs.
1677 : * <ul>
1678 : * <li>/vsis3/: to set S3 Object tagging information</li>
1679 : * <li>/vsiaz/: to set blob tags. Refer to
1680 : * https://docs.microsoft.com/en-us/rest/api/storageservices/set-blob-tags.
1681 : * Note: storageV2 must be enabled on the account</li>
1682 : * </ul>
1683 : * </li>
1684 : * <li>PROPERTIES:
1685 : * <ul>
1686 : * <li>to /vsiaz/: to set properties. Refer to
1687 : * https://docs.microsoft.com/en-us/rest/api/storageservices/set-blob-properties.</li>
1688 : * <li>to /vsiadls/: to set properties. Refer to
1689 : * https://docs.microsoft.com/en-us/rest/api/storageservices/datalakestoragegen2/path/update
1690 : * for headers valid for action=setProperties.</li>
1691 : * </ul>
1692 : * </li>
1693 : * <li>ACL: specific to /vsiadls/ and /vsigs/: to set access control list.
1694 : * For /vsiadls/, refer to
1695 : * https://docs.microsoft.com/en-us/rest/api/storageservices/datalakestoragegen2/path/update
1696 : * for headers valid for action=setAccessControl or setAccessControlRecursive.
1697 : * In setAccessControlRecursive, x-ms-acl must be specified in papszMetadata.
1698 : * For /vsigs/, refer to
1699 : * https://cloud.google.com/storage/docs/xml-api/put-object-acls. A single
1700 : * XML=xml_content string should be specified as in papszMetadata.
1701 : * </li>
1702 : * <li>METADATA: specific to /vsiaz/: to set blob metadata. Refer to
1703 : * https://docs.microsoft.com/en-us/rest/api/storageservices/set-blob-metadata.
1704 : * Content of papszMetadata should be strings in the form
1705 : * x-ms-meta-name=value</li>
1706 : * </ul>
1707 : * @param papszOptions NULL or NULL terminated list of options.
1708 : * For /vsiadls/ and pszDomain=ACL, "RECURSIVE=TRUE" can be
1709 : * set to set the access control list recursively. When
1710 : * RECURSIVE=TRUE is set, MODE should also be set to one of
1711 : * "set", "modify" or "remove".
1712 : *
1713 : * @return TRUE in case of success.
1714 : *
1715 : * @since GDAL 3.1.0
1716 : */
1717 :
1718 17 : int VSISetFileMetadata(const char *pszFilename, CSLConstList papszMetadata,
1719 : const char *pszDomain, CSLConstList papszOptions)
1720 : {
1721 17 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
1722 34 : return poFSHandler->SetFileMetadata(pszFilename, papszMetadata, pszDomain,
1723 17 : papszOptions)
1724 17 : ? 1
1725 17 : : 0;
1726 : }
1727 :
1728 : /************************************************************************/
1729 : /* VSIIsCaseSensitiveFS() */
1730 : /************************************************************************/
1731 :
1732 : /**
1733 : * \brief Returns if the filenames of the filesystem are case sensitive.
1734 : *
1735 : * This method retrieves to which filesystem belongs the passed filename
1736 : * and return TRUE if the filenames of that filesystem are case sensitive.
1737 : *
1738 : * Currently, this will return FALSE only for Windows real filenames. Other
1739 : * VSI virtual filesystems are case sensitive.
1740 : *
1741 : * This methods avoid ugly \#ifndef _WIN32 / \#endif code, that is wrong when
1742 : * dealing with virtual filenames.
1743 : *
1744 : * @param pszFilename the path of the filesystem object to be tested.
1745 : * UTF-8 encoded.
1746 : *
1747 : * @return TRUE if the filenames of the filesystem are case sensitive.
1748 : *
1749 : */
1750 :
1751 28056 : int VSIIsCaseSensitiveFS(const char *pszFilename)
1752 : {
1753 28056 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
1754 :
1755 28056 : return poFSHandler->IsCaseSensitive(pszFilename);
1756 : }
1757 :
1758 : /************************************************************************/
1759 : /* VSISupportsSparseFiles() */
1760 : /************************************************************************/
1761 :
1762 : /**
1763 : * \brief Returns if the filesystem supports sparse files.
1764 : *
1765 : * Only supported on Linux (and no other Unix derivatives) and
1766 : * Windows. On Linux, the answer depends on a few hardcoded
1767 : * signatures for common filesystems. Other filesystems will be
1768 : * considered as not supporting sparse files.
1769 : *
1770 : * @param pszPath the path of the filesystem object to be tested.
1771 : * UTF-8 encoded.
1772 : *
1773 : * @return TRUE if the file system is known to support sparse files. FALSE may
1774 : * be returned both in cases where it is known to not support them,
1775 : * or when it is unknown.
1776 : *
1777 : */
1778 :
1779 2 : int VSISupportsSparseFiles(const char *pszPath)
1780 : {
1781 2 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszPath);
1782 :
1783 2 : return poFSHandler->SupportsSparseFiles(pszPath);
1784 : }
1785 :
1786 : /************************************************************************/
1787 : /* VSIIsLocal() */
1788 : /************************************************************************/
1789 :
1790 : /**
1791 : * \brief Returns if the file/filesystem is "local".
1792 : *
1793 : * The concept of local is mostly by opposition with a network / remote
1794 : * file system whose access time can be long.
1795 : *
1796 : * /vsimem/ is considered to be a local file system, although a non-persistent
1797 : * one.
1798 : *
1799 : * @param pszPath the path of the filesystem object to be tested.
1800 : * UTF-8 encoded.
1801 : *
1802 : * @return TRUE or FALSE
1803 : *
1804 : * @since GDAL 3.6
1805 : */
1806 :
1807 270 : bool VSIIsLocal(const char *pszPath)
1808 : {
1809 270 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszPath);
1810 :
1811 270 : return poFSHandler->IsLocal(pszPath);
1812 : }
1813 :
1814 : /************************************************************************/
1815 : /* VSIGetCanonicalFilename() */
1816 : /************************************************************************/
1817 :
1818 : /**
1819 : * \brief Returns the canonical filename.
1820 : *
1821 : * May be implemented by case-insensitive filesystems
1822 : * (currently Win32 and MacOSX) to return the filename with its actual case
1823 : * (i.e. the one that would be used when listing the content of the directory).
1824 : *
1825 : * @param pszPath UTF-8 encoded path
1826 : *
1827 : * @return UTF-8 encoded string, to free with VSIFree()
1828 : *
1829 : * @since GDAL 3.8
1830 : */
1831 :
1832 252 : char *VSIGetCanonicalFilename(const char *pszPath)
1833 : {
1834 252 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszPath);
1835 :
1836 252 : return CPLStrdup(poFSHandler->GetCanonicalFilename(pszPath).c_str());
1837 : }
1838 :
1839 : /************************************************************************/
1840 : /* VSISupportsSequentialWrite() */
1841 : /************************************************************************/
1842 :
1843 : /**
1844 : * \brief Returns if the filesystem supports sequential write.
1845 : *
1846 : * @param pszPath the path of the filesystem object to be tested.
1847 : * UTF-8 encoded.
1848 : * @param bAllowLocalTempFile whether the file system is allowed to use a
1849 : * local temporary file before uploading to the target location.
1850 : *
1851 : * @return TRUE or FALSE
1852 : *
1853 : * @since GDAL 3.6
1854 : */
1855 :
1856 120 : bool VSISupportsSequentialWrite(const char *pszPath, bool bAllowLocalTempFile)
1857 : {
1858 120 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszPath);
1859 :
1860 120 : return poFSHandler->SupportsSequentialWrite(pszPath, bAllowLocalTempFile);
1861 : }
1862 :
1863 : /************************************************************************/
1864 : /* VSISupportsRandomWrite() */
1865 : /************************************************************************/
1866 :
1867 : /**
1868 : * \brief Returns if the filesystem supports random write.
1869 : *
1870 : * @param pszPath the path of the filesystem object to be tested.
1871 : * UTF-8 encoded.
1872 : * @param bAllowLocalTempFile whether the file system is allowed to use a
1873 : * local temporary file before uploading to the target location.
1874 : *
1875 : * @return TRUE or FALSE
1876 : *
1877 : * @since GDAL 3.6
1878 : */
1879 :
1880 332 : bool VSISupportsRandomWrite(const char *pszPath, bool bAllowLocalTempFile)
1881 : {
1882 332 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszPath);
1883 :
1884 332 : return poFSHandler->SupportsRandomWrite(pszPath, bAllowLocalTempFile);
1885 : }
1886 :
1887 : /************************************************************************/
1888 : /* VSIHasOptimizedReadMultiRange() */
1889 : /************************************************************************/
1890 :
1891 : /**
1892 : * \brief Returns if the filesystem supports efficient multi-range reading.
1893 : *
1894 : * Currently only returns TRUE for /vsicurl/ and derived file systems.
1895 : *
1896 : * @param pszPath the path of the filesystem object to be tested.
1897 : * UTF-8 encoded.
1898 : *
1899 : * @return TRUE if the file system is known to have an efficient multi-range
1900 : * reading.
1901 : *
1902 : */
1903 :
1904 15476 : int VSIHasOptimizedReadMultiRange(const char *pszPath)
1905 : {
1906 15476 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszPath);
1907 :
1908 15475 : return poFSHandler->HasOptimizedReadMultiRange(pszPath);
1909 : }
1910 :
1911 : /************************************************************************/
1912 : /* VSIGetActualURL() */
1913 : /************************************************************************/
1914 :
1915 : /**
1916 : * \brief Returns the actual URL of a supplied filename.
1917 : *
1918 : * Currently only returns a non-NULL value for network-based virtual file
1919 : * systems. For example "/vsis3/bucket/filename" will be expanded as
1920 : * "https://bucket.s3.amazon.com/filename"
1921 : *
1922 : * Note that the lifetime of the returned string, is short, and may be
1923 : * invalidated by any following GDAL functions.
1924 : *
1925 : * @param pszFilename the path of the filesystem object. UTF-8 encoded.
1926 : *
1927 : * @return the actual URL corresponding to the supplied filename, or NULL.
1928 : * Should not be freed.
1929 : *
1930 : */
1931 :
1932 9 : const char *VSIGetActualURL(const char *pszFilename)
1933 : {
1934 9 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
1935 :
1936 9 : return poFSHandler->GetActualURL(pszFilename);
1937 : }
1938 :
1939 : /************************************************************************/
1940 : /* VSIGetSignedURL() */
1941 : /************************************************************************/
1942 :
1943 : /**
1944 : * \brief Returns a signed URL of a supplied filename.
1945 : *
1946 : * Currently only returns a non-NULL value for /vsis3/, /vsigs/, /vsiaz/ and
1947 : * /vsioss/ For example "/vsis3/bucket/filename" will be expanded as
1948 : * "https://bucket.s3.amazon.com/filename?X-Amz-Algorithm=AWS4-HMAC-SHA256..."
1949 : * Configuration options that apply for file opening (typically to provide
1950 : * credentials), and are returned by VSIGetFileSystemOptions(), are also valid
1951 : * in that context.
1952 : *
1953 : * @param pszFilename the path of the filesystem object. UTF-8 encoded.
1954 : * @param papszOptions list of options, or NULL. Depend on file system handler.
1955 : * For /vsis3/, /vsigs/, /vsiaz/ and /vsioss/, the following options are
1956 : * supported: <ul> <li>START_DATE=YYMMDDTHHMMSSZ: date and time in UTC following
1957 : * ISO 8601 standard, corresponding to the start of validity of the URL. If not
1958 : * specified, current date time.</li> <li>EXPIRATION_DELAY=number_of_seconds:
1959 : * number between 1 and 604800 (seven days) for the validity of the signed URL.
1960 : * Defaults to 3600 (one hour)</li> <li>VERB=GET/HEAD/DELETE/PUT/POST: HTTP VERB
1961 : * for which the request will be used. Default to GET.</li>
1962 : * </ul>
1963 : *
1964 : * /vsiaz/ supports additional options:
1965 : * <ul>
1966 : * <li>SIGNEDIDENTIFIER=value: to relate the given shared access signature
1967 : * to a corresponding stored access policy.</li>
1968 : * <li>SIGNEDPERMISSIONS=r|w: permissions associated with the shared access
1969 : * signature. Normally deduced from VERB.</li>
1970 : * </ul>
1971 : *
1972 : * @return a signed URL, or NULL. Should be freed with CPLFree().
1973 : */
1974 :
1975 25 : char *VSIGetSignedURL(const char *pszFilename, CSLConstList papszOptions)
1976 : {
1977 25 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
1978 :
1979 25 : return poFSHandler->GetSignedURL(pszFilename, papszOptions);
1980 : }
1981 :
1982 : /************************************************************************/
1983 : /* VSIFOpenL() */
1984 : /************************************************************************/
1985 :
1986 : /**
1987 : * \brief Open file.
1988 : *
1989 : * This function opens a file with the desired access. Large files (larger
1990 : * than 2GB) should be supported. Binary access is always implied and
1991 : * the "b" does not need to be included in the pszAccess string.
1992 : *
1993 : * *NOT* a standard C library FILE *, and cannot be used with any functions
1994 : * other than the "VSI*L" family of functions. They aren't "real" FILE objects.
1995 : *
1996 : * On windows it is possible to define the configuration option
1997 : * GDAL_FILE_IS_UTF8 to have pszFilename treated as being in the local
1998 : * encoding instead of UTF-8, restoring the pre-1.8.0 behavior of VSIFOpenL().
1999 : *
2000 : * This method goes through the VSIFileHandler virtualization and may
2001 : * work on unusual filesystems such as in memory.
2002 : *
2003 : * Analog of the POSIX fopen() function.
2004 : *
2005 : * @param pszFilename the file to open. UTF-8 encoded.
2006 : * @param pszAccess access requested (i.e. "r", "r+", "w")
2007 : *
2008 : * @return NULL on failure, or the file handle.
2009 : */
2010 :
2011 232393 : VSILFILE *VSIFOpenL(const char *pszFilename, const char *pszAccess)
2012 :
2013 : {
2014 232393 : return VSIFOpenExL(pszFilename, pszAccess, false);
2015 : }
2016 :
2017 : /************************************************************************/
2018 : /* Open() */
2019 : /************************************************************************/
2020 :
2021 : #ifndef DOXYGEN_SKIP
2022 :
2023 : VSIVirtualHandleUniquePtr
2024 16419 : VSIFilesystemHandler::OpenStatic(const char *pszFilename, const char *pszAccess,
2025 : bool bSetError, CSLConstList papszOptions)
2026 : {
2027 16419 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
2028 :
2029 16419 : return poFSHandler->Open(pszFilename, pszAccess, bSetError, papszOptions);
2030 : }
2031 :
2032 : /************************************************************************/
2033 : /* CopyFile() */
2034 : /************************************************************************/
2035 :
2036 2274 : int VSIFilesystemHandler::CopyFile(const char *pszSource, const char *pszTarget,
2037 : VSILFILE *fpSource, vsi_l_offset nSourceSize,
2038 : CSLConstList papszOptions,
2039 : GDALProgressFunc pProgressFunc,
2040 : void *pProgressData)
2041 : {
2042 2274 : VSIVirtualHandleUniquePtr poFileHandleAutoClose;
2043 2274 : if (!fpSource)
2044 : {
2045 2257 : CPLAssert(pszSource);
2046 2257 : fpSource = VSIFOpenExL(pszSource, "rb", TRUE);
2047 2257 : if (!fpSource)
2048 : {
2049 1 : CPLError(CE_Failure, CPLE_FileIO, "Cannot open %s", pszSource);
2050 1 : return -1;
2051 : }
2052 2256 : poFileHandleAutoClose.reset(fpSource);
2053 : }
2054 2273 : if (nSourceSize == static_cast<vsi_l_offset>(-1) &&
2055 3 : pProgressFunc != nullptr && pszSource != nullptr)
2056 : {
2057 : VSIStatBufL sStat;
2058 3 : if (VSIStatL(pszSource, &sStat) == 0)
2059 : {
2060 3 : nSourceSize = sStat.st_size;
2061 : }
2062 : }
2063 :
2064 2273 : VSILFILE *fpOut = VSIFOpenEx2L(pszTarget, "wb", TRUE, papszOptions);
2065 2273 : if (!fpOut)
2066 : {
2067 3 : CPLError(CE_Failure, CPLE_FileIO, "Cannot create %s", pszTarget);
2068 3 : return -1;
2069 : }
2070 :
2071 4540 : CPLString osMsg;
2072 2270 : if (pszSource)
2073 2268 : osMsg.Printf("Copying of %s", pszSource);
2074 : else
2075 2 : pszSource = "(unknown filename)";
2076 :
2077 2270 : int ret = 0;
2078 2270 : constexpr size_t nBufferSize = 10 * 4096;
2079 2270 : std::vector<GByte> abyBuffer(nBufferSize, 0);
2080 2270 : GUIntBig nOffset = 0;
2081 : while (true)
2082 : {
2083 2572 : const size_t nRead = VSIFReadL(&abyBuffer[0], 1, nBufferSize, fpSource);
2084 2572 : if (nRead < nBufferSize && VSIFErrorL(fpSource))
2085 : {
2086 7 : CPLError(
2087 : CE_Failure, CPLE_FileIO,
2088 : "Copying of %s to %s failed: error while reading source file",
2089 : pszSource, pszTarget);
2090 7 : ret = -1;
2091 7 : break;
2092 : }
2093 2565 : if (nRead > 0)
2094 : {
2095 2564 : const size_t nWritten = VSIFWriteL(&abyBuffer[0], 1, nRead, fpOut);
2096 2564 : if (nWritten != nRead)
2097 : {
2098 20 : CPLError(CE_Failure, CPLE_FileIO,
2099 : "Copying of %s to %s failed: error while writing into "
2100 : "target file",
2101 : pszSource, pszTarget);
2102 20 : ret = -1;
2103 20 : break;
2104 : }
2105 2544 : nOffset += nRead;
2106 2593 : if (pProgressFunc &&
2107 98 : !pProgressFunc(
2108 : nSourceSize == 0 ? 1.0
2109 49 : : nSourceSize > 0 &&
2110 : nSourceSize != static_cast<vsi_l_offset>(-1)
2111 98 : ? double(nOffset) / nSourceSize
2112 : : 0.0,
2113 49 : !osMsg.empty() ? osMsg.c_str() : nullptr, pProgressData))
2114 : {
2115 1 : ret = -1;
2116 1 : break;
2117 : }
2118 : }
2119 2544 : if (nRead < nBufferSize)
2120 : {
2121 2242 : break;
2122 : }
2123 302 : }
2124 :
2125 2270 : if (nSourceSize != static_cast<vsi_l_offset>(-1) && nOffset != nSourceSize)
2126 : {
2127 2 : CPLError(CE_Failure, CPLE_FileIO,
2128 : "Copying of %s to %s failed: %" PRIu64 " bytes were copied "
2129 : "whereas %" PRIu64 " were expected",
2130 : pszSource, pszTarget, static_cast<uint64_t>(nOffset),
2131 : static_cast<uint64_t>(nSourceSize));
2132 2 : ret = -1;
2133 : }
2134 :
2135 2270 : if (VSIFCloseL(fpOut) != 0)
2136 : {
2137 1 : ret = -1;
2138 : }
2139 :
2140 2270 : if (ret != 0)
2141 29 : VSIUnlink(pszTarget);
2142 :
2143 2270 : return ret;
2144 : }
2145 :
2146 : /************************************************************************/
2147 : /* CopyFileRestartable() */
2148 : /************************************************************************/
2149 :
2150 2 : int VSIFilesystemHandler::CopyFileRestartable(
2151 : const char *pszSource, const char *pszTarget,
2152 : const char * /* pszInputPayload */, char **ppszOutputPayload,
2153 : CSLConstList papszOptions, GDALProgressFunc pProgressFunc,
2154 : void *pProgressData)
2155 : {
2156 2 : *ppszOutputPayload = nullptr;
2157 2 : return CopyFile(pszSource, pszTarget, nullptr,
2158 : static_cast<vsi_l_offset>(-1), papszOptions, pProgressFunc,
2159 2 : pProgressData);
2160 : }
2161 :
2162 : /************************************************************************/
2163 : /* Sync() */
2164 : /************************************************************************/
2165 :
2166 31 : bool VSIFilesystemHandler::Sync(const char *pszSource, const char *pszTarget,
2167 : const char *const *papszOptions,
2168 : GDALProgressFunc pProgressFunc,
2169 : void *pProgressData, char ***ppapszOutputs)
2170 : {
2171 31 : const char SOURCE_SEP = VSIGetDirectorySeparator(pszSource)[0];
2172 :
2173 31 : if (ppapszOutputs)
2174 : {
2175 0 : *ppapszOutputs = nullptr;
2176 : }
2177 :
2178 : VSIStatBufL sSource;
2179 62 : CPLString osSource(pszSource);
2180 62 : CPLString osSourceWithoutSlash(pszSource);
2181 40 : if (osSourceWithoutSlash.back() == '/' ||
2182 9 : osSourceWithoutSlash.back() == '\\')
2183 : {
2184 22 : osSourceWithoutSlash.pop_back();
2185 : }
2186 31 : if (VSIStatL(osSourceWithoutSlash, &sSource) < 0)
2187 : {
2188 2 : CPLError(CE_Failure, CPLE_FileIO, "%s does not exist", pszSource);
2189 2 : return false;
2190 : }
2191 :
2192 29 : if (VSI_ISDIR(sSource.st_mode))
2193 : {
2194 22 : std::string osTargetDir(pszTarget);
2195 11 : if (osSource.back() != '/' && osSource.back() != '\\')
2196 : {
2197 2 : osTargetDir = CPLFormFilenameSafe(
2198 1 : osTargetDir.c_str(), CPLGetFilename(pszSource), nullptr);
2199 : }
2200 :
2201 : VSIStatBufL sTarget;
2202 11 : bool ret = true;
2203 11 : if (VSIStatL(osTargetDir.c_str(), &sTarget) < 0)
2204 : {
2205 9 : if (VSIMkdirRecursive(osTargetDir.c_str(), 0755) < 0)
2206 : {
2207 1 : CPLError(CE_Failure, CPLE_FileIO, "Cannot create directory %s",
2208 : osTargetDir.c_str());
2209 1 : return false;
2210 : }
2211 : }
2212 :
2213 10 : if (!CPLFetchBool(papszOptions, "STOP_ON_DIR", false))
2214 : {
2215 20 : CPLStringList aosChildOptions(CSLDuplicate(papszOptions));
2216 10 : if (!CPLFetchBool(papszOptions, "RECURSIVE", true))
2217 : {
2218 0 : aosChildOptions.SetNameValue("RECURSIVE", nullptr);
2219 0 : aosChildOptions.AddString("STOP_ON_DIR=TRUE");
2220 : }
2221 :
2222 10 : char **papszSrcFiles = VSIReadDir(osSourceWithoutSlash);
2223 10 : int nFileCount = 0;
2224 27 : for (auto iter = papszSrcFiles; iter && *iter; ++iter)
2225 : {
2226 17 : if (strcmp(*iter, ".") != 0 && strcmp(*iter, "..") != 0)
2227 : {
2228 17 : nFileCount++;
2229 : }
2230 : }
2231 10 : int iFile = 0;
2232 10 : const int nDenom = std::max(1, nFileCount);
2233 27 : for (auto iter = papszSrcFiles; iter && *iter; ++iter, ++iFile)
2234 : {
2235 17 : if (strcmp(*iter, ".") == 0 || strcmp(*iter, "..") == 0)
2236 : {
2237 0 : continue;
2238 : }
2239 : const std::string osSubSource(CPLFormFilenameSafe(
2240 17 : osSourceWithoutSlash.c_str(), *iter, nullptr));
2241 : const std::string osSubTarget(
2242 17 : CPLFormFilenameSafe(osTargetDir.c_str(), *iter, nullptr));
2243 34 : void *pScaledProgress = GDALCreateScaledProgress(
2244 17 : double(iFile) / nDenom, double(iFile + 1) / nDenom,
2245 : pProgressFunc, pProgressData);
2246 17 : ret = Sync((osSubSource + SOURCE_SEP).c_str(),
2247 17 : osSubTarget.c_str(), aosChildOptions.List(),
2248 17 : GDALScaledProgress, pScaledProgress, nullptr);
2249 17 : GDALDestroyScaledProgress(pScaledProgress);
2250 17 : if (!ret)
2251 : {
2252 0 : break;
2253 : }
2254 : }
2255 10 : CSLDestroy(papszSrcFiles);
2256 : }
2257 10 : return ret;
2258 : }
2259 :
2260 : VSIStatBufL sTarget;
2261 36 : std::string osTarget(pszTarget);
2262 18 : if (VSIStatL(osTarget.c_str(), &sTarget) == 0)
2263 : {
2264 4 : bool bTargetIsFile = true;
2265 4 : if (VSI_ISDIR(sTarget.st_mode))
2266 : {
2267 4 : osTarget = CPLFormFilenameSafe(osTarget.c_str(),
2268 2 : CPLGetFilename(pszSource), nullptr);
2269 3 : bTargetIsFile = VSIStatL(osTarget.c_str(), &sTarget) == 0 &&
2270 1 : !CPL_TO_BOOL(VSI_ISDIR(sTarget.st_mode));
2271 : }
2272 4 : if (bTargetIsFile)
2273 : {
2274 3 : if (sSource.st_size == sTarget.st_size &&
2275 3 : sSource.st_mtime == sTarget.st_mtime && sSource.st_mtime != 0)
2276 : {
2277 2 : CPLDebug("VSI",
2278 : "%s and %s have same size and modification "
2279 : "date. Skipping copying",
2280 : osSourceWithoutSlash.c_str(), osTarget.c_str());
2281 2 : return true;
2282 : }
2283 : }
2284 : }
2285 :
2286 16 : VSILFILE *fpIn = VSIFOpenExL(osSourceWithoutSlash, "rb", TRUE);
2287 16 : if (fpIn == nullptr)
2288 : {
2289 0 : CPLError(CE_Failure, CPLE_FileIO, "Cannot open %s",
2290 : osSourceWithoutSlash.c_str());
2291 0 : return false;
2292 : }
2293 :
2294 16 : VSILFILE *fpOut = VSIFOpenExL(osTarget.c_str(), "wb", TRUE);
2295 16 : if (fpOut == nullptr)
2296 : {
2297 2 : CPLError(CE_Failure, CPLE_FileIO, "Cannot create %s", osTarget.c_str());
2298 2 : VSIFCloseL(fpIn);
2299 2 : return false;
2300 : }
2301 :
2302 14 : bool ret = true;
2303 14 : constexpr size_t nBufferSize = 10 * 4096;
2304 28 : std::vector<GByte> abyBuffer(nBufferSize, 0);
2305 14 : GUIntBig nOffset = 0;
2306 14 : CPLString osMsg;
2307 14 : osMsg.Printf("Copying of %s", osSourceWithoutSlash.c_str());
2308 : while (true)
2309 : {
2310 17 : size_t nRead = VSIFReadL(&abyBuffer[0], 1, nBufferSize, fpIn);
2311 17 : size_t nWritten = VSIFWriteL(&abyBuffer[0], 1, nRead, fpOut);
2312 17 : if (nWritten != nRead)
2313 : {
2314 0 : CPLError(CE_Failure, CPLE_FileIO, "Copying of %s to %s failed",
2315 : osSourceWithoutSlash.c_str(), osTarget.c_str());
2316 0 : ret = false;
2317 0 : break;
2318 : }
2319 17 : nOffset += nRead;
2320 17 : if (pProgressFunc && !pProgressFunc(double(nOffset) / sSource.st_size,
2321 : osMsg.c_str(), pProgressData))
2322 : {
2323 0 : ret = false;
2324 0 : break;
2325 : }
2326 17 : if (nRead < nBufferSize)
2327 : {
2328 14 : break;
2329 : }
2330 3 : }
2331 :
2332 14 : VSIFCloseL(fpIn);
2333 14 : if (VSIFCloseL(fpOut) != 0)
2334 : {
2335 0 : ret = false;
2336 : }
2337 14 : return ret;
2338 : }
2339 :
2340 : /************************************************************************/
2341 : /* VSIVirtualHandleOnlyVisibleAtCloseTime() */
2342 : /************************************************************************/
2343 :
2344 : class VSIVirtualHandleOnlyVisibleAtCloseTime final : public VSIProxyFileHandle
2345 : {
2346 : const std::string m_osTargetName;
2347 : const std::string m_osTmpName;
2348 : bool m_bAlreadyClosed = false;
2349 : bool m_bCancelCreation = false;
2350 :
2351 : public:
2352 199 : VSIVirtualHandleOnlyVisibleAtCloseTime(
2353 : VSIVirtualHandleUniquePtr &&nativeHandle,
2354 : const std::string &osTargetName, const std::string &osTmpName)
2355 199 : : VSIProxyFileHandle(std::move(nativeHandle)),
2356 199 : m_osTargetName(osTargetName), m_osTmpName(osTmpName)
2357 : {
2358 199 : }
2359 :
2360 400 : ~VSIVirtualHandleOnlyVisibleAtCloseTime() override
2361 200 : {
2362 200 : VSIVirtualHandleOnlyVisibleAtCloseTime::Close();
2363 400 : }
2364 :
2365 8 : void CancelCreation() override
2366 : {
2367 8 : VSIProxyFileHandle::CancelCreation();
2368 8 : m_bCancelCreation = true;
2369 8 : }
2370 :
2371 : int Close() override;
2372 : };
2373 :
2374 569 : int VSIVirtualHandleOnlyVisibleAtCloseTime::Close()
2375 : {
2376 569 : if (m_bAlreadyClosed)
2377 369 : return 0;
2378 200 : m_bAlreadyClosed = true;
2379 200 : int ret = VSIProxyFileHandle::Close();
2380 200 : if (ret == 0)
2381 : {
2382 200 : if (m_bCancelCreation)
2383 : {
2384 5 : ret = VSIUnlink(m_osTmpName.c_str());
2385 : VSIStatBufL sStatBuf;
2386 5 : if (ret != 0 && VSIStatL(m_osTmpName.c_str(), &sStatBuf) != 0)
2387 4 : ret = 0;
2388 : }
2389 : else
2390 : {
2391 195 : ret = VSIRename(m_osTmpName.c_str(), m_osTargetName.c_str());
2392 : }
2393 : }
2394 200 : return ret;
2395 : }
2396 :
2397 : /************************************************************************/
2398 : /* CreateOnlyVisibleAtCloseTime() */
2399 : /************************************************************************/
2400 :
2401 200 : VSIVirtualHandleUniquePtr VSIFilesystemHandler::CreateOnlyVisibleAtCloseTime(
2402 : const char *pszFilename, bool bEmulationAllowed, CSLConstList papszOptions)
2403 : {
2404 200 : if (!bEmulationAllowed)
2405 0 : return nullptr;
2406 :
2407 599 : const std::string tmpName = std::string(pszFilename).append(".tmp");
2408 : VSIVirtualHandleUniquePtr nativeHandle(
2409 400 : Open(tmpName.c_str(), "wb+", true, papszOptions));
2410 200 : if (!nativeHandle)
2411 0 : return nullptr;
2412 : return VSIVirtualHandleUniquePtr(
2413 399 : std::make_unique<VSIVirtualHandleOnlyVisibleAtCloseTime>(
2414 199 : std::move(nativeHandle), pszFilename, tmpName)
2415 199 : .release());
2416 : }
2417 :
2418 : /************************************************************************/
2419 : /* VSIDIREntry() */
2420 : /************************************************************************/
2421 :
2422 3947 : VSIDIREntry::VSIDIREntry()
2423 : : pszName(nullptr), nMode(0), nSize(0), nMTime(0), bModeKnown(false),
2424 3947 : bSizeKnown(false), bMTimeKnown(false), papszExtra(nullptr)
2425 : {
2426 3947 : }
2427 :
2428 : /************************************************************************/
2429 : /* VSIDIREntry() */
2430 : /************************************************************************/
2431 :
2432 6 : VSIDIREntry::VSIDIREntry(const VSIDIREntry &other)
2433 6 : : pszName(VSIStrdup(other.pszName)), nMode(other.nMode), nSize(other.nSize),
2434 6 : nMTime(other.nMTime), bModeKnown(other.bModeKnown),
2435 6 : bSizeKnown(other.bSizeKnown), bMTimeKnown(other.bMTimeKnown),
2436 6 : papszExtra(CSLDuplicate(other.papszExtra))
2437 : {
2438 6 : }
2439 :
2440 : /************************************************************************/
2441 : /* ~VSIDIREntry() */
2442 : /************************************************************************/
2443 :
2444 7906 : VSIDIREntry::~VSIDIREntry()
2445 : {
2446 3953 : CPLFree(pszName);
2447 3953 : CSLDestroy(papszExtra);
2448 3953 : }
2449 :
2450 : /************************************************************************/
2451 : /* ~VSIDIR() */
2452 : /************************************************************************/
2453 :
2454 3551 : VSIDIR::~VSIDIR()
2455 : {
2456 3551 : }
2457 :
2458 : /************************************************************************/
2459 : /* VSIDIRGeneric */
2460 : /************************************************************************/
2461 :
2462 : namespace
2463 : {
2464 : struct VSIDIRGeneric : public VSIDIR
2465 : {
2466 : CPLString osRootPath{};
2467 : CPLString osBasePath{};
2468 : char **papszContent = nullptr;
2469 : int nRecurseDepth = 0;
2470 : int nPos = 0;
2471 : VSIDIREntry entry{};
2472 : std::vector<VSIDIRGeneric *> aoStackSubDir{};
2473 : VSIFilesystemHandler *poFS = nullptr;
2474 : std::string m_osFilterPrefix{};
2475 :
2476 2667 : explicit VSIDIRGeneric(VSIFilesystemHandler *poFSIn) : poFS(poFSIn)
2477 : {
2478 2667 : }
2479 :
2480 : ~VSIDIRGeneric() override;
2481 :
2482 : const VSIDIREntry *NextDirEntry() override;
2483 :
2484 : VSIDIRGeneric(const VSIDIRGeneric &) = delete;
2485 : VSIDIRGeneric &operator=(const VSIDIRGeneric &) = delete;
2486 : };
2487 :
2488 : /************************************************************************/
2489 : /* ~VSIDIRGeneric() */
2490 : /************************************************************************/
2491 :
2492 8001 : VSIDIRGeneric::~VSIDIRGeneric()
2493 : {
2494 2674 : while (!aoStackSubDir.empty())
2495 : {
2496 7 : delete aoStackSubDir.back();
2497 7 : aoStackSubDir.pop_back();
2498 : }
2499 2667 : CSLDestroy(papszContent);
2500 5334 : }
2501 :
2502 : } // namespace
2503 :
2504 : /************************************************************************/
2505 : /* OpenDir() */
2506 : /************************************************************************/
2507 :
2508 2677 : VSIDIR *VSIFilesystemHandler::OpenDir(const char *pszPath, int nRecurseDepth,
2509 : const char *const *papszOptions)
2510 : {
2511 2677 : char **papszContent = VSIReadDir(pszPath);
2512 : VSIStatBufL sStatL;
2513 3203 : if (papszContent == nullptr &&
2514 526 : (VSIStatL(pszPath, &sStatL) != 0 || !VSI_ISDIR(sStatL.st_mode)))
2515 : {
2516 10 : return nullptr;
2517 : }
2518 2667 : VSIDIRGeneric *dir = new VSIDIRGeneric(this);
2519 2667 : dir->osRootPath = pszPath;
2520 5334 : if (!dir->osRootPath.empty() &&
2521 2667 : (dir->osRootPath.back() == '/' || dir->osRootPath.back() == '\\'))
2522 15 : dir->osRootPath.pop_back();
2523 2667 : dir->nRecurseDepth = nRecurseDepth;
2524 2667 : dir->papszContent = papszContent;
2525 2667 : dir->m_osFilterPrefix = CSLFetchNameValueDef(papszOptions, "PREFIX", "");
2526 2667 : return dir;
2527 : }
2528 :
2529 : /************************************************************************/
2530 : /* NextDirEntry() */
2531 : /************************************************************************/
2532 :
2533 508378 : const VSIDIREntry *VSIDIRGeneric::NextDirEntry()
2534 : {
2535 508378 : const char SEP = VSIGetDirectorySeparator(osRootPath.c_str())[0];
2536 :
2537 508379 : begin:
2538 508379 : if (VSI_ISDIR(entry.nMode) && nRecurseDepth != 0)
2539 : {
2540 1342 : CPLString osCurFile(osRootPath);
2541 1342 : if (!osCurFile.empty())
2542 1342 : osCurFile += SEP;
2543 1342 : osCurFile += entry.pszName;
2544 : auto subdir =
2545 1342 : static_cast<VSIDIRGeneric *>(poFS->VSIFilesystemHandler::OpenDir(
2546 1342 : osCurFile, nRecurseDepth - 1, nullptr));
2547 1342 : if (subdir)
2548 : {
2549 1342 : subdir->osRootPath = osRootPath;
2550 1342 : subdir->osBasePath = entry.pszName;
2551 1342 : subdir->m_osFilterPrefix = m_osFilterPrefix;
2552 1342 : aoStackSubDir.push_back(subdir);
2553 : }
2554 1342 : entry.nMode = 0;
2555 : }
2556 :
2557 509714 : while (!aoStackSubDir.empty())
2558 : {
2559 503803 : auto l_entry = aoStackSubDir.back()->NextDirEntry();
2560 503803 : if (l_entry)
2561 : {
2562 502468 : return l_entry;
2563 : }
2564 1335 : delete aoStackSubDir.back();
2565 1335 : aoStackSubDir.pop_back();
2566 : }
2567 :
2568 5911 : if (papszContent == nullptr)
2569 : {
2570 516 : return nullptr;
2571 : }
2572 :
2573 : while (true)
2574 : {
2575 5423 : if (!papszContent[nPos])
2576 : {
2577 2134 : return nullptr;
2578 : }
2579 : // Skip . and ..entries
2580 3289 : if (papszContent[nPos][0] == '.' &&
2581 115 : (papszContent[nPos][1] == '\0' ||
2582 115 : (papszContent[nPos][1] == '.' && papszContent[nPos][2] == '\0')))
2583 : {
2584 23 : nPos++;
2585 : }
2586 : else
2587 : {
2588 3266 : CPLFree(entry.pszName);
2589 3266 : CPLString osName(osBasePath);
2590 3266 : if (!osName.empty())
2591 2032 : osName += SEP;
2592 3266 : osName += papszContent[nPos];
2593 3266 : nPos++;
2594 :
2595 3266 : entry.pszName = CPLStrdup(osName);
2596 3266 : entry.nMode = 0;
2597 3266 : CPLString osCurFile(osRootPath);
2598 3266 : if (!osCurFile.empty())
2599 3266 : osCurFile += SEP;
2600 3266 : osCurFile += entry.pszName;
2601 :
2602 6522 : const auto StatFile = [&osCurFile, this]()
2603 : {
2604 : VSIStatBufL sStatL;
2605 3261 : if (VSIStatL(osCurFile, &sStatL) == 0)
2606 : {
2607 3259 : entry.nMode = sStatL.st_mode;
2608 3259 : entry.nSize = sStatL.st_size;
2609 3259 : entry.nMTime = sStatL.st_mtime;
2610 3259 : entry.bModeKnown = true;
2611 3259 : entry.bSizeKnown = true;
2612 3259 : entry.bMTimeKnown = true;
2613 : }
2614 : else
2615 : {
2616 2 : entry.nMode = 0;
2617 2 : entry.nSize = 0;
2618 2 : entry.nMTime = 0;
2619 2 : entry.bModeKnown = false;
2620 2 : entry.bSizeKnown = false;
2621 2 : entry.bMTimeKnown = false;
2622 : }
2623 3261 : };
2624 :
2625 3278 : if (!m_osFilterPrefix.empty() &&
2626 12 : m_osFilterPrefix.size() > osName.size())
2627 : {
2628 6 : if (STARTS_WITH(m_osFilterPrefix.c_str(), osName.c_str()) &&
2629 2 : m_osFilterPrefix[osName.size()] == SEP)
2630 : {
2631 1 : StatFile();
2632 1 : if (VSI_ISDIR(entry.nMode))
2633 : {
2634 1 : goto begin;
2635 : }
2636 : }
2637 3 : continue;
2638 : }
2639 3270 : if (!m_osFilterPrefix.empty() &&
2640 8 : !STARTS_WITH(osName.c_str(), m_osFilterPrefix.c_str()))
2641 : {
2642 2 : continue;
2643 : }
2644 :
2645 3260 : StatFile();
2646 :
2647 3260 : break;
2648 : }
2649 28 : }
2650 :
2651 3260 : return &(entry);
2652 : }
2653 :
2654 : /************************************************************************/
2655 : /* UnlinkBatch() */
2656 : /************************************************************************/
2657 :
2658 1 : int *VSIFilesystemHandler::UnlinkBatch(CSLConstList papszFiles)
2659 : {
2660 : int *panRet =
2661 1 : static_cast<int *>(CPLMalloc(sizeof(int) * CSLCount(papszFiles)));
2662 3 : for (int i = 0; papszFiles && papszFiles[i]; ++i)
2663 : {
2664 2 : panRet[i] = VSIUnlink(papszFiles[i]) == 0;
2665 : }
2666 1 : return panRet;
2667 : }
2668 :
2669 : /************************************************************************/
2670 : /* RmdirRecursive() */
2671 : /************************************************************************/
2672 :
2673 34 : int VSIFilesystemHandler::RmdirRecursive(const char *pszDirname)
2674 : {
2675 68 : CPLString osDirnameWithoutEndSlash(pszDirname);
2676 68 : if (!osDirnameWithoutEndSlash.empty() &&
2677 34 : (osDirnameWithoutEndSlash.back() == '/' ||
2678 34 : osDirnameWithoutEndSlash.back() == '\\'))
2679 : {
2680 0 : osDirnameWithoutEndSlash.pop_back();
2681 : }
2682 :
2683 34 : const char SEP = VSIGetDirectorySeparator(pszDirname)[0];
2684 :
2685 68 : CPLStringList aosOptions;
2686 : auto poDir =
2687 68 : std::unique_ptr<VSIDIR>(OpenDir(pszDirname, -1, aosOptions.List()));
2688 34 : if (!poDir)
2689 8 : return -1;
2690 52 : std::vector<std::string> aosDirs;
2691 : while (true)
2692 : {
2693 141 : auto entry = poDir->NextDirEntry();
2694 141 : if (!entry)
2695 26 : break;
2696 :
2697 230 : CPLString osFilename(osDirnameWithoutEndSlash + SEP + entry->pszName);
2698 115 : if ((entry->nMode & S_IFDIR))
2699 : {
2700 10 : aosDirs.push_back(std::move(osFilename));
2701 : }
2702 : else
2703 : {
2704 105 : if (VSIUnlink(osFilename) != 0)
2705 0 : return -1;
2706 : }
2707 115 : }
2708 :
2709 : // Sort in reverse order, so that inner-most directories are deleted first
2710 26 : std::sort(aosDirs.begin(), aosDirs.end(),
2711 2 : [](const std::string &a, const std::string &b) { return a > b; });
2712 :
2713 36 : for (const auto &osDir : aosDirs)
2714 : {
2715 10 : if (VSIRmdir(osDir.c_str()) != 0)
2716 0 : return -1;
2717 : }
2718 :
2719 26 : return VSIRmdir(pszDirname);
2720 : }
2721 :
2722 : /************************************************************************/
2723 : /* GetFileMetadata() */
2724 : /************************************************************************/
2725 :
2726 13 : char **VSIFilesystemHandler::GetFileMetadata(const char * /* pszFilename*/,
2727 : const char * /*pszDomain*/,
2728 : CSLConstList /*papszOptions*/)
2729 : {
2730 13 : return nullptr;
2731 : }
2732 :
2733 : /************************************************************************/
2734 : /* SetFileMetadata() */
2735 : /************************************************************************/
2736 :
2737 0 : bool VSIFilesystemHandler::SetFileMetadata(const char * /* pszFilename*/,
2738 : CSLConstList /* papszMetadata */,
2739 : const char * /* pszDomain */,
2740 : CSLConstList /* papszOptions */)
2741 : {
2742 0 : CPLError(CE_Failure, CPLE_NotSupported, "SetFileMetadata() not supported");
2743 0 : return false;
2744 : }
2745 :
2746 : #endif
2747 :
2748 : /************************************************************************/
2749 : /* VSIFOpenExL() */
2750 : /************************************************************************/
2751 :
2752 : /**
2753 : * \brief Open/create file.
2754 : *
2755 : * This function opens (or creates) a file with the desired access.
2756 : * Binary access is always implied and
2757 : * the "b" does not need to be included in the pszAccess string.
2758 : *
2759 : * Note that the "VSILFILE *" returned by this function is
2760 : * *NOT* a standard C library FILE *, and cannot be used with any functions
2761 : * other than the "VSI*L" family of functions. They aren't "real" FILE objects.
2762 : *
2763 : * On windows it is possible to define the configuration option
2764 : * GDAL_FILE_IS_UTF8 to have pszFilename treated as being in the local
2765 : * encoding instead of UTF-8, restoring the pre-1.8.0 behavior of VSIFOpenL().
2766 : *
2767 : * This method goes through the VSIFileHandler virtualization and may
2768 : * work on unusual filesystems such as in memory.
2769 : *
2770 : * Analog of the POSIX fopen() function.
2771 : *
2772 : * @param pszFilename the file to open. UTF-8 encoded.
2773 : * @param pszAccess access requested (i.e. "r", "r+", "w")
2774 : * @param bSetError flag determining whether or not this open call
2775 : * should set VSIErrors on failure.
2776 : *
2777 : * @return NULL on failure, or the file handle.
2778 : *
2779 : */
2780 :
2781 400672 : VSILFILE *VSIFOpenExL(const char *pszFilename, const char *pszAccess,
2782 : int bSetError)
2783 :
2784 : {
2785 400672 : return VSIFOpenEx2L(pszFilename, pszAccess, bSetError, nullptr);
2786 : }
2787 :
2788 : /************************************************************************/
2789 : /* VSIFOpenEx2L() */
2790 : /************************************************************************/
2791 :
2792 : /**
2793 : * \brief Open/create file.
2794 : *
2795 : * This function opens (or creates) a file with the desired access.
2796 : * Binary access is always implied and
2797 : * the "b" does not need to be included in the pszAccess string.
2798 : *
2799 : * Note that the "VSILFILE *" returned by this function is
2800 : * *NOT* a standard C library FILE *, and cannot be used with any functions
2801 : * other than the "VSI*L" family of functions. They aren't "real" FILE objects.
2802 : *
2803 : * On windows it is possible to define the configuration option
2804 : * GDAL_FILE_IS_UTF8 to have pszFilename treated as being in the local
2805 : * encoding instead of UTF-8, restoring the pre-1.8.0 behavior of VSIFOpenL().
2806 : *
2807 : * This method goes through the VSIFileHandler virtualization and may
2808 : * work on unusual filesystems such as in memory.
2809 : *
2810 : * The following options are supported:
2811 : * <ul>
2812 : * <li>MIME headers such as Content-Type and Content-Encoding
2813 : * are supported for the /vsis3/, /vsigs/, /vsiaz/, /vsiadls/ file systems.</li>
2814 : * <li>DISABLE_READDIR_ON_OPEN=YES/NO (GDAL >= 3.6) for /vsicurl/ and other
2815 : * network-based file systems. By default, directory file listing is done,
2816 : * unless YES is specified.</li>
2817 : * <li>WRITE_THROUGH=YES (GDAL >= 3.8) for the Windows regular files to
2818 : * set the FILE_FLAG_WRITE_THROUGH flag to the CreateFile() function. In that
2819 : * mode, the data is written to the system cache but is flushed to disk without
2820 : * delay.</li>
2821 : * </ul>
2822 : *
2823 : * Options specifics to /vsis3/, /vsigs/, /vsioss/ and /vsiaz/:
2824 : * <ul>
2825 : * <li>CACHE=YES/NO. (GDAL >= 3.10) Whether file metadata and content that is
2826 : * read from the network should be kept cached in memory, after file handle
2827 : * closing. Default is YES, unless the filename is set in CPL_VSIL_CURL_NON_CACHED.
2828 : * </li>
2829 : * <li>CHUNK_SIZE=val in MiB. (GDAL >= 3.10) For "w" mode. Size of a block. Default is 50 MiB.
2830 : * For /vsis3/, /vsigz/, /vsioss/, it can be up to 5000 MiB.
2831 : * For /vsiaz/, only taken into account when BLOB_TYPE=BLOCK. It can be up to 4000 MiB.
2832 : * </li>
2833 : * </ul>
2834 : *
2835 : * Options specifics to /vsiaz/ in "w" mode:
2836 : * <ul>
2837 : * <li>BLOB_TYPE=APPEND/BLOCK. (GDAL >= 3.10) Type of blob. Defaults to APPEND.
2838 : * Append blocks are limited to 195 GiB
2839 : * (however if the file size is below 4 MiB, a block blob will be created in a
2840 : * single PUT operation)
2841 : * </li>
2842 : * </ul>
2843 : *
2844 : * Analog of the POSIX fopen() function.
2845 : *
2846 : * @param pszFilename the file to open. UTF-8 encoded.
2847 : * @param pszAccess access requested (i.e. "r", "r+", "w")
2848 : * @param bSetError flag determining whether or not this open call
2849 : * should set VSIErrors on failure.
2850 : * @param papszOptions NULL or NULL-terminated list of strings. The content is
2851 : * highly file system dependent.
2852 : *
2853 : *
2854 : * @return NULL on failure, or the file handle.
2855 : *
2856 : * @since GDAL 3.3
2857 : */
2858 :
2859 429899 : VSILFILE *VSIFOpenEx2L(const char *pszFilename, const char *pszAccess,
2860 : int bSetError, CSLConstList papszOptions)
2861 :
2862 : {
2863 : // Too long filenames can cause excessive memory allocation due to
2864 : // recursion in some filesystem handlers
2865 429899 : constexpr size_t knMaxPath = 8192;
2866 429899 : if (CPLStrnlen(pszFilename, knMaxPath) == knMaxPath)
2867 5 : return nullptr;
2868 :
2869 429995 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
2870 :
2871 430012 : auto fp = poFSHandler->Open(pszFilename, pszAccess, CPL_TO_BOOL(bSetError),
2872 859337 : papszOptions);
2873 :
2874 : VSIDebug4("VSIFOpenEx2L(%s,%s,%d) = %p", pszFilename, pszAccess, bSetError,
2875 : fp.get());
2876 :
2877 429763 : return fp.release();
2878 : }
2879 :
2880 : /************************************************************************/
2881 : /* VSIFCloseL() */
2882 : /************************************************************************/
2883 :
2884 : /**
2885 : * \fn VSIVirtualHandle::Close()
2886 : * \brief Close file.
2887 : *
2888 : * This function closes the indicated file.
2889 : *
2890 : * This method goes through the VSIFileHandler virtualization and may
2891 : * work on unusual filesystems such as in memory.
2892 : *
2893 : * Analog of the POSIX fclose() function.
2894 : *
2895 : * @return 0 on success or -1 on failure.
2896 : */
2897 :
2898 : /**
2899 : * \brief Close file.
2900 : *
2901 : * This function closes the indicated file.
2902 : *
2903 : * This method goes through the VSIFileHandler virtualization and may
2904 : * work on unusual filesystems such as in memory.
2905 : *
2906 : * Analog of the POSIX fclose() function.
2907 : *
2908 : * @param fp file handle opened with VSIFOpenL(). Passing a nullptr produces
2909 : * undefined behavior.
2910 : *
2911 : * @return 0 on success or -1 on failure.
2912 : */
2913 :
2914 311346 : int VSIFCloseL(VSILFILE *fp)
2915 :
2916 : {
2917 : VSIDebug1("VSIFCloseL(%p)", fp);
2918 :
2919 311346 : const int nResult = fp->Close();
2920 :
2921 310841 : delete fp;
2922 :
2923 310702 : return nResult;
2924 : }
2925 :
2926 : /************************************************************************/
2927 : /* VSIFSeekL() */
2928 : /************************************************************************/
2929 :
2930 : /**
2931 : * \fn int VSIVirtualHandle::Seek( vsi_l_offset nOffset, int nWhence )
2932 : * \brief Seek to requested offset.
2933 : *
2934 : * Seek to the desired offset (nOffset) in the indicated file.
2935 : *
2936 : * This method goes through the VSIFileHandler virtualization and may
2937 : * work on unusual filesystems such as in memory.
2938 : *
2939 : * Analog of the POSIX fseek() call.
2940 : *
2941 : * Caution: vsi_l_offset is a unsigned type, so SEEK_CUR can only be used
2942 : * for positive seek. If negative seek is needed, use
2943 : * handle->Seek( handle->Tell() + negative_offset, SEEK_SET ).
2944 : *
2945 : * @param nOffset offset in bytes.
2946 : * @param nWhence one of SEEK_SET, SEEK_CUR or SEEK_END.
2947 : *
2948 : * @return 0 on success or -1 one failure.
2949 : */
2950 :
2951 : /**
2952 : * \brief Seek to requested offset.
2953 : *
2954 : * Seek to the desired offset (nOffset) in the indicated file.
2955 : *
2956 : * This method goes through the VSIFileHandler virtualization and may
2957 : * work on unusual filesystems such as in memory.
2958 : *
2959 : * Analog of the POSIX fseek() call.
2960 : *
2961 : * Caution: vsi_l_offset is a unsigned type, so SEEK_CUR can only be used
2962 : * for positive seek. If negative seek is needed, use
2963 : * VSIFSeekL( fp, VSIFTellL(fp) + negative_offset, SEEK_SET ).
2964 : *
2965 : * @param fp file handle opened with VSIFOpenL().
2966 : * @param nOffset offset in bytes.
2967 : * @param nWhence one of SEEK_SET, SEEK_CUR or SEEK_END.
2968 : *
2969 : * @return 0 on success or -1 one failure.
2970 : */
2971 :
2972 9080820 : int VSIFSeekL(VSILFILE *fp, vsi_l_offset nOffset, int nWhence)
2973 :
2974 : {
2975 9080820 : return fp->Seek(nOffset, nWhence);
2976 : }
2977 :
2978 : /************************************************************************/
2979 : /* VSIFTellL() */
2980 : /************************************************************************/
2981 :
2982 : /**
2983 : * \fn VSIVirtualHandle::Tell()
2984 : * \brief Tell current file offset.
2985 : *
2986 : * Returns the current file read/write offset in bytes from the beginning of
2987 : * the file.
2988 : *
2989 : * This method goes through the VSIFileHandler virtualization and may
2990 : * work on unusual filesystems such as in memory.
2991 : *
2992 : * Analog of the POSIX ftell() call.
2993 : *
2994 : * @return file offset in bytes.
2995 : */
2996 :
2997 : /**
2998 : * \brief Tell current file offset.
2999 : *
3000 : * Returns the current file read/write offset in bytes from the beginning of
3001 : * the file.
3002 : *
3003 : * This method goes through the VSIFileHandler virtualization and may
3004 : * work on unusual filesystems such as in memory.
3005 : *
3006 : * Analog of the POSIX ftell() call.
3007 : *
3008 : * @param fp file handle opened with VSIFOpenL().
3009 : *
3010 : * @return file offset in bytes.
3011 : */
3012 :
3013 6284560 : vsi_l_offset VSIFTellL(VSILFILE *fp)
3014 :
3015 : {
3016 6284560 : return fp->Tell();
3017 : }
3018 :
3019 : /************************************************************************/
3020 : /* VSIRewindL() */
3021 : /************************************************************************/
3022 :
3023 : /**
3024 : * \brief Rewind the file pointer to the beginning of the file.
3025 : *
3026 : * This is equivalent to VSIFSeekL( fp, 0, SEEK_SET )
3027 : *
3028 : * Analog of the POSIX rewind() call.
3029 : *
3030 : * @param fp file handle opened with VSIFOpenL().
3031 : */
3032 :
3033 88477 : void VSIRewindL(VSILFILE *fp)
3034 :
3035 : {
3036 88477 : CPL_IGNORE_RET_VAL(VSIFSeekL(fp, 0, SEEK_SET));
3037 88468 : }
3038 :
3039 : /************************************************************************/
3040 : /* VSIFFlushL() */
3041 : /************************************************************************/
3042 :
3043 : /**
3044 : * \fn VSIVirtualHandle::Flush()
3045 : * \brief Flush pending writes to disk.
3046 : *
3047 : * For files in write or update mode and on filesystem types where it is
3048 : * applicable, all pending output on the file is flushed to the physical disk.
3049 : *
3050 : * This method goes through the VSIFileHandler virtualization and may
3051 : * work on unusual filesystems such as in memory.
3052 : *
3053 : * Analog of the POSIX fflush() call.
3054 : *
3055 : * On Windows regular files, this method does nothing, unless the
3056 : * VSI_FLUSH configuration option is set to YES (and only when the file has
3057 : * *not* been opened with the WRITE_THROUGH option).
3058 : *
3059 : * @return 0 on success or -1 on error.
3060 : */
3061 :
3062 : /**
3063 : * \brief Flush pending writes to disk.
3064 : *
3065 : * For files in write or update mode and on filesystem types where it is
3066 : * applicable, all pending output on the file is flushed to the physical disk.
3067 : *
3068 : * This method goes through the VSIFileHandler virtualization and may
3069 : * work on unusual filesystems such as in memory.
3070 : *
3071 : * Analog of the POSIX fflush() call.
3072 : *
3073 : * On Windows regular files, this method does nothing, unless the
3074 : * VSI_FLUSH configuration option is set to YES (and only when the file has
3075 : * *not* been opened with the WRITE_THROUGH option).
3076 : *
3077 : * @param fp file handle opened with VSIFOpenL().
3078 : *
3079 : * @return 0 on success or -1 on error.
3080 : */
3081 :
3082 90945 : int VSIFFlushL(VSILFILE *fp)
3083 :
3084 : {
3085 90945 : return fp->Flush();
3086 : }
3087 :
3088 : /************************************************************************/
3089 : /* VSIFReadL() */
3090 : /************************************************************************/
3091 :
3092 : /**
3093 : * \fn VSIVirtualHandle::Read( void *pBuffer, size_t nSize, size_t nCount )
3094 : * \brief Read bytes from file.
3095 : *
3096 : * Reads nCount objects of nSize bytes from the indicated file at the
3097 : * current offset into the indicated buffer.
3098 : *
3099 : * This method goes through the VSIFileHandler virtualization and may
3100 : * work on unusual filesystems such as in memory.
3101 : *
3102 : * Analog of the POSIX fread() call.
3103 : *
3104 : * @param pBuffer the buffer into which the data should be read (at least
3105 : * nCount * nSize bytes in size.
3106 : * @param nSize size of objects to read in bytes.
3107 : * @param nCount number of objects to read.
3108 : *
3109 : * @return number of objects successfully read. If that number is less than
3110 : * nCount, VSIFEofL() or VSIFErrorL() can be used to determine the reason for
3111 : * the short read.
3112 : */
3113 :
3114 : /**
3115 : * \brief Read bytes from file.
3116 : *
3117 : * Reads nCount objects of nSize bytes from the indicated file at the
3118 : * current offset into the indicated buffer.
3119 : *
3120 : * This method goes through the VSIFileHandler virtualization and may
3121 : * work on unusual filesystems such as in memory.
3122 : *
3123 : * Analog of the POSIX fread() call.
3124 : *
3125 : * @param pBuffer the buffer into which the data should be read (at least
3126 : * nCount * nSize bytes in size.
3127 : * @param nSize size of objects to read in bytes.
3128 : * @param nCount number of objects to read.
3129 : * @param fp file handle opened with VSIFOpenL().
3130 : *
3131 : * @return number of objects successfully read. If that number is less than
3132 : * nCount, VSIFEofL() or VSIFErrorL() can be used to determine the reason for
3133 : * the short read.
3134 : */
3135 :
3136 16310600 : size_t VSIFReadL(void *pBuffer, size_t nSize, size_t nCount, VSILFILE *fp)
3137 :
3138 : {
3139 16310600 : return fp->Read(pBuffer, nSize, nCount);
3140 : }
3141 :
3142 : /************************************************************************/
3143 : /* VSIFReadMultiRangeL() */
3144 : /************************************************************************/
3145 :
3146 : /**
3147 : * \fn VSIVirtualHandle::ReadMultiRange( int nRanges, void ** ppData,
3148 : * const vsi_l_offset* panOffsets,
3149 : * const size_t* panSizes )
3150 : * \brief Read several ranges of bytes from file.
3151 : *
3152 : * Reads nRanges objects of panSizes[i] bytes from the indicated file at the
3153 : * offset panOffsets[i] into the buffer ppData[i].
3154 : *
3155 : * Ranges must be sorted in ascending start offset, and must not overlap each
3156 : * other.
3157 : *
3158 : * This method goes through the VSIFileHandler virtualization and may
3159 : * work on unusual filesystems such as in memory or /vsicurl/.
3160 : *
3161 : * @param nRanges number of ranges to read.
3162 : * @param ppData array of nRanges buffer into which the data should be read
3163 : * (ppData[i] must be at list panSizes[i] bytes).
3164 : * @param panOffsets array of nRanges offsets at which the data should be read.
3165 : * @param panSizes array of nRanges sizes of objects to read (in bytes).
3166 : *
3167 : * @return 0 in case of success, -1 otherwise.
3168 : */
3169 :
3170 : /**
3171 : * \brief Read several ranges of bytes from file.
3172 : *
3173 : * Reads nRanges objects of panSizes[i] bytes from the indicated file at the
3174 : * offset panOffsets[i] into the buffer ppData[i].
3175 : *
3176 : * Ranges must be sorted in ascending start offset, and must not overlap each
3177 : * other.
3178 : *
3179 : * This method goes through the VSIFileHandler virtualization and may
3180 : * work on unusual filesystems such as in memory or /vsicurl/.
3181 : *
3182 : * @param nRanges number of ranges to read.
3183 : * @param ppData array of nRanges buffer into which the data should be read
3184 : * (ppData[i] must be at list panSizes[i] bytes).
3185 : * @param panOffsets array of nRanges offsets at which the data should be read.
3186 : * @param panSizes array of nRanges sizes of objects to read (in bytes).
3187 : * @param fp file handle opened with VSIFOpenL().
3188 : *
3189 : * @return 0 in case of success, -1 otherwise.
3190 : */
3191 :
3192 763 : int VSIFReadMultiRangeL(int nRanges, void **ppData,
3193 : const vsi_l_offset *panOffsets, const size_t *panSizes,
3194 : VSILFILE *fp)
3195 : {
3196 763 : return fp->ReadMultiRange(nRanges, ppData, panOffsets, panSizes);
3197 : }
3198 :
3199 : /************************************************************************/
3200 : /* VSIFWriteL() */
3201 : /************************************************************************/
3202 :
3203 : /**
3204 : * \fn VSIVirtualHandle::Write( const void *pBuffer,
3205 : * size_t nSize, size_t nCount )
3206 : * \brief Write bytes to file.
3207 : *
3208 : * Writes nCount objects of nSize bytes to the indicated file at the
3209 : * current offset into the indicated buffer.
3210 : *
3211 : * This method goes through the VSIFileHandler virtualization and may
3212 : * work on unusual filesystems such as in memory.
3213 : *
3214 : * Analog of the POSIX fwrite() call.
3215 : *
3216 : * @param pBuffer the buffer from which the data should be written (at least
3217 : * nCount * nSize bytes in size.
3218 : * @param nSize size of objects to write in bytes.
3219 : * @param nCount number of objects to write.
3220 : *
3221 : * @return number of objects successfully written.
3222 : */
3223 :
3224 : /**
3225 : * \brief Write bytes to file.
3226 : *
3227 : * Writes nCount objects of nSize bytes to the indicated file at the
3228 : * current offset into the indicated buffer.
3229 : *
3230 : * This method goes through the VSIFileHandler virtualization and may
3231 : * work on unusual filesystems such as in memory.
3232 : *
3233 : * Analog of the POSIX fwrite() call.
3234 : *
3235 : * @param pBuffer the buffer from which the data should be written (at least
3236 : * nCount * nSize bytes in size.
3237 : * @param nSize size of objects to write in bytes.
3238 : * @param nCount number of objects to write.
3239 : * @param fp file handle opened with VSIFOpenL().
3240 : *
3241 : * @return number of objects successfully written.
3242 : */
3243 :
3244 4267840 : size_t VSIFWriteL(const void *pBuffer, size_t nSize, size_t nCount,
3245 : VSILFILE *fp)
3246 :
3247 : {
3248 4267840 : return fp->Write(pBuffer, nSize, nCount);
3249 : }
3250 :
3251 : /************************************************************************/
3252 : /* VSIFEofL() */
3253 : /************************************************************************/
3254 :
3255 : /**
3256 : * \fn VSIVirtualHandle::Eof()
3257 : * \brief Test for end of file.
3258 : *
3259 : * Returns TRUE (non-zero) if an end-of-file condition occurred during the
3260 : * previous read operation. The end-of-file flag is cleared by a successful
3261 : * VSIFSeekL() call, or a call to VSIFClearErrL().
3262 : *
3263 : * This method goes through the VSIFileHandler virtualization and may
3264 : * work on unusual filesystems such as in memory.
3265 : *
3266 : * Analog of the POSIX feof() call.
3267 : *
3268 : * @return TRUE if at EOF, else FALSE.
3269 : */
3270 :
3271 : /**
3272 : * \brief Test for end of file.
3273 : *
3274 : * Returns TRUE (non-zero) if an end-of-file condition occurred during the
3275 : * previous read operation. The end-of-file flag is cleared by a successful
3276 : * VSIFSeekL() call, or a call to VSIFClearErrL().
3277 : *
3278 : * This method goes through the VSIFileHandler virtualization and may
3279 : * work on unusual filesystems such as in memory.
3280 : *
3281 : * Analog of the POSIX feof() call.
3282 : *
3283 : * @param fp file handle opened with VSIFOpenL().
3284 : *
3285 : * @return TRUE if at EOF, else FALSE.
3286 : */
3287 :
3288 282014 : int VSIFEofL(VSILFILE *fp)
3289 :
3290 : {
3291 282014 : return fp->Eof();
3292 : }
3293 :
3294 : /************************************************************************/
3295 : /* VSIFErrorL() */
3296 : /************************************************************************/
3297 :
3298 : /**
3299 : * \fn VSIVirtualHandle::Error()
3300 : * \brief Test the error indicator.
3301 : *
3302 : * Returns TRUE (non-zero) if an error condition occurred during the
3303 : * previous read operation. The error indicator is cleared by a call to
3304 : * VSIFClearErrL(). Note that a end-of-file situation, reported by VSIFEofL(),
3305 : * is *not* an error reported by VSIFErrorL().
3306 : *
3307 : * This method goes through the VSIFileHandler virtualization and may
3308 : * work on unusual filesystems such as in memory.
3309 : *
3310 : * Analog of the POSIX ferror() call.
3311 : *
3312 : * @return TRUE if the error indicator is set, else FALSE.
3313 : * @since 3.10
3314 : */
3315 :
3316 : /**
3317 : * \brief Test the error indicator.
3318 : *
3319 : * Returns TRUE (non-zero) if an error condition occurred during the
3320 : * previous read operation. The error indicator is cleared by a call to
3321 : * VSIFClearErrL(). Note that a end-of-file situation, reported by VSIFEofL(),
3322 : * is *not* an error reported by VSIFErrorL().
3323 : *
3324 : * This method goes through the VSIFileHandler virtualization and may
3325 : * work on unusual filesystems such as in memory.
3326 : *
3327 : * Analog of the POSIX feof() call.
3328 : *
3329 : * @param fp file handle opened with VSIFOpenL().
3330 : *
3331 : * @return TRUE if the error indicator is set, else FALSE.
3332 : * @since 3.10
3333 : */
3334 :
3335 91061 : int VSIFErrorL(VSILFILE *fp)
3336 :
3337 : {
3338 91061 : return fp->Error();
3339 : }
3340 :
3341 : /************************************************************************/
3342 : /* VSIFClearErrL() */
3343 : /************************************************************************/
3344 :
3345 : /**
3346 : * \fn VSIVirtualHandle::ClearErr()
3347 : * \brief Reset the error and end-of-file indicators.
3348 : *
3349 : * This method goes through the VSIFileHandler virtualization and may
3350 : * work on unusual filesystems such as in memory.
3351 : *
3352 : * Analog of the POSIX clearerr() call.
3353 : *
3354 : * @since 3.10
3355 : */
3356 :
3357 : /**
3358 : * \brief Reset the error and end-of-file indicators.
3359 : *
3360 : * This method goes through the VSIFileHandler virtualization and may
3361 : * work on unusual filesystems such as in memory.
3362 : *
3363 : * Analog of the POSIX clearerr() call.
3364 : *
3365 : * @param fp file handle opened with VSIFOpenL().
3366 : *
3367 : * @since 3.10
3368 : */
3369 :
3370 31166 : void VSIFClearErrL(VSILFILE *fp)
3371 :
3372 : {
3373 31166 : fp->ClearErr();
3374 31166 : }
3375 :
3376 : /************************************************************************/
3377 : /* VSIFTruncateL() */
3378 : /************************************************************************/
3379 :
3380 : /**
3381 : * \fn VSIVirtualHandle::Truncate( vsi_l_offset nNewSize )
3382 : * \brief Truncate/expand the file to the specified size
3383 :
3384 : * This method goes through the VSIFileHandler virtualization and may
3385 : * work on unusual filesystems such as in memory.
3386 : *
3387 : * Analog of the POSIX ftruncate() call.
3388 : *
3389 : * @param nNewSize new size in bytes.
3390 : *
3391 : * @return 0 on success
3392 : */
3393 :
3394 : /**
3395 : * \brief Truncate/expand the file to the specified size
3396 :
3397 : * This method goes through the VSIFileHandler virtualization and may
3398 : * work on unusual filesystems such as in memory.
3399 : *
3400 : * Analog of the POSIX ftruncate() call.
3401 : *
3402 : * @param fp file handle opened with VSIFOpenL().
3403 : * @param nNewSize new size in bytes.
3404 : *
3405 : * @return 0 on success
3406 : */
3407 :
3408 1311 : int VSIFTruncateL(VSILFILE *fp, vsi_l_offset nNewSize)
3409 :
3410 : {
3411 1311 : return fp->Truncate(nNewSize);
3412 : }
3413 :
3414 : /************************************************************************/
3415 : /* VSIFPrintfL() */
3416 : /************************************************************************/
3417 :
3418 : /**
3419 : * \brief Formatted write to file.
3420 : *
3421 : * Provides fprintf() style formatted output to a VSI*L file. This formats
3422 : * an internal buffer which is written using VSIFWriteL().
3423 : *
3424 : * Analog of the POSIX fprintf() call.
3425 : *
3426 : * @param fp file handle opened with VSIFOpenL().
3427 : * @param pszFormat the printf() style format string.
3428 : *
3429 : * @return the number of bytes written or -1 on an error.
3430 : */
3431 :
3432 79734 : int VSIFPrintfL(VSILFILE *fp, CPL_FORMAT_STRING(const char *pszFormat), ...)
3433 :
3434 : {
3435 : va_list args;
3436 :
3437 79734 : va_start(args, pszFormat);
3438 79734 : CPLString osResult;
3439 79734 : osResult.vPrintf(pszFormat, args);
3440 79734 : va_end(args);
3441 :
3442 : return static_cast<int>(
3443 159468 : VSIFWriteL(osResult.c_str(), 1, osResult.length(), fp));
3444 : }
3445 :
3446 : /************************************************************************/
3447 : /* VSIVirtualHandle::Printf() */
3448 : /************************************************************************/
3449 :
3450 : /**
3451 : * \brief Formatted write to file.
3452 : *
3453 : * Provides fprintf() style formatted output to a VSI*L file. This formats
3454 : * an internal buffer which is written using VSIFWriteL().
3455 : *
3456 : * Analog of the POSIX fprintf() call.
3457 : *
3458 : * @param pszFormat the printf() style format string.
3459 : *
3460 : * @return the number of bytes written or -1 on an error.
3461 : */
3462 :
3463 2601 : int VSIVirtualHandle::Printf(CPL_FORMAT_STRING(const char *pszFormat), ...)
3464 : {
3465 : va_list args;
3466 :
3467 2601 : va_start(args, pszFormat);
3468 2601 : CPLString osResult;
3469 2601 : osResult.vPrintf(pszFormat, args);
3470 2601 : va_end(args);
3471 :
3472 5202 : return static_cast<int>(Write(osResult.c_str(), 1, osResult.length()));
3473 : }
3474 :
3475 : /************************************************************************/
3476 : /* VSIFPutcL() */
3477 : /************************************************************************/
3478 :
3479 : // TODO: should we put in conformance with POSIX regarding the return
3480 : // value. As of today (2015-08-29), no code in GDAL sources actually
3481 : // check the return value.
3482 :
3483 : /**
3484 : * \brief Write a single byte to the file
3485 : *
3486 : * Writes the character nChar, cast to an unsigned char, to file.
3487 : *
3488 : * Almost an analog of the POSIX fputc() call, except that it returns
3489 : * the number of character written (1 or 0), and not the (cast)
3490 : * character itself or EOF.
3491 : *
3492 : * @param nChar character to write.
3493 : * @param fp file handle opened with VSIFOpenL().
3494 : *
3495 : * @return 1 in case of success, 0 on error.
3496 : */
3497 :
3498 509 : int VSIFPutcL(int nChar, VSILFILE *fp)
3499 :
3500 : {
3501 509 : const unsigned char cChar = static_cast<unsigned char>(nChar);
3502 509 : return static_cast<int>(VSIFWriteL(&cChar, 1, 1, fp));
3503 : }
3504 :
3505 : /************************************************************************/
3506 : /* VSIFGetRangeStatusL() */
3507 : /************************************************************************/
3508 :
3509 : /**
3510 : * \fn VSIVirtualHandle::GetRangeStatus( vsi_l_offset nOffset,
3511 : * vsi_l_offset nLength )
3512 : * \brief Return if a given file range contains data or holes filled with zeroes
3513 : *
3514 : * This uses the filesystem capabilities of querying which regions of
3515 : * a sparse file are allocated or not. This is currently only
3516 : * implemented for Linux (and no other Unix derivatives) and Windows.
3517 : *
3518 : * Note: A return of VSI_RANGE_STATUS_DATA doesn't exclude that the
3519 : * extent is filled with zeroes! It must be interpreted as "may
3520 : * contain non-zero data".
3521 : *
3522 : * @param nOffset offset of the start of the extent.
3523 : * @param nLength extent length.
3524 : *
3525 : * @return extent status: VSI_RANGE_STATUS_UNKNOWN, VSI_RANGE_STATUS_DATA or
3526 : * VSI_RANGE_STATUS_HOLE
3527 : */
3528 :
3529 : /**
3530 : * \brief Return if a given file range contains data or holes filled with zeroes
3531 : *
3532 : * This uses the filesystem capabilities of querying which regions of
3533 : * a sparse file are allocated or not. This is currently only
3534 : * implemented for Linux (and no other Unix derivatives) and Windows.
3535 : *
3536 : * Note: A return of VSI_RANGE_STATUS_DATA doesn't exclude that the
3537 : * extent is filled with zeroes! It must be interpreted as "may
3538 : * contain non-zero data".
3539 : *
3540 : * @param fp file handle opened with VSIFOpenL().
3541 : * @param nOffset offset of the start of the extent.
3542 : * @param nLength extent length.
3543 : *
3544 : * @return extent status: VSI_RANGE_STATUS_UNKNOWN, VSI_RANGE_STATUS_DATA or
3545 : * VSI_RANGE_STATUS_HOLE
3546 : */
3547 :
3548 690 : VSIRangeStatus VSIFGetRangeStatusL(VSILFILE *fp, vsi_l_offset nOffset,
3549 : vsi_l_offset nLength)
3550 : {
3551 690 : return fp->GetRangeStatus(nOffset, nLength);
3552 : }
3553 :
3554 : /************************************************************************/
3555 : /* VSIIngestFile() */
3556 : /************************************************************************/
3557 :
3558 : /**
3559 : * \brief Ingest a file into memory.
3560 : *
3561 : * Read the whole content of a file into a memory buffer.
3562 : *
3563 : * Either fp or pszFilename can be NULL, but not both at the same time.
3564 : *
3565 : * If fp is passed non-NULL, it is the responsibility of the caller to
3566 : * close it.
3567 : *
3568 : * If non-NULL, the returned buffer is guaranteed to be NUL-terminated.
3569 : *
3570 : * @param fp file handle opened with VSIFOpenL().
3571 : * @param pszFilename filename.
3572 : * @param ppabyRet pointer to the target buffer. *ppabyRet must be freed with
3573 : * VSIFree()
3574 : * @param pnSize pointer to variable to store the file size. May be NULL.
3575 : * @param nMaxSize maximum size of file allowed. If no limit, set to a negative
3576 : * value.
3577 : *
3578 : * @return TRUE in case of success.
3579 : *
3580 : */
3581 :
3582 13108 : int VSIIngestFile(VSILFILE *fp, const char *pszFilename, GByte **ppabyRet,
3583 : vsi_l_offset *pnSize, GIntBig nMaxSize)
3584 : {
3585 13108 : if (fp == nullptr && pszFilename == nullptr)
3586 0 : return FALSE;
3587 13108 : if (ppabyRet == nullptr)
3588 0 : return FALSE;
3589 :
3590 13108 : *ppabyRet = nullptr;
3591 13108 : if (pnSize != nullptr)
3592 6341 : *pnSize = 0;
3593 :
3594 13108 : bool bFreeFP = false;
3595 13108 : if (nullptr == fp)
3596 : {
3597 11830 : fp = VSIFOpenL(pszFilename, "rb");
3598 11830 : if (nullptr == fp)
3599 : {
3600 937 : CPLError(CE_Failure, CPLE_FileIO, "Cannot open file '%s'",
3601 : pszFilename);
3602 937 : return FALSE;
3603 : }
3604 10893 : bFreeFP = true;
3605 : }
3606 : else
3607 : {
3608 1278 : if (VSIFSeekL(fp, 0, SEEK_SET) != 0)
3609 0 : return FALSE;
3610 : }
3611 :
3612 12171 : vsi_l_offset nDataLen = 0;
3613 :
3614 12171 : if (pszFilename == nullptr || strcmp(pszFilename, "/vsistdin/") == 0)
3615 : {
3616 100 : vsi_l_offset nDataAlloc = 0;
3617 100 : if (VSIFSeekL(fp, 0, SEEK_SET) != 0)
3618 : {
3619 0 : if (bFreeFP)
3620 0 : CPL_IGNORE_RET_VAL(VSIFCloseL(fp));
3621 0 : return FALSE;
3622 : }
3623 : while (true)
3624 : {
3625 507 : if (nDataLen + 8192 + 1 > nDataAlloc)
3626 : {
3627 233 : nDataAlloc = (nDataAlloc * 4) / 3 + 8192 + 1;
3628 : if (nDataAlloc >
3629 : static_cast<vsi_l_offset>(static_cast<size_t>(nDataAlloc)))
3630 : {
3631 : CPLError(CE_Failure, CPLE_AppDefined,
3632 : "Input file too large to be opened");
3633 : VSIFree(*ppabyRet);
3634 : *ppabyRet = nullptr;
3635 : if (bFreeFP)
3636 : CPL_IGNORE_RET_VAL(VSIFCloseL(fp));
3637 : return FALSE;
3638 : }
3639 : GByte *pabyNew = static_cast<GByte *>(
3640 233 : VSIRealloc(*ppabyRet, static_cast<size_t>(nDataAlloc)));
3641 233 : if (pabyNew == nullptr)
3642 : {
3643 0 : CPLError(CE_Failure, CPLE_OutOfMemory,
3644 : "Cannot allocate " CPL_FRMT_GIB " bytes",
3645 : nDataAlloc);
3646 0 : VSIFree(*ppabyRet);
3647 0 : *ppabyRet = nullptr;
3648 0 : if (bFreeFP)
3649 0 : CPL_IGNORE_RET_VAL(VSIFCloseL(fp));
3650 0 : return FALSE;
3651 : }
3652 233 : *ppabyRet = pabyNew;
3653 : }
3654 : const int nRead =
3655 507 : static_cast<int>(VSIFReadL(*ppabyRet + nDataLen, 1, 8192, fp));
3656 507 : nDataLen += nRead;
3657 :
3658 507 : if (nMaxSize >= 0 && nDataLen > static_cast<vsi_l_offset>(nMaxSize))
3659 : {
3660 0 : CPLError(CE_Failure, CPLE_AppDefined,
3661 : "Input file too large to be opened");
3662 0 : VSIFree(*ppabyRet);
3663 0 : *ppabyRet = nullptr;
3664 0 : if (pnSize != nullptr)
3665 0 : *pnSize = 0;
3666 0 : if (bFreeFP)
3667 0 : CPL_IGNORE_RET_VAL(VSIFCloseL(fp));
3668 0 : return FALSE;
3669 : }
3670 :
3671 507 : if (pnSize != nullptr)
3672 245 : *pnSize += nRead;
3673 507 : (*ppabyRet)[nDataLen] = '\0';
3674 507 : if (nRead == 0)
3675 100 : break;
3676 507 : }
3677 : }
3678 : else
3679 : {
3680 12071 : if (VSIFSeekL(fp, 0, SEEK_END) != 0)
3681 : {
3682 0 : if (bFreeFP)
3683 0 : CPL_IGNORE_RET_VAL(VSIFCloseL(fp));
3684 0 : return FALSE;
3685 : }
3686 12071 : nDataLen = VSIFTellL(fp);
3687 :
3688 : // With "large" VSI I/O API we can read data chunks larger than
3689 : // VSIMalloc could allocate. Catch it here.
3690 12071 : if (nDataLen !=
3691 : static_cast<vsi_l_offset>(static_cast<size_t>(nDataLen)) ||
3692 : nDataLen + 1 < nDataLen
3693 : // opening a directory returns nDataLen = INT_MAX (on 32bit) or
3694 : // INT64_MAX (on 64bit)
3695 19004 : || nDataLen + 1 > std::numeric_limits<size_t>::max() / 2 ||
3696 6933 : (nMaxSize >= 0 && nDataLen > static_cast<vsi_l_offset>(nMaxSize)))
3697 : {
3698 0 : CPLError(CE_Failure, CPLE_AppDefined,
3699 : "Input file too large to be opened");
3700 0 : if (bFreeFP)
3701 0 : CPL_IGNORE_RET_VAL(VSIFCloseL(fp));
3702 0 : return FALSE;
3703 : }
3704 :
3705 12071 : if (VSIFSeekL(fp, 0, SEEK_SET) != 0)
3706 : {
3707 0 : if (bFreeFP)
3708 0 : CPL_IGNORE_RET_VAL(VSIFCloseL(fp));
3709 0 : return FALSE;
3710 : }
3711 :
3712 12071 : *ppabyRet =
3713 12071 : static_cast<GByte *>(VSIMalloc(static_cast<size_t>(nDataLen + 1)));
3714 12071 : if (nullptr == *ppabyRet)
3715 : {
3716 0 : CPLError(CE_Failure, CPLE_OutOfMemory,
3717 : "Cannot allocate " CPL_FRMT_GIB " bytes", nDataLen + 1);
3718 0 : if (bFreeFP)
3719 0 : CPL_IGNORE_RET_VAL(VSIFCloseL(fp));
3720 0 : return FALSE;
3721 : }
3722 :
3723 12071 : (*ppabyRet)[nDataLen] = '\0';
3724 12071 : if (nDataLen !=
3725 12071 : VSIFReadL(*ppabyRet, 1, static_cast<size_t>(nDataLen), fp))
3726 : {
3727 0 : CPLError(CE_Failure, CPLE_FileIO,
3728 : "Cannot read " CPL_FRMT_GIB " bytes", nDataLen);
3729 0 : VSIFree(*ppabyRet);
3730 0 : *ppabyRet = nullptr;
3731 0 : if (bFreeFP)
3732 0 : CPL_IGNORE_RET_VAL(VSIFCloseL(fp));
3733 0 : return FALSE;
3734 : }
3735 12071 : if (pnSize != nullptr)
3736 5416 : *pnSize = nDataLen;
3737 : }
3738 12171 : if (bFreeFP)
3739 10893 : CPL_IGNORE_RET_VAL(VSIFCloseL(fp));
3740 12171 : return TRUE;
3741 : }
3742 :
3743 : /************************************************************************/
3744 : /* VSIOverwriteFile() */
3745 : /************************************************************************/
3746 :
3747 : /**
3748 : * \brief Overwrite an existing file with content from another one
3749 : *
3750 : * @param fpTarget file handle opened with VSIFOpenL() with "rb+" flag.
3751 : * @param pszSourceFilename source filename
3752 : *
3753 : * @return TRUE in case of success.
3754 : *
3755 : * @since GDAL 3.1
3756 : */
3757 :
3758 4 : int VSIOverwriteFile(VSILFILE *fpTarget, const char *pszSourceFilename)
3759 : {
3760 4 : VSILFILE *fpSource = VSIFOpenL(pszSourceFilename, "rb");
3761 4 : if (fpSource == nullptr)
3762 : {
3763 0 : CPLError(CE_Failure, CPLE_FileIO, "Cannot open %s", pszSourceFilename);
3764 0 : return false;
3765 : }
3766 :
3767 4 : const size_t nBufferSize = 4096;
3768 4 : void *pBuffer = CPLMalloc(nBufferSize);
3769 4 : VSIFSeekL(fpTarget, 0, SEEK_SET);
3770 4 : bool bRet = true;
3771 : while (true)
3772 : {
3773 4 : size_t nRead = VSIFReadL(pBuffer, 1, nBufferSize, fpSource);
3774 4 : size_t nWritten = VSIFWriteL(pBuffer, 1, nRead, fpTarget);
3775 4 : if (nWritten != nRead)
3776 : {
3777 0 : bRet = false;
3778 0 : break;
3779 : }
3780 4 : if (nRead < nBufferSize)
3781 4 : break;
3782 0 : }
3783 :
3784 4 : if (bRet)
3785 : {
3786 4 : bRet = VSIFTruncateL(fpTarget, VSIFTellL(fpTarget)) == 0;
3787 4 : if (!bRet)
3788 : {
3789 0 : CPLError(CE_Failure, CPLE_FileIO, "Truncation failed");
3790 : }
3791 : }
3792 :
3793 4 : CPLFree(pBuffer);
3794 4 : VSIFCloseL(fpSource);
3795 4 : return bRet;
3796 : }
3797 :
3798 : /************************************************************************/
3799 : /* VSIFGetNativeFileDescriptorL() */
3800 : /************************************************************************/
3801 :
3802 : /**
3803 : * \fn VSIVirtualHandle::GetNativeFileDescriptor()
3804 : * \brief Returns the "native" file descriptor for the virtual handle.
3805 : *
3806 : * This will only return a non-NULL value for "real" files handled by the
3807 : * operating system (to be opposed to GDAL virtual file systems).
3808 : *
3809 : * On POSIX systems, this will be a integer value ("fd") cast as a void*.
3810 : * On Windows systems, this will be the HANDLE.
3811 : *
3812 : * @return the native file descriptor, or NULL.
3813 : */
3814 :
3815 : /**
3816 : * \brief Returns the "native" file descriptor for the virtual handle.
3817 : *
3818 : * This will only return a non-NULL value for "real" files handled by the
3819 : * operating system (to be opposed to GDAL virtual file systems).
3820 : *
3821 : * On POSIX systems, this will be a integer value ("fd") cast as a void*.
3822 : * On Windows systems, this will be the HANDLE.
3823 : *
3824 : * @param fp file handle opened with VSIFOpenL().
3825 : *
3826 : * @return the native file descriptor, or NULL.
3827 : */
3828 :
3829 64 : void *VSIFGetNativeFileDescriptorL(VSILFILE *fp)
3830 : {
3831 64 : return fp->GetNativeFileDescriptor();
3832 : }
3833 :
3834 : /************************************************************************/
3835 : /* VSIGetDiskFreeSpace() */
3836 : /************************************************************************/
3837 :
3838 : /**
3839 : * \brief Return free disk space available on the filesystem.
3840 : *
3841 : * This function returns the free disk space available on the filesystem.
3842 : *
3843 : * @param pszDirname a directory of the filesystem to query.
3844 : * @return The free space in bytes. Or -1 in case of error.
3845 : */
3846 :
3847 74 : GIntBig VSIGetDiskFreeSpace(const char *pszDirname)
3848 : {
3849 74 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszDirname);
3850 :
3851 74 : return poFSHandler->GetDiskFreeSpace(pszDirname);
3852 : }
3853 :
3854 : /************************************************************************/
3855 : /* VSIGetFileSystemsPrefixes() */
3856 : /************************************************************************/
3857 :
3858 : /**
3859 : * \brief Return the list of prefixes for virtual file system handlers
3860 : * currently registered.
3861 : *
3862 : * Typically: "", "/vsimem/", "/vsicurl/", etc
3863 : *
3864 : * @return a NULL terminated list of prefixes. Must be freed with CSLDestroy()
3865 : */
3866 :
3867 9 : char **VSIGetFileSystemsPrefixes(void)
3868 : {
3869 9 : return VSIFileManager::GetPrefixes();
3870 : }
3871 :
3872 : /************************************************************************/
3873 : /* VSIGetFileSystemOptions() */
3874 : /************************************************************************/
3875 :
3876 : /**
3877 : * \brief Return the list of options associated with a virtual file system
3878 : * handler as a serialized XML string.
3879 : *
3880 : * Those options may be set as configuration options with CPLSetConfigOption().
3881 : *
3882 : * @param pszFilename a filename, or prefix of a virtual file system handler.
3883 : * @return a string, which must not be freed, or NULL if no options is declared.
3884 : */
3885 :
3886 36 : const char *VSIGetFileSystemOptions(const char *pszFilename)
3887 : {
3888 36 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
3889 :
3890 36 : return poFSHandler->GetOptions();
3891 : }
3892 :
3893 : /************************************************************************/
3894 : /* VSISetPathSpecificOption() */
3895 : /************************************************************************/
3896 :
3897 : static std::mutex oMutexPathSpecificOptions;
3898 :
3899 : // key is a path prefix
3900 : // value is a map of key, value pair
3901 : static std::map<std::string, std::map<std::string, std::string>>
3902 : oMapPathSpecificOptions;
3903 :
3904 : /**
3905 : * \brief Set a credential (or more generally an option related to a
3906 : * virtual file system) for a given path prefix.
3907 : * @deprecated in GDAL 3.6 for the better named VSISetPathSpecificOption()
3908 : * @see VSISetPathSpecificOption()
3909 : */
3910 0 : void VSISetCredential(const char *pszPathPrefix, const char *pszKey,
3911 : const char *pszValue)
3912 : {
3913 0 : VSISetPathSpecificOption(pszPathPrefix, pszKey, pszValue);
3914 0 : }
3915 :
3916 : /**
3917 : * \brief Set a path specific option for a given path prefix.
3918 : *
3919 : * Such option is typically, but not limited to, a credential setting for a
3920 : * virtual file system.
3921 : *
3922 : * That option may also be set as a configuration option with
3923 : * CPLSetConfigOption(), but this function allows to specify them with a
3924 : * granularity at the level of a file path, which makes it easier if using the
3925 : * same virtual file system but with different credentials (e.g. different
3926 : * credentials for bucket "/vsis3/foo" and "/vsis3/bar")
3927 : *
3928 : * This is supported for the following virtual file systems:
3929 : * /vsis3/, /vsigs/, /vsiaz/, /vsioss/, /vsiwebhdfs, /vsiswift.
3930 : * Note: setting them for a path starting with /vsiXXX/ will also apply for
3931 : * /vsiXXX_streaming/ requests.
3932 : *
3933 : * Note that no particular care is taken to store them in RAM in a secure way.
3934 : * So they might accidentally hit persistent storage if swapping occurs, or
3935 : * someone with access to the memory allocated by the process may be able to
3936 : * read them.
3937 : *
3938 : * @param pszPathPrefix a path prefix of a virtual file system handler.
3939 : * Typically of the form "/vsiXXX/bucket". Must NOT be
3940 : * NULL. Should not include trailing slashes.
3941 : * @param pszKey Option name. Must NOT be NULL.
3942 : * @param pszValue Option value. May be NULL to erase it.
3943 : *
3944 : * @since GDAL 3.6
3945 : */
3946 :
3947 90 : void VSISetPathSpecificOption(const char *pszPathPrefix, const char *pszKey,
3948 : const char *pszValue)
3949 : {
3950 180 : std::lock_guard<std::mutex> oLock(oMutexPathSpecificOptions);
3951 90 : auto oIter = oMapPathSpecificOptions.find(pszPathPrefix);
3952 180 : CPLString osKey(pszKey);
3953 90 : osKey.toupper();
3954 90 : if (oIter == oMapPathSpecificOptions.end())
3955 : {
3956 24 : if (pszValue != nullptr)
3957 24 : oMapPathSpecificOptions[pszPathPrefix][osKey] = pszValue;
3958 : }
3959 66 : else if (pszValue != nullptr)
3960 58 : oIter->second[osKey] = pszValue;
3961 : else
3962 8 : oIter->second.erase(osKey);
3963 90 : }
3964 :
3965 : /************************************************************************/
3966 : /* VSIClearPathSpecificOptions() */
3967 : /************************************************************************/
3968 :
3969 : /**
3970 : * \brief Clear path specific options set with VSISetPathSpecificOption()
3971 : * @deprecated in GDAL 3.6 for the better named VSIClearPathSpecificOptions()
3972 : * @see VSIClearPathSpecificOptions()
3973 : */
3974 0 : void VSIClearCredentials(const char *pszPathPrefix)
3975 : {
3976 0 : return VSIClearPathSpecificOptions(pszPathPrefix);
3977 : }
3978 :
3979 : /**
3980 : * \brief Clear path specific options set with VSISetPathSpecificOption()
3981 : *
3982 : * Note that no particular care is taken to remove them from RAM in a secure
3983 : * way.
3984 : *
3985 : * @param pszPathPrefix If set to NULL, all path specific options are cleared.
3986 : * If set to not-NULL, only those set with
3987 : * VSISetPathSpecificOption(pszPathPrefix, ...) will be
3988 : * cleared.
3989 : *
3990 : * @since GDAL 3.6
3991 : */
3992 19 : void VSIClearPathSpecificOptions(const char *pszPathPrefix)
3993 : {
3994 38 : std::lock_guard<std::mutex> oLock(oMutexPathSpecificOptions);
3995 19 : if (pszPathPrefix == nullptr)
3996 : {
3997 3 : oMapPathSpecificOptions.clear();
3998 : }
3999 : else
4000 : {
4001 16 : oMapPathSpecificOptions.erase(pszPathPrefix);
4002 : }
4003 19 : }
4004 :
4005 : /************************************************************************/
4006 : /* VSIGetPathSpecificOption() */
4007 : /************************************************************************/
4008 :
4009 : /**
4010 : * \brief Get the value of a credential (or more generally an option related to
4011 : * a virtual file system) for a given path.
4012 : * @deprecated in GDAL 3.6 for the better named VSIGetPathSpecificOption()
4013 : * @see VSIGetPathSpecificOption()
4014 : */
4015 0 : const char *VSIGetCredential(const char *pszPath, const char *pszKey,
4016 : const char *pszDefault)
4017 : {
4018 0 : return VSIGetPathSpecificOption(pszPath, pszKey, pszDefault);
4019 : }
4020 :
4021 : /**
4022 : * \brief Get the value a path specific option.
4023 : *
4024 : * Such option is typically, but not limited to, a credential setting for a
4025 : * virtual file system.
4026 : *
4027 : * If no match occurs, CPLGetConfigOption(pszKey, pszDefault) is returned.
4028 : *
4029 : * Mostly to be used by virtual file system implementations.
4030 : *
4031 : * @since GDAL 3.6
4032 : * @see VSISetPathSpecificOption()
4033 : */
4034 145576 : const char *VSIGetPathSpecificOption(const char *pszPath, const char *pszKey,
4035 : const char *pszDefault)
4036 : {
4037 : {
4038 145576 : std::lock_guard<std::mutex> oLock(oMutexPathSpecificOptions);
4039 183859 : for (auto it = oMapPathSpecificOptions.rbegin();
4040 222137 : it != oMapPathSpecificOptions.rend(); ++it)
4041 : {
4042 38547 : if (STARTS_WITH(pszPath, it->first.c_str()))
4043 : {
4044 3748 : auto oIter = it->second.find(CPLString(pszKey).toupper());
4045 3748 : if (oIter != it->second.end())
4046 269 : return oIter->second.c_str();
4047 : }
4048 : }
4049 : }
4050 145312 : return CPLGetConfigOption(pszKey, pszDefault);
4051 : }
4052 :
4053 : /************************************************************************/
4054 : /* VSIDuplicateFileSystemHandler() */
4055 : /************************************************************************/
4056 :
4057 : /**
4058 : * \brief Duplicate an existing file system handler.
4059 : *
4060 : * A number of virtual file system for remote object stores use protocols
4061 : * identical or close to popular ones (typically AWS S3), but with slightly
4062 : * different settings (at the very least the endpoint).
4063 : *
4064 : * This functions allows to duplicate the source virtual file system handler
4065 : * as a new one with a different prefix (when the source virtual file system
4066 : * handler supports the duplication operation).
4067 : *
4068 : * VSISetPathSpecificOption() will typically be called afterwards to change
4069 : * configurable settings on the cloned file system handler (e.g. AWS_S3_ENDPOINT
4070 : * for a clone of /vsis3/).
4071 : *
4072 : * @since GDAL 3.7
4073 : */
4074 4 : bool VSIDuplicateFileSystemHandler(const char *pszSourceFSName,
4075 : const char *pszNewFSName)
4076 : {
4077 : VSIFilesystemHandler *poTargetFSHandler =
4078 4 : VSIFileManager::GetHandler(pszNewFSName);
4079 4 : if (poTargetFSHandler != VSIFileManager::GetHandler("/"))
4080 : {
4081 1 : CPLError(CE_Failure, CPLE_AppDefined,
4082 : "%s is already a known virtual file system", pszNewFSName);
4083 1 : return false;
4084 : }
4085 :
4086 : VSIFilesystemHandler *poSourceFSHandler =
4087 3 : VSIFileManager::GetHandler(pszSourceFSName);
4088 3 : if (!poSourceFSHandler)
4089 : {
4090 0 : CPLError(CE_Failure, CPLE_AppDefined,
4091 : "%s is not a known virtual file system", pszSourceFSName);
4092 0 : return false;
4093 : }
4094 :
4095 3 : poTargetFSHandler = poSourceFSHandler->Duplicate(pszNewFSName);
4096 3 : if (!poTargetFSHandler)
4097 2 : return false;
4098 :
4099 1 : VSIFileManager::InstallHandler(pszNewFSName, poTargetFSHandler);
4100 1 : return true;
4101 : }
4102 :
4103 : /************************************************************************/
4104 : /* ==================================================================== */
4105 : /* VSIFileManager() */
4106 : /* ==================================================================== */
4107 : /************************************************************************/
4108 :
4109 : #ifndef DOXYGEN_SKIP
4110 :
4111 : /*
4112 : ** Notes on Multithreading:
4113 : **
4114 : ** The VSIFileManager maintains a list of file type handlers (mem, large
4115 : ** file, etc). It should be thread safe.
4116 : **/
4117 :
4118 : /************************************************************************/
4119 : /* VSIFileManager() */
4120 : /************************************************************************/
4121 :
4122 1768 : VSIFileManager::VSIFileManager() : poDefaultHandler(nullptr)
4123 : {
4124 1768 : }
4125 :
4126 : /************************************************************************/
4127 : /* ~VSIFileManager() */
4128 : /************************************************************************/
4129 :
4130 1123 : VSIFileManager::~VSIFileManager()
4131 : {
4132 2246 : std::set<VSIFilesystemHandler *> oSetAlreadyDeleted;
4133 34815 : for (std::map<std::string, VSIFilesystemHandler *>::const_iterator iter =
4134 1123 : oHandlers.begin();
4135 70753 : iter != oHandlers.end(); ++iter)
4136 : {
4137 34815 : if (oSetAlreadyDeleted.find(iter->second) == oSetAlreadyDeleted.end())
4138 : {
4139 31446 : oSetAlreadyDeleted.insert(iter->second);
4140 31446 : delete iter->second;
4141 : }
4142 : }
4143 :
4144 1123 : delete poDefaultHandler;
4145 1123 : }
4146 :
4147 : /************************************************************************/
4148 : /* Get() */
4149 : /************************************************************************/
4150 :
4151 : static VSIFileManager *poManager = nullptr;
4152 : static CPLMutex *hVSIFileManagerMutex = nullptr;
4153 :
4154 2775330 : VSIFileManager *VSIFileManager::Get()
4155 : {
4156 5550840 : CPLMutexHolder oHolder(&hVSIFileManagerMutex);
4157 2775510 : if (poManager != nullptr)
4158 : {
4159 2773740 : return poManager;
4160 : }
4161 :
4162 1768 : poManager = new VSIFileManager;
4163 1768 : VSIInstallLargeFileHandler();
4164 1768 : VSIInstallSubFileHandler();
4165 1768 : VSIInstallMemFileHandler();
4166 : #ifdef HAVE_LIBZ
4167 1768 : VSIInstallGZipFileHandler();
4168 1768 : VSIInstallZipFileHandler();
4169 : #endif
4170 : #ifdef HAVE_LIBARCHIVE
4171 : VSIInstall7zFileHandler();
4172 : VSIInstallRarFileHandler();
4173 : #endif
4174 : #ifdef HAVE_CURL
4175 1768 : VSIInstallCurlFileHandler();
4176 1768 : VSIInstallCurlStreamingFileHandler();
4177 1768 : VSIInstallS3FileHandler();
4178 1768 : VSIInstallS3StreamingFileHandler();
4179 1768 : VSIInstallGSFileHandler();
4180 1768 : VSIInstallGSStreamingFileHandler();
4181 1768 : VSIInstallAzureFileHandler();
4182 1768 : VSIInstallAzureStreamingFileHandler();
4183 1768 : VSIInstallADLSFileHandler();
4184 1768 : VSIInstallOSSFileHandler();
4185 1768 : VSIInstallOSSStreamingFileHandler();
4186 1768 : VSIInstallSwiftFileHandler();
4187 1768 : VSIInstallSwiftStreamingFileHandler();
4188 1768 : VSIInstallWebHdfsHandler();
4189 : #endif
4190 1768 : VSIInstallStdinHandler();
4191 1768 : VSIInstallHdfsHandler();
4192 1768 : VSIInstallStdoutHandler();
4193 1768 : VSIInstallSparseFileHandler();
4194 1768 : VSIInstallTarFileHandler();
4195 1768 : VSIInstallCachedFileHandler();
4196 1768 : VSIInstallCryptFileHandler();
4197 :
4198 1768 : return poManager;
4199 : }
4200 :
4201 : /************************************************************************/
4202 : /* GetPrefixes() */
4203 : /************************************************************************/
4204 :
4205 685 : char **VSIFileManager::GetPrefixes()
4206 : {
4207 1370 : CPLMutexHolder oHolder(&hVSIFileManagerMutex);
4208 1370 : CPLStringList aosList;
4209 21920 : for (const auto &oIter : Get()->oHandlers)
4210 : {
4211 21235 : if (oIter.first != "/vsicurl?")
4212 : {
4213 20550 : aosList.AddString(oIter.first.c_str());
4214 : }
4215 : }
4216 1370 : return aosList.StealList();
4217 : }
4218 :
4219 : /************************************************************************/
4220 : /* GetHandler() */
4221 : /************************************************************************/
4222 :
4223 2718140 : VSIFilesystemHandler *VSIFileManager::GetHandler(const char *pszPath)
4224 :
4225 : {
4226 2718140 : VSIFileManager *poThis = Get();
4227 2718290 : const size_t nPathLen = strlen(pszPath);
4228 :
4229 60262200 : for (std::map<std::string, VSIFilesystemHandler *>::const_iterator iter =
4230 2718290 : poThis->oHandlers.begin();
4231 123239000 : iter != poThis->oHandlers.end(); ++iter)
4232 : {
4233 62115300 : const char *pszIterKey = iter->first.c_str();
4234 62112500 : const size_t nIterKeyLen = iter->first.size();
4235 62121200 : if (strncmp(pszPath, pszIterKey, nIterKeyLen) == 0)
4236 1862500 : return iter->second;
4237 :
4238 : // "/vsimem\foo" should be handled as "/vsimem/foo".
4239 60368200 : if (nIterKeyLen && nPathLen > nIterKeyLen &&
4240 53274800 : pszIterKey[nIterKeyLen - 1] == '/' &&
4241 47055100 : pszPath[nIterKeyLen - 1] == '\\' &&
4242 30 : strncmp(pszPath, pszIterKey, nIterKeyLen - 1) == 0)
4243 0 : return iter->second;
4244 :
4245 : // /vsimem should be treated as a match for /vsimem/.
4246 60368200 : if (nPathLen + 1 == nIterKeyLen &&
4247 570703 : strncmp(pszPath, pszIterKey, nPathLen) == 0)
4248 109529 : return iter->second;
4249 : }
4250 :
4251 855710 : return poThis->poDefaultHandler;
4252 : }
4253 :
4254 : /************************************************************************/
4255 : /* InstallHandler() */
4256 : /************************************************************************/
4257 :
4258 56527 : void VSIFileManager::InstallHandler(const std::string &osPrefix,
4259 : VSIFilesystemHandler *poHandler)
4260 :
4261 : {
4262 56527 : if (osPrefix == "")
4263 1768 : Get()->poDefaultHandler = poHandler;
4264 : else
4265 54759 : Get()->oHandlers[osPrefix] = poHandler;
4266 56527 : }
4267 :
4268 : /************************************************************************/
4269 : /* RemoveHandler() */
4270 : /************************************************************************/
4271 :
4272 3 : void VSIFileManager::RemoveHandler(const std::string &osPrefix)
4273 : {
4274 3 : if (osPrefix == "")
4275 0 : Get()->poDefaultHandler = nullptr;
4276 : else
4277 3 : Get()->oHandlers.erase(osPrefix);
4278 3 : }
4279 :
4280 : /************************************************************************/
4281 : /* VSICleanupFileManager() */
4282 : /************************************************************************/
4283 :
4284 1123 : void VSICleanupFileManager()
4285 :
4286 : {
4287 1123 : if (poManager)
4288 : {
4289 1123 : delete poManager;
4290 1123 : poManager = nullptr;
4291 : }
4292 :
4293 1123 : if (hVSIFileManagerMutex != nullptr)
4294 : {
4295 1123 : CPLDestroyMutex(hVSIFileManagerMutex);
4296 1123 : hVSIFileManagerMutex = nullptr;
4297 : }
4298 :
4299 : #ifdef HAVE_CURL
4300 1123 : VSICURLDestroyCacheFileProp();
4301 : #endif
4302 1123 : }
4303 :
4304 : /************************************************************************/
4305 : /* Truncate() */
4306 : /************************************************************************/
4307 :
4308 2 : int VSIVirtualHandle::Truncate(vsi_l_offset nNewSize)
4309 : {
4310 2 : const vsi_l_offset nOriginalPos = Tell();
4311 2 : if (Seek(0, SEEK_END) == 0 && nNewSize >= Tell())
4312 : {
4313 : // Fill with zeroes
4314 2 : std::vector<GByte> aoBytes(4096, 0);
4315 1 : vsi_l_offset nCurOffset = nOriginalPos;
4316 3 : while (nCurOffset < nNewSize)
4317 : {
4318 2 : constexpr vsi_l_offset nMaxOffset = 4096;
4319 : const int nSize =
4320 2 : static_cast<int>(std::min(nMaxOffset, nNewSize - nCurOffset));
4321 2 : if (Write(&aoBytes[0], nSize, 1) != 1)
4322 : {
4323 0 : Seek(nOriginalPos, SEEK_SET);
4324 0 : return -1;
4325 : }
4326 2 : nCurOffset += nSize;
4327 : }
4328 1 : return Seek(nOriginalPos, SEEK_SET) == 0 ? 0 : -1;
4329 : }
4330 :
4331 1 : CPLDebug("VSI", "Truncation is not supported in generic implementation "
4332 : "of Truncate()");
4333 1 : Seek(nOriginalPos, SEEK_SET);
4334 1 : return -1;
4335 : }
4336 :
4337 : /************************************************************************/
4338 : /* ReadMultiRange() */
4339 : /************************************************************************/
4340 :
4341 762 : int VSIVirtualHandle::ReadMultiRange(int nRanges, void **ppData,
4342 : const vsi_l_offset *panOffsets,
4343 : const size_t *panSizes)
4344 : {
4345 762 : int nRet = 0;
4346 762 : const vsi_l_offset nCurOffset = Tell();
4347 58777 : for (int i = 0; i < nRanges; i++)
4348 : {
4349 58034 : if (Seek(panOffsets[i], SEEK_SET) < 0)
4350 : {
4351 0 : nRet = -1;
4352 0 : break;
4353 : }
4354 :
4355 58034 : const size_t nRead = Read(ppData[i], 1, panSizes[i]);
4356 58034 : if (panSizes[i] != nRead)
4357 : {
4358 19 : nRet = -1;
4359 19 : break;
4360 : }
4361 : }
4362 :
4363 762 : Seek(nCurOffset, SEEK_SET);
4364 :
4365 762 : return nRet;
4366 : }
4367 :
4368 : #endif // #ifndef DOXYGEN_SKIP
4369 :
4370 : /************************************************************************/
4371 : /* HasPRead() */
4372 : /************************************************************************/
4373 :
4374 : /** Returns whether this file handle supports the PRead() method.
4375 : *
4376 : * @since GDAL 3.6
4377 : */
4378 0 : bool VSIVirtualHandle::HasPRead() const
4379 : {
4380 0 : return false;
4381 : }
4382 :
4383 : /************************************************************************/
4384 : /* PRead() */
4385 : /************************************************************************/
4386 :
4387 : /** Do a parallel-compatible read operation.
4388 : *
4389 : * This methods reads into pBuffer up to nSize bytes starting at offset nOffset
4390 : * in the file. The current file offset is not affected by this method.
4391 : *
4392 : * The implementation is thread-safe: several threads can issue PRead()
4393 : * concurrently on the same VSIVirtualHandle object.
4394 : *
4395 : * This method has the same semantics as pread() Linux operation. It is only
4396 : * available if HasPRead() returns true.
4397 : *
4398 : * @param pBuffer output buffer (must be at least nSize bytes large).
4399 : * @param nSize number of bytes to read in the file.
4400 : * @param nOffset file offset from which to read.
4401 : * @return number of bytes read.
4402 : * @since GDAL 3.6
4403 : */
4404 0 : size_t VSIVirtualHandle::PRead(CPL_UNUSED void *pBuffer,
4405 : CPL_UNUSED size_t nSize,
4406 : CPL_UNUSED vsi_l_offset nOffset) const
4407 : {
4408 0 : return 0;
4409 : }
4410 :
4411 : #ifndef DOXYGEN_SKIP
4412 : /************************************************************************/
4413 : /* VSIProxyFileHandle::CancelCreation() */
4414 : /************************************************************************/
4415 :
4416 8 : void VSIProxyFileHandle::CancelCreation()
4417 : {
4418 8 : m_nativeHandle->CancelCreation();
4419 8 : }
4420 : #endif
4421 :
4422 : /************************************************************************/
4423 : /* VSIURIToVSIPath() */
4424 : /************************************************************************/
4425 :
4426 : /** Return a VSI compatible path from a URI / URL
4427 : *
4428 : * Substitute URI / URLs starting with s3://, gs://, etc. by their VSI
4429 : * prefix equivalent. If no known substitution is found, the input string is
4430 : * returned unmodified.
4431 : *
4432 : * @since GDAL 3.12
4433 : */
4434 693 : std::string VSIURIToVSIPath(const std::string &osURI)
4435 : {
4436 : static const struct
4437 : {
4438 : const char *pszFSSpecPrefix;
4439 : const char *pszVSIPrefix;
4440 : } substitutions[] = {
4441 : {"s3://", "/vsis3/"},
4442 : {"gs://", "/vsigs/"},
4443 : {"gcs://", "/vsigs/"},
4444 : {"az://", "/vsiaz/"},
4445 : {"azure://", "/vsiaz/"},
4446 : {"http://", "/vsicurl/http://"},
4447 : {"https://", "/vsicurl/https://"},
4448 : {"file://", ""},
4449 : };
4450 :
4451 6222 : for (const auto &substitution : substitutions)
4452 : {
4453 5538 : if (STARTS_WITH(osURI.c_str(), substitution.pszFSSpecPrefix))
4454 : {
4455 18 : return std::string(substitution.pszVSIPrefix)
4456 9 : .append(osURI.c_str() + strlen(substitution.pszFSSpecPrefix));
4457 : }
4458 : }
4459 :
4460 684 : return osURI;
4461 : }
|
