Line data Source code
1 : /******************************************************************************
2 : *
3 : * Project: VSI Virtual File System
4 : * Purpose: Implementation VSI*L File API and other file system access
5 : * methods going through file virtualization.
6 : * Author: Frank Warmerdam, warmerdam@pobox.com
7 : *
8 : ******************************************************************************
9 : * Copyright (c) 2005, Frank Warmerdam <warmerdam@pobox.com>
10 : * Copyright (c) 2008-2014, Even Rouault <even dot rouault at spatialys.com>
11 : *
12 : * SPDX-License-Identifier: MIT
13 : ****************************************************************************/
14 :
15 : #include "cpl_port.h"
16 : #include "cpl_vsi.h"
17 :
18 : #include <cassert>
19 : #include <cinttypes>
20 : #include <cstdarg>
21 : #include <cstddef>
22 : #include <cstring>
23 :
24 : #include <fcntl.h>
25 :
26 : #include <algorithm>
27 : #include <limits>
28 : #include <map>
29 : #include <memory>
30 : #include <mutex>
31 : #include <set>
32 : #include <string>
33 : #include <utility>
34 : #include <vector>
35 :
36 : #include "cpl_conv.h"
37 : #include "cpl_error.h"
38 : #include "cpl_multiproc.h"
39 : #include "cpl_string.h"
40 : #include "cpl_vsi_virtual.h"
41 : #include "cpl_vsil_curl_class.h"
42 :
43 : // To avoid aliasing to GetDiskFreeSpace to GetDiskFreeSpaceA on Windows
44 : #ifdef GetDiskFreeSpace
45 : #undef GetDiskFreeSpace
46 : #endif
47 :
48 : /************************************************************************/
49 : /* VSIReadDir() */
50 : /************************************************************************/
51 :
52 : /**
53 : * \brief Read names in a directory.
54 : *
55 : * This function abstracts access to directory contains. It returns a
56 : * list of strings containing the names of files, and directories in this
57 : * directory. The resulting string list becomes the responsibility of the
58 : * application and should be freed with CSLDestroy() when no longer needed.
59 : *
60 : * Note that no error is issued via CPLError() if the directory path is
61 : * invalid, though NULL is returned.
62 : *
63 : * This function used to be known as CPLReadDir(), but the old name is now
64 : * deprecated.
65 : *
66 : * @param pszPath the relative, or absolute path of a directory to read.
67 : * UTF-8 encoded.
68 : * @return The list of entries in the directory, or NULL if the directory
69 : * doesn't exist. Filenames are returned in UTF-8 encoding.
70 : */
71 :
72 16266 : char **VSIReadDir(const char *pszPath)
73 : {
74 16266 : return VSIReadDirEx(pszPath, 0);
75 : }
76 :
77 : /************************************************************************/
78 : /* VSIReadDirEx() */
79 : /************************************************************************/
80 :
81 : /**
82 : * \brief Read names in a directory.
83 : *
84 : * This function abstracts access to directory contains. It returns a
85 : * list of strings containing the names of files, and directories in this
86 : * directory. The resulting string list becomes the responsibility of the
87 : * application and should be freed with CSLDestroy() when no longer needed.
88 : *
89 : * Note that no error is issued via CPLError() if the directory path is
90 : * invalid, though NULL is returned.
91 : *
92 : * If nMaxFiles is set to a positive number, directory listing will stop after
93 : * that limit has been reached. Note that to indicate truncate, at least one
94 : * element more than the nMaxFiles limit will be returned. If CSLCount() on the
95 : * result is lesser or equal to nMaxFiles, then no truncation occurred.
96 : *
97 : * @param pszPath the relative, or absolute path of a directory to read.
98 : * UTF-8 encoded.
99 : * @param nMaxFiles maximum number of files after which to stop, or 0 for no
100 : * limit.
101 : * @return The list of entries in the directory, or NULL if the directory
102 : * doesn't exist. Filenames are returned in UTF-8 encoding.
103 : */
104 :
105 57947 : char **VSIReadDirEx(const char *pszPath, int nMaxFiles)
106 : {
107 57947 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszPath);
108 :
109 57947 : return poFSHandler->ReadDirEx(pszPath, nMaxFiles);
110 : }
111 :
112 : /************************************************************************/
113 : /* VSISiblingFiles() */
114 : /************************************************************************/
115 :
116 : /**
117 : * \brief Return related filenames
118 : *
119 : * This function is essentially meant at being used by GDAL internals.
120 : *
121 : * @param pszFilename the path of a filename to inspect
122 : * UTF-8 encoded.
123 : * @return The list of entries, relative to the directory, of all sidecar
124 : * files available or NULL if the list is not known.
125 : * Filenames are returned in UTF-8 encoding.
126 : * Most implementations will return NULL, and a subsequent ReadDir will
127 : * list all files available in the file's directory. This function will be
128 : * overridden by VSI FilesystemHandlers that wish to force e.g. an empty list
129 : * to avoid opening non-existent files on slow filesystems. The return value
130 : * shall be destroyed with CSLDestroy()
131 : * @since GDAL 3.2
132 : */
133 82548 : char **VSISiblingFiles(const char *pszFilename)
134 : {
135 82548 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
136 :
137 82550 : return poFSHandler->SiblingFiles(pszFilename);
138 : }
139 :
140 : /************************************************************************/
141 : /* VSIFnMatch() */
142 : /************************************************************************/
143 :
144 707 : static bool VSIFnMatch(const char *pszPattern, const char *pszStr)
145 : {
146 707 : for (; *pszPattern && *pszStr; pszPattern++, pszStr++)
147 : {
148 688 : if (*pszPattern == '*')
149 : {
150 27 : if (pszPattern[1] == 0)
151 4 : return true;
152 78 : for (; *pszStr; ++pszStr)
153 : {
154 71 : if (VSIFnMatch(pszPattern + 1, pszStr))
155 16 : return true;
156 : }
157 7 : return false;
158 : }
159 661 : else if (*pszPattern == '?')
160 : {
161 : // match single any char
162 : }
163 657 : else if (*pszPattern == '[')
164 : {
165 : // match character classes and ranges
166 : // "[abcd]" will match a character that is a, b, c or d
167 : // "[a-z]" will match a character that is a to z
168 : // "[!abcd] will match a character that is *not* a, b, c or d
169 : // "[]]" will match character ]
170 : // "[]-]" will match character ] or -
171 : // "[!]a-]" will match a character that is *not* ], a or -
172 :
173 10 : const char *pszOpenBracket = pszPattern;
174 10 : ++pszPattern;
175 10 : const bool isNot = (*pszPattern == '!');
176 10 : if (isNot)
177 : {
178 3 : ++pszOpenBracket;
179 3 : ++pszPattern;
180 : }
181 10 : bool res = false;
182 22 : for (; *pszPattern; ++pszPattern)
183 : {
184 21 : if ((*pszPattern == ']' || *pszPattern == '-') &&
185 13 : pszPattern == pszOpenBracket + 1)
186 : {
187 3 : if (*pszStr == *pszPattern)
188 : {
189 2 : res = true;
190 : }
191 : }
192 18 : else if (*pszPattern == ']')
193 : {
194 9 : break;
195 : }
196 9 : else if (pszPattern[1] == '-' && pszPattern[2] != 0 &&
197 2 : pszPattern[2] != ']')
198 : {
199 1 : if (*pszStr >= pszPattern[0] && *pszStr <= pszPattern[2])
200 : {
201 1 : res = true;
202 : }
203 1 : pszPattern += 2;
204 : }
205 8 : else if (*pszStr == *pszPattern)
206 : {
207 2 : res = true;
208 : }
209 : }
210 10 : if (*pszPattern == 0)
211 1 : return false;
212 9 : if (!res && !isNot)
213 1 : return false;
214 8 : if (res && isNot)
215 0 : return false;
216 : }
217 647 : else if (*pszPattern != *pszStr)
218 : {
219 501 : return false;
220 : }
221 : }
222 19 : return *pszPattern == 0 && *pszStr == 0;
223 : }
224 :
225 : /************************************************************************/
226 : /* VSIGlob() */
227 : /************************************************************************/
228 :
229 : /**
230 : \brief Return a list of file and directory names matching
231 : a pattern that can contain wildcards.
232 :
233 : This function has similar behavior to the POSIX glob() function:
234 : https://man7.org/linux/man-pages/man7/glob.7.html
235 :
236 : In particular it supports the following wildcards:
237 : <ul>
238 : <li>'*': match any string</li>
239 : <li>'?': match any single character</li>
240 : <li>'[': match character class or range, with '!' immediately after '['
241 : to indicate negation.</li>
242 : </ul>
243 : Refer to to the above man page for more details.
244 :
245 : It also supports the "**" recursive wildcard, behaving similarly to Python
246 : glob.glob() with recursive=True. Be careful of the amount of memory and time
247 : required when using that recursive wildcard on directories with a large
248 : amount of files and subdirectories.
249 :
250 : Examples, given a file hierarchy:
251 : - one.tif
252 : - my_subdir/two.tif
253 : - my_subdir/subsubdir/three.tif
254 :
255 : \code{.cpp}
256 : VSIGlob("one.tif",NULL,NULL,NULL) returns ["one.tif", NULL]
257 : VSIGlob("*.tif",NULL,NULL,NULL) returns ["one.tif", NULL]
258 : VSIGlob("on?.tif",NULL,NULL,NULL) returns ["one.tif", NULL]
259 : VSIGlob("on[a-z].tif",NULL,NULL,NULL) returns ["one.tif", NULL]
260 : VSIGlob("on[ef].tif",NULL,NULL,NULL) returns ["one.tif", NULL]
261 : VSIGlob("on[!e].tif",NULL,NULL,NULL) returns NULL
262 : VSIGlob("my_subdir" "/" "*.tif",NULL,NULL,NULL) returns ["my_subdir/two.tif", NULL]
263 : VSIGlob("**" "/" "*.tif",NULL,NULL,NULL) returns ["one.tif", "my_subdir/two.tif", "my_subdir/subsubdir/three.tif", NULL]
264 : \endcode
265 :
266 : In the current implementation, matching is done based on the assumption that
267 : a character fits into a single byte, which will not work properly on
268 : non-ASCII UTF-8 filenames.
269 :
270 : VSIGlob() works with any virtual file systems supported by GDAL, including
271 : network file systems such as /vsis3/, /vsigs/, /vsiaz/, etc. But note that
272 : for those ones, the pattern is not passed to the remote server, and thus large
273 : amount of filenames can be transferred from the remote server to the host
274 : where the filtering is done.
275 :
276 : @param pszPattern the relative, or absolute path of a directory to read.
277 : UTF-8 encoded.
278 : @param papszOptions NULL-terminate list of options, or NULL. None supported
279 : currently.
280 : @param pProgressFunc Progress function, or NULL. This is only used as a way
281 : for the user to cancel operation if it takes too much time. The percentage
282 : passed to the callback is not significant (always at 0).
283 : @param pProgressData User data passed to the progress function, or NULL.
284 : @return The list of matched filenames, which must be freed with CSLDestroy().
285 : Filenames are returned in UTF-8 encoding.
286 :
287 : @since GDAL 3.11
288 : */
289 :
290 23 : char **VSIGlob(const char *pszPattern, const char *const *papszOptions,
291 : GDALProgressFunc pProgressFunc, void *pProgressData)
292 : {
293 23 : CPL_IGNORE_RET_VAL(papszOptions);
294 :
295 46 : CPLStringList aosRes;
296 46 : std::vector<std::pair<std::string, size_t>> candidates;
297 23 : candidates.emplace_back(pszPattern, 0);
298 72 : while (!candidates.empty())
299 : {
300 49 : auto [osPattern, nPosStart] = candidates.back();
301 49 : pszPattern = osPattern.c_str() + nPosStart;
302 49 : candidates.pop_back();
303 :
304 49 : std::string osPath = osPattern.substr(0, nPosStart);
305 49 : std::string osCurPath;
306 884 : for (;; ++pszPattern)
307 : {
308 933 : if (*pszPattern == 0 || *pszPattern == '/' || *pszPattern == '\\')
309 : {
310 : struct VSIDirCloser
311 : {
312 21 : void operator()(VSIDIR *dir)
313 : {
314 21 : VSICloseDir(dir);
315 21 : }
316 : };
317 :
318 147 : if (osCurPath == "**")
319 : {
320 : std::unique_ptr<VSIDIR, VSIDirCloser> psDir(
321 1 : VSIOpenDir(osPath.c_str(), -1, nullptr));
322 1 : if (!psDir)
323 0 : return nullptr;
324 : while (const VSIDIREntry *psEntry =
325 5 : VSIGetNextDirEntry(psDir.get()))
326 : {
327 4 : if (pProgressFunc &&
328 0 : !pProgressFunc(0, "", pProgressData))
329 : {
330 0 : return nullptr;
331 : }
332 : {
333 8 : std::string osCandidate(osPath);
334 4 : osCandidate += psEntry->pszName;
335 4 : nPosStart = osCandidate.size();
336 4 : if (*pszPattern)
337 : {
338 4 : osCandidate += pszPattern;
339 : }
340 4 : candidates.emplace_back(std::move(osCandidate),
341 4 : nPosStart);
342 : }
343 4 : }
344 1 : osPath.clear();
345 1 : break;
346 : }
347 146 : else if (osCurPath.find_first_of("*?[") != std::string::npos)
348 : {
349 20 : if (osPath.empty())
350 1 : osPath = ".";
351 : std::unique_ptr<VSIDIR, VSIDirCloser> psDir(
352 20 : VSIOpenDir(osPath.c_str(), 0, nullptr));
353 20 : if (!psDir)
354 0 : return nullptr;
355 : while (const VSIDIREntry *psEntry =
356 498 : VSIGetNextDirEntry(psDir.get()))
357 : {
358 489 : if (pProgressFunc &&
359 11 : !pProgressFunc(0, "", pProgressData))
360 : {
361 0 : return nullptr;
362 : }
363 478 : if (VSIFnMatch(osCurPath.c_str(), psEntry->pszName))
364 : {
365 44 : std::string osCandidate;
366 22 : if (osPath != ".")
367 21 : osCandidate = osPath;
368 22 : osCandidate += psEntry->pszName;
369 22 : nPosStart = osCandidate.size();
370 22 : if (*pszPattern)
371 : {
372 2 : osCandidate += pszPattern;
373 : }
374 22 : candidates.emplace_back(std::move(osCandidate),
375 22 : nPosStart);
376 : }
377 478 : }
378 20 : osPath.clear();
379 20 : break;
380 : }
381 126 : else if (*pszPattern == 0)
382 : {
383 28 : osPath += osCurPath;
384 28 : break;
385 : }
386 : else
387 : {
388 98 : osPath += osCurPath;
389 98 : osPath += *pszPattern;
390 98 : osCurPath.clear();
391 98 : }
392 : }
393 : else
394 : {
395 786 : osCurPath += *pszPattern;
396 : }
397 884 : }
398 49 : if (!osPath.empty())
399 : {
400 : VSIStatBufL sStat;
401 28 : if (VSIStatL(osPath.c_str(), &sStat) == 0)
402 23 : aosRes.AddString(osPath.c_str());
403 : }
404 : }
405 :
406 23 : return aosRes.StealList();
407 : }
408 :
409 : /************************************************************************/
410 : /* VSIGetDirectorySeparator() */
411 : /************************************************************************/
412 :
413 : /** Return the directory separator for the specified path.
414 : *
415 : * Default is forward slash. The only exception currently is the Windows
416 : * file system which returns backslash, unless the specified path is of the
417 : * form "{drive_letter}:/{rest_of_the_path}".
418 : *
419 : * @since 3.9
420 : */
421 956783 : const char *VSIGetDirectorySeparator(const char *pszPath)
422 : {
423 956783 : if (STARTS_WITH(pszPath, "http://") || STARTS_WITH(pszPath, "https://"))
424 663 : return "/";
425 :
426 956120 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszPath);
427 956147 : return poFSHandler->GetDirectorySeparator(pszPath);
428 : }
429 :
430 : /************************************************************************/
431 : /* VSIReadRecursive() */
432 : /************************************************************************/
433 :
434 : /**
435 : * \brief Read names in a directory recursively.
436 : *
437 : * This function abstracts access to directory contents and subdirectories.
438 : * It returns a list of strings containing the names of files and directories
439 : * in this directory and all subdirectories. The resulting string list becomes
440 : * the responsibility of the application and should be freed with CSLDestroy()
441 : * when no longer needed.
442 : *
443 : * Note that no error is issued via CPLError() if the directory path is
444 : * invalid, though NULL is returned.
445 : *
446 : * Note: since GDAL 3.9, for recursive mode, the directory separator will no
447 : * longer be always forward slash, but will be the one returned by
448 : * VSIGetDirectorySeparator(pszPathIn), so potentially backslash on Windows
449 : * file systems.
450 : *
451 : * @param pszPathIn the relative, or absolute path of a directory to read.
452 : * UTF-8 encoded.
453 : *
454 : * @return The list of entries in the directory and subdirectories
455 : * or NULL if the directory doesn't exist. Filenames are returned in UTF-8
456 : * encoding.
457 : *
458 : */
459 :
460 1163 : char **VSIReadDirRecursive(const char *pszPathIn)
461 : {
462 1163 : const char SEP = VSIGetDirectorySeparator(pszPathIn)[0];
463 :
464 1163 : const char *const apszOptions[] = {"NAME_AND_TYPE_ONLY=YES", nullptr};
465 1163 : VSIDIR *psDir = VSIOpenDir(pszPathIn, -1, apszOptions);
466 1163 : if (!psDir)
467 3 : return nullptr;
468 2320 : CPLStringList oFiles;
469 5180 : while (auto psEntry = VSIGetNextDirEntry(psDir))
470 : {
471 4020 : if (VSI_ISDIR(psEntry->nMode) && psEntry->pszName[0] &&
472 1513 : psEntry->pszName[strlen(psEntry->pszName) - 1] != SEP)
473 : {
474 1513 : oFiles.AddString((std::string(psEntry->pszName) + SEP).c_str());
475 : }
476 : else
477 : {
478 2507 : oFiles.AddString(psEntry->pszName);
479 : }
480 4020 : }
481 1160 : VSICloseDir(psDir);
482 :
483 1160 : return oFiles.StealList();
484 : }
485 :
486 : /************************************************************************/
487 : /* CPLReadDir() */
488 : /* */
489 : /* This is present only to provide ABI compatibility with older */
490 : /* versions. */
491 : /************************************************************************/
492 : #undef CPLReadDir
493 :
494 : CPL_C_START
495 : char CPL_DLL **CPLReadDir(const char *pszPath);
496 : CPL_C_END
497 :
498 0 : char **CPLReadDir(const char *pszPath)
499 : {
500 0 : return VSIReadDir(pszPath);
501 : }
502 :
503 : /************************************************************************/
504 : /* VSIOpenDir() */
505 : /************************************************************************/
506 :
507 : /**
508 : * \brief Open a directory to read its entries.
509 : *
510 : * This function is close to the POSIX opendir() function.
511 : *
512 : * For /vsis3/, /vsigs/, /vsioss/, /vsiaz/ and /vsiadls/, this function has an
513 : * efficient implementation, minimizing the number of network requests, when
514 : * invoked with nRecurseDepth <= 0.
515 : *
516 : * Entries are read by calling VSIGetNextDirEntry() on the handled returned by
517 : * that function, until it returns NULL. VSICloseDir() must be called once done
518 : * with the returned directory handle.
519 : *
520 : * @param pszPath the relative, or absolute path of a directory to read.
521 : * UTF-8 encoded.
522 : * @param nRecurseDepth 0 means do not recurse in subdirectories, 1 means
523 : * recurse only in the first level of subdirectories, etc. -1 means unlimited
524 : * recursion level
525 : * @param papszOptions NULL terminated list of options, or NULL. The following
526 : * options are implemented:
527 : * <ul>
528 : * <li>PREFIX=string: (GDAL >= 3.4) Filter to select filenames only starting
529 : * with the specified prefix. Implemented efficiently for /vsis3/, /vsigs/,
530 : * and /vsiaz/ (but not /vsiadls/)
531 : * </li>
532 : * <li>NAME_AND_TYPE_ONLY=YES/NO: (GDAL >= 3.4) Defaults to NO. If set to YES,
533 : * only the pszName and nMode members of VSIDIR are guaranteed to be set.
534 : * This is implemented efficiently for the Unix virtual file system.
535 : * </li>
536 : * </ul>
537 : *
538 : * @return a handle, or NULL in case of error
539 : *
540 : */
541 :
542 1627 : VSIDIR *VSIOpenDir(const char *pszPath, int nRecurseDepth,
543 : const char *const *papszOptions)
544 : {
545 1627 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszPath);
546 :
547 1627 : return poFSHandler->OpenDir(pszPath, nRecurseDepth, papszOptions);
548 : }
549 :
550 : /************************************************************************/
551 : /* VSIGetNextDirEntry() */
552 : /************************************************************************/
553 :
554 : /**
555 : * \brief Return the next entry of the directory
556 : *
557 : * This function is close to the POSIX readdir() function. It actually returns
558 : * more information (file size, last modification time), which on 'real' file
559 : * systems involve one 'stat' call per file.
560 : *
561 : * For filesystems that can have both a regular file and a directory name of
562 : * the same name (typically /vsis3/), when this situation of duplicate happens,
563 : * the directory name will be suffixed by a slash character. Otherwise directory
564 : * names are not suffixed by slash.
565 : *
566 : * The returned entry remains valid until the next call to VSINextDirEntry()
567 : * or VSICloseDir() with the same handle.
568 : *
569 : * Note: since GDAL 3.9, for recursive mode, the directory separator will no
570 : * longer be always forward slash, but will be the one returned by
571 : * VSIGetDirectorySeparator(pszPathIn), so potentially backslash on Windows
572 : * file systems.
573 : *
574 : * @param dir Directory handled returned by VSIOpenDir(). Must not be NULL.
575 : *
576 : * @return a entry, or NULL if there is no more entry in the directory. This
577 : * return value must not be freed.
578 : *
579 : */
580 :
581 16324 : const VSIDIREntry *VSIGetNextDirEntry(VSIDIR *dir)
582 : {
583 16324 : return dir->NextDirEntry();
584 : }
585 :
586 : /************************************************************************/
587 : /* VSICloseDir() */
588 : /************************************************************************/
589 :
590 : /**
591 : * \brief Close a directory
592 : *
593 : * This function is close to the POSIX closedir() function.
594 : *
595 : * @param dir Directory handled returned by VSIOpenDir().
596 : *
597 : */
598 :
599 1588 : void VSICloseDir(VSIDIR *dir)
600 : {
601 1588 : delete dir;
602 1588 : }
603 :
604 : /************************************************************************/
605 : /* VSIMkdir() */
606 : /************************************************************************/
607 :
608 : /**
609 : * \brief Create a directory.
610 : *
611 : * Create a new directory with the indicated mode. For POSIX-style systems,
612 : * the mode is modified by the file creation mask (umask). However, some
613 : * file systems and platforms may not use umask, or they may ignore the mode
614 : * completely. So a reasonable cross-platform default mode value is 0755.
615 : *
616 : * Analog of the POSIX mkdir() function.
617 : *
618 : * @param pszPathname the path to the directory to create. UTF-8 encoded.
619 : * @param mode the permissions mode.
620 : *
621 : * @return 0 on success or -1 on an error.
622 : */
623 :
624 103290 : int VSIMkdir(const char *pszPathname, long mode)
625 :
626 : {
627 103290 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszPathname);
628 :
629 103290 : return poFSHandler->Mkdir(pszPathname, mode);
630 : }
631 :
632 : /************************************************************************/
633 : /* VSIMkdirRecursive() */
634 : /************************************************************************/
635 :
636 : /**
637 : * \brief Create a directory and all its ancestors
638 : *
639 : * @param pszPathname the path to the directory to create. UTF-8 encoded.
640 : * @param mode the permissions mode.
641 : *
642 : * @return 0 on success or -1 on an error.
643 : */
644 :
645 103446 : int VSIMkdirRecursive(const char *pszPathname, long mode)
646 : {
647 103446 : if (!pszPathname)
648 0 : return -1;
649 :
650 206892 : std::string osPathnameOri(pszPathname);
651 103446 : if (cpl::starts_with(osPathnameOri, "/vsimem/"))
652 : {
653 91419 : osPathnameOri = VSIFileManager::GetHandler(pszPathname)
654 91422 : ->GetCanonicalFilename(osPathnameOri);
655 : }
656 : // Limit to avoid performance issues such as in
657 : // https://issues.oss-fuzz.com/issues/471096341
658 103445 : constexpr size_t CPL_MAX_PATH = 4096;
659 206890 : if (osPathnameOri.empty() || osPathnameOri == "/" ||
660 103445 : osPathnameOri.size() > CPL_MAX_PATH)
661 1 : return -1;
662 :
663 : VSIStatBufL sStat;
664 103444 : if (VSIStatL(osPathnameOri.c_str(), &sStat) == 0)
665 : {
666 57797 : return VSI_ISDIR(sStat.st_mode) ? 0 : -1;
667 : }
668 :
669 91296 : std::string osCurrentPath(osPathnameOri);
670 91296 : std::vector<std::string> aosQueue;
671 : while (true)
672 : {
673 93446 : std::string osParentPath(CPLGetPathSafe(osCurrentPath.c_str()));
674 :
675 : // Prevent crazy paths from recursing forever.
676 93446 : if (osParentPath.length() >= osCurrentPath.length())
677 : {
678 0 : break;
679 : }
680 :
681 186892 : if (!osParentPath.empty() &&
682 93446 : VSIStatL(osParentPath.c_str(), &sStat) != 0)
683 : {
684 47798 : osCurrentPath = std::move(osParentPath);
685 47798 : aosQueue.push_back(osCurrentPath);
686 : }
687 : else
688 : {
689 45648 : break;
690 : }
691 47798 : }
692 :
693 93427 : for (auto oIter = aosQueue.rbegin(); oIter != aosQueue.rend(); ++oIter)
694 : {
695 47794 : if (VSIMkdir(oIter->c_str(), mode) != 0 &&
696 : // In case of concurrent VSIMkdirRecursive() on the same directory
697 8 : (VSIStatL(oIter->c_str(), &sStat) != 0 ||
698 1 : !VSI_ISDIR(sStat.st_mode)))
699 : {
700 7 : return -1;
701 : }
702 : }
703 :
704 45642 : if (VSIMkdir(osPathnameOri.c_str(), mode) != 0 &&
705 : // In case of concurrent VSIMkdirRecursive() on the same directory
706 1 : (VSIStatL(osPathnameOri.c_str(), &sStat) != 0 ||
707 1 : !VSI_ISDIR(sStat.st_mode)))
708 : {
709 0 : return -1;
710 : }
711 45641 : return 0;
712 : }
713 :
714 : /************************************************************************/
715 : /* VSIUnlink() */
716 : /************************************************************************/
717 :
718 : /**
719 : * \brief Delete a file.
720 : *
721 : * Deletes a file object from the file system.
722 : *
723 : * This method goes through the VSIFileHandler virtualization and may
724 : * work on unusual filesystems such as in memory.
725 : *
726 : * Analog of the POSIX unlink() function.
727 : *
728 : * @param pszFilename the path of the file to be deleted. UTF-8 encoded.
729 : *
730 : * @return 0 on success or -1 on an error.
731 : */
732 :
733 87541 : int VSIUnlink(const char *pszFilename)
734 :
735 : {
736 87541 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
737 :
738 87540 : return poFSHandler->Unlink(pszFilename);
739 : }
740 :
741 : /************************************************************************/
742 : /* VSIUnlinkBatch() */
743 : /************************************************************************/
744 :
745 : /**
746 : * \brief Delete several files, possibly in a batch.
747 : *
748 : * All files should belong to the same file system handler.
749 : *
750 : * This is implemented efficiently for /vsis3/ and /vsigs/ (provided for /vsigs/
751 : * that OAuth2 authentication is used).
752 : *
753 : * @param papszFiles NULL terminated list of files. UTF-8 encoded.
754 : *
755 : * @return an array of size CSLCount(papszFiles), whose values are TRUE or FALSE
756 : * depending on the success of deletion of the corresponding file. The array
757 : * should be freed with VSIFree().
758 : * NULL might be return in case of a more general error (for example,
759 : * files belonging to different file system handlers)
760 : *
761 : * @since GDAL 3.1
762 : */
763 :
764 11 : int *VSIUnlinkBatch(CSLConstList papszFiles)
765 : {
766 11 : VSIFilesystemHandler *poFSHandler = nullptr;
767 30 : for (CSLConstList papszIter = papszFiles; papszIter && *papszIter;
768 : ++papszIter)
769 : {
770 20 : auto poFSHandlerThisFile = VSIFileManager::GetHandler(*papszIter);
771 20 : if (poFSHandler == nullptr)
772 10 : poFSHandler = poFSHandlerThisFile;
773 10 : else if (poFSHandler != poFSHandlerThisFile)
774 : {
775 1 : CPLError(CE_Failure, CPLE_AppDefined,
776 : "Files belong to different file system handlers");
777 1 : poFSHandler = nullptr;
778 1 : break;
779 : }
780 : }
781 11 : if (poFSHandler == nullptr)
782 2 : return nullptr;
783 9 : return poFSHandler->UnlinkBatch(papszFiles);
784 : }
785 :
786 : /************************************************************************/
787 : /* VSIRename() */
788 : /************************************************************************/
789 :
790 : /**
791 : * \brief Rename a file.
792 : *
793 : * Renames a file object in the file system. It should be possible
794 : * to rename a file onto a new directory, but it is safest if this
795 : * function is only used to rename files that remain in the same directory.
796 : *
797 : * This function only works if the new path is located on the same VSI
798 : * virtual file system than the old path. If not, use VSIMove() instead.
799 : *
800 : * This method goes through the VSIFileHandler virtualization and may
801 : * work on unusual filesystems such as in memory or cloud object storage.
802 : * Note that for cloud object storage, renaming a directory may involve
803 : * renaming all files it contains recursively, and is thus not an atomic
804 : * operation (and could be expensive on directories with many files!)
805 : *
806 : * Analog of the POSIX rename() function.
807 : *
808 : * @param oldpath the name of the file to be renamed. UTF-8 encoded.
809 : * @param newpath the name the file should be given. UTF-8 encoded.
810 : *
811 : * @return 0 on success or -1 on an error.
812 : */
813 :
814 1559 : int VSIRename(const char *oldpath, const char *newpath)
815 :
816 : {
817 1559 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(oldpath);
818 :
819 1559 : return poFSHandler->Rename(oldpath, newpath, nullptr, nullptr);
820 : }
821 :
822 : /************************************************************************/
823 : /* VSIMove() */
824 : /************************************************************************/
825 :
826 : /**
827 : * \brief Move (or rename) a file.
828 : *
829 : * If the new path is an existing directory, the file will be moved to it.
830 : *
831 : * The function can work even if the files are not located on the same VSI
832 : * virtual file system, but it will involve copying and deletion.
833 : *
834 : * Note that for cloud object storage, moving/renaming a directory may involve
835 : * renaming all files it contains recursively, and is thus not an atomic
836 : * operation (and could be slow and expensive on directories with many files!)
837 : *
838 : * @param oldpath the path of the file to be renamed/moved. UTF-8 encoded.
839 : * @param newpath the new path the file should be given. UTF-8 encoded.
840 : * @param papszOptions Null terminated list of options, or NULL.
841 : * @param pProgressFunc Progress callback, or NULL.
842 : * @param pProgressData User data of progress callback, or NULL.
843 : *
844 : * @return 0 on success or -1 on error.
845 : * @since GDAL 3.11
846 : */
847 :
848 12 : int VSIMove(const char *oldpath, const char *newpath,
849 : const char *const *papszOptions, GDALProgressFunc pProgressFunc,
850 : void *pProgressData)
851 : {
852 :
853 12 : if (strcmp(oldpath, newpath) == 0)
854 1 : return 0;
855 :
856 11 : VSIFilesystemHandler *poOldFSHandler = VSIFileManager::GetHandler(oldpath);
857 11 : VSIFilesystemHandler *poNewFSHandler = VSIFileManager::GetHandler(newpath);
858 :
859 : VSIStatBufL sStat;
860 11 : if (VSIStatL(oldpath, &sStat) != 0)
861 : {
862 2 : CPLDebug("VSI", "%s is not a object", oldpath);
863 2 : errno = ENOENT;
864 2 : return -1;
865 : }
866 :
867 18 : std::string sNewpath(newpath);
868 : VSIStatBufL sStatNew;
869 9 : if (VSIStatL(newpath, &sStatNew) == 0 && VSI_ISDIR(sStatNew.st_mode))
870 : {
871 : sNewpath =
872 4 : CPLFormFilenameSafe(newpath, CPLGetFilename(oldpath), nullptr);
873 : }
874 :
875 9 : int ret = 0;
876 :
877 9 : if (poOldFSHandler == poNewFSHandler)
878 : {
879 2 : ret = poOldFSHandler->Rename(oldpath, sNewpath.c_str(), pProgressFunc,
880 2 : pProgressData);
881 2 : if (ret == 0 && pProgressFunc)
882 1 : ret = pProgressFunc(1.0, "", pProgressData) ? 0 : -1;
883 2 : return ret;
884 : }
885 :
886 7 : if (VSI_ISDIR(sStat.st_mode))
887 : {
888 6 : const CPLStringList aosList(VSIReadDir(oldpath));
889 3 : poNewFSHandler->Mkdir(sNewpath.c_str(), 0755);
890 3 : bool bFoundFiles = false;
891 3 : const int nListSize = aosList.size();
892 6 : for (int i = 0; ret == 0 && i < nListSize; i++)
893 : {
894 3 : if (strcmp(aosList[i], ".") != 0 && strcmp(aosList[i], "..") != 0)
895 : {
896 1 : bFoundFiles = true;
897 : const std::string osSrc =
898 2 : CPLFormFilenameSafe(oldpath, aosList[i], nullptr);
899 : const std::string osTarget =
900 2 : CPLFormFilenameSafe(sNewpath.c_str(), aosList[i], nullptr);
901 2 : void *pScaledProgress = GDALCreateScaledProgress(
902 1 : static_cast<double>(i) / nListSize,
903 1 : static_cast<double>(i + 1) / nListSize, pProgressFunc,
904 : pProgressData);
905 1 : ret = VSIMove(osSrc.c_str(), osTarget.c_str(), papszOptions,
906 : pScaledProgress ? GDALScaledProgress : nullptr,
907 : pScaledProgress);
908 1 : GDALDestroyScaledProgress(pScaledProgress);
909 : }
910 : }
911 3 : if (!bFoundFiles)
912 2 : ret = VSIStatL(sNewpath.c_str(), &sStat);
913 3 : if (ret == 0)
914 3 : ret = poOldFSHandler->Rmdir(oldpath);
915 : }
916 : else
917 : {
918 8 : ret = VSICopyFile(oldpath, sNewpath.c_str(), nullptr, sStat.st_size,
919 3 : nullptr, pProgressFunc, pProgressData) == 0 &&
920 3 : VSIUnlink(oldpath) == 0
921 7 : ? 0
922 : : -1;
923 : }
924 7 : if (ret == 0 && pProgressFunc)
925 4 : ret = pProgressFunc(1.0, "", pProgressData) ? 0 : -1;
926 7 : return ret;
927 : }
928 :
929 : /************************************************************************/
930 : /* VSICopyFile() */
931 : /************************************************************************/
932 :
933 : /**
934 : * \brief Copy a source file into a target file.
935 : *
936 : * For a /vsizip/foo.zip/bar target, the options available are those of
937 : * CPLAddFileInZip()
938 : *
939 : * The following copies are made fully on the target server, without local
940 : * download from source and upload to target:
941 : * - /vsis3/ -> /vsis3/
942 : * - /vsigs/ -> /vsigs/
943 : * - /vsiaz/ -> /vsiaz/
944 : * - /vsiadls/ -> /vsiadls/
945 : * - any of the above or /vsicurl/ -> /vsiaz/ (starting with GDAL 3.8)
946 : *
947 : * @param pszSource Source filename. UTF-8 encoded. May be NULL if fpSource is
948 : * not NULL.
949 : * @param pszTarget Target filename. UTF-8 encoded. Must not be NULL
950 : * @param fpSource File handle on the source file. May be NULL if pszSource is
951 : * not NULL.
952 : * @param nSourceSize Size of the source file. Pass -1 if unknown.
953 : * If set to -1, and progress callback is used, VSIStatL() will be used on
954 : * pszSource to retrieve the source size.
955 : * @param papszOptions Null terminated list of options, or NULL.
956 : * @param pProgressFunc Progress callback, or NULL.
957 : * @param pProgressData User data of progress callback, or NULL.
958 : *
959 : * @return 0 on success.
960 : * @since GDAL 3.7
961 : */
962 :
963 2332 : int VSICopyFile(const char *pszSource, const char *pszTarget,
964 : VSILFILE *fpSource, vsi_l_offset nSourceSize,
965 : const char *const *papszOptions, GDALProgressFunc pProgressFunc,
966 : void *pProgressData)
967 :
968 : {
969 2332 : if (!pszSource && !fpSource)
970 : {
971 1 : CPLError(CE_Failure, CPLE_AppDefined,
972 : "pszSource == nullptr && fpSource == nullptr");
973 1 : return -1;
974 : }
975 2331 : if (!pszTarget || pszTarget[0] == '\0')
976 : {
977 0 : return -1;
978 : }
979 :
980 : VSIFilesystemHandler *poFSHandlerTarget =
981 2331 : VSIFileManager::GetHandler(pszTarget);
982 2331 : return poFSHandlerTarget->CopyFile(pszSource, pszTarget, fpSource,
983 : nSourceSize, papszOptions, pProgressFunc,
984 2331 : pProgressData);
985 : }
986 :
987 : /************************************************************************/
988 : /* VSICopyFileRestartable() */
989 : /************************************************************************/
990 :
991 : /**
992 : \brief Copy a source file into a target file in a way that can (potentially)
993 : be restarted.
994 :
995 : This function provides the possibility of efficiently restarting upload of
996 : large files to cloud storage that implements upload in a chunked way,
997 : such as /vsis3/ and /vsigs/.
998 : For other destination file systems, this function may fallback to
999 : VSICopyFile() and not provide any smart restartable implementation.
1000 :
1001 : Example of a potential workflow:
1002 :
1003 : @code{.cpp}
1004 : char* pszOutputPayload = NULL;
1005 : int ret = VSICopyFileRestartable(pszSource, pszTarget, NULL,
1006 : &pszOutputPayload, NULL, NULL, NULL);
1007 : while( ret == 1 ) // add also a limiting counter to avoid potentiall endless looping
1008 : {
1009 : // TODO: wait for some time
1010 :
1011 : char* pszOutputPayloadNew = NULL;
1012 : const char* pszInputPayload = pszOutputPayload;
1013 : ret = VSICopyFileRestartable(pszSource, pszTarget, pszInputPayload,
1014 : &pszOutputPayloadNew, NULL, NULL, NULL);
1015 : VSIFree(pszOutputPayload);
1016 : pszOutputPayload = pszOutputPayloadNew;
1017 : }
1018 : VSIFree(pszOutputPayload);
1019 : @endcode
1020 :
1021 : @param pszSource Source filename. UTF-8 encoded. Must not be NULL
1022 : @param pszTarget Target filename. UTF-8 encoded. Must not be NULL
1023 : @param pszInputPayload NULL at the first invocation. When doing a retry,
1024 : should be the content of *ppszOutputPayload from a
1025 : previous invocation.
1026 : @param[out] ppszOutputPayload Pointer to an output string that will be set to
1027 : a value that can be provided as pszInputPayload
1028 : for a next call to VSICopyFileRestartable().
1029 : ppszOutputPayload must not be NULL.
1030 : The string set in *ppszOutputPayload, if not NULL,
1031 : is JSON-encoded, and can be re-used in another
1032 : process instance. It must be freed with VSIFree()
1033 : when no longer needed.
1034 : @param papszOptions Null terminated list of options, or NULL.
1035 : Currently accepted options are:
1036 : <ul>
1037 : <li>NUM_THREADS=integer or ALL_CPUS. Number of threads to use for parallel
1038 : file copying. Only use for when /vsis3/, /vsigs/, /vsiaz/ or /vsiadls/ is in
1039 : source or target. The default is 10.
1040 : </li>
1041 : <li>CHUNK_SIZE=integer. Maximum size of chunk (in bytes) to use
1042 : to split large objects. For upload to /vsis3/, this chunk size must be set at
1043 : least to 5 MB. The default is 50 MB.
1044 : </li>
1045 : </ul>
1046 : @param pProgressFunc Progress callback, or NULL.
1047 : @param pProgressData User data of progress callback, or NULL.
1048 : @return 0 on success,
1049 : -1 on (non-restartable) failure,
1050 : 1 if VSICopyFileRestartable() can be called again in a restartable way
1051 : @since GDAL 3.10
1052 :
1053 : @see VSIAbortPendingUploads()
1054 : */
1055 :
1056 20 : int VSICopyFileRestartable(const char *pszSource, const char *pszTarget,
1057 : const char *pszInputPayload,
1058 : char **ppszOutputPayload,
1059 : const char *const *papszOptions,
1060 : GDALProgressFunc pProgressFunc, void *pProgressData)
1061 :
1062 : {
1063 20 : if (!pszSource)
1064 : {
1065 0 : return -1;
1066 : }
1067 20 : if (!pszTarget || pszTarget[0] == '\0')
1068 : {
1069 0 : return -1;
1070 : }
1071 20 : if (!ppszOutputPayload)
1072 : {
1073 0 : return -1;
1074 : }
1075 :
1076 : VSIFilesystemHandler *poFSHandlerTarget =
1077 20 : VSIFileManager::GetHandler(pszTarget);
1078 20 : return poFSHandlerTarget->CopyFileRestartable(
1079 : pszSource, pszTarget, pszInputPayload, ppszOutputPayload, papszOptions,
1080 20 : pProgressFunc, pProgressData);
1081 : }
1082 :
1083 : /************************************************************************/
1084 : /* VSISync() */
1085 : /************************************************************************/
1086 :
1087 : /**
1088 : * \brief Synchronize a source file/directory with a target file/directory.
1089 : *
1090 : * This is a analog of the 'rsync' utility. In the current implementation,
1091 : * rsync would be more efficient for local file copying, but VSISync() main
1092 : * interest is when the source or target is a remote
1093 : * file system like /vsis3/ or /vsigs/, in which case it can take into account
1094 : * the timestamps of the files (or optionally the ETag/MD5Sum) to avoid
1095 : * unneeded copy operations.
1096 : *
1097 : * This is only implemented efficiently for:
1098 : * <ul>
1099 : * <li> local filesystem <--> remote filesystem.</li>
1100 : * <li> remote filesystem <--> remote filesystem (starting with GDAL 3.1).
1101 : * Where the source and target remote filesystems are the same and one of
1102 : * /vsis3/, /vsigs/ or /vsiaz/. Or when the target is /vsiaz/ and the source
1103 : * is /vsis3/, /vsigs/, /vsiadls/ or /vsicurl/ (starting with GDAL 3.8)</li>
1104 : * </ul>
1105 : *
1106 : * Similarly to rsync behavior, if the source filename ends with a slash,
1107 : * it means that the content of the directory must be copied, but not the
1108 : * directory name. For example, assuming "/home/even/foo" contains a file "bar",
1109 : * VSISync("/home/even/foo/", "/mnt/media", ...) will create a "/mnt/media/bar"
1110 : * file. Whereas VSISync("/home/even/foo", "/mnt/media", ...) will create a
1111 : * "/mnt/media/foo" directory which contains a bar file.
1112 : *
1113 : * @param pszSource Source file or directory. UTF-8 encoded.
1114 : * @param pszTarget Target file or directory. UTF-8 encoded.
1115 : * @param papszOptions Null terminated list of options, or NULL.
1116 : * Currently accepted options are:
1117 : * <ul>
1118 : * <li>RECURSIVE=NO (the default is YES)</li>
1119 : * <li>SYNC_STRATEGY=TIMESTAMP/ETAG/OVERWRITE.
1120 : *
1121 : * Determines which criterion is used to determine if a target file must be
1122 : * replaced when it already exists and has the same file size as the source.
1123 : * Only applies for a source or target being a network filesystem.
1124 : *
1125 : * The default is TIMESTAMP (similarly to how 'aws s3 sync' works), that is
1126 : * to say that for an upload operation, a remote file is
1127 : * replaced if it has a different size or if it is older than the source.
1128 : * For a download operation, a local file is replaced if it has a different
1129 : * size or if it is newer than the remote file.
1130 : *
1131 : * The ETAG strategy assumes that the ETag metadata of the remote file is
1132 : * the MD5Sum of the file content, which is only true in the case of /vsis3/
1133 : * for files not using KMS server side encryption and uploaded in a single
1134 : * PUT operation (so smaller than 50 MB given the default used by GDAL).
1135 : * Only to be used for /vsis3/, /vsigs/ or other filesystems using a
1136 : * MD5Sum as ETAG.
1137 : *
1138 : * The OVERWRITE strategy (GDAL >= 3.2) will always overwrite the target
1139 : * file with the source one.
1140 : * </li>
1141 : * <li>NUM_THREADS=integer. (GDAL >= 3.1) Number of threads to use for parallel
1142 : * file copying. Only use for when /vsis3/, /vsigs/, /vsiaz/ or /vsiadls/ is in
1143 : * source or target. The default is 10 since GDAL 3.3</li>
1144 : * <li>CHUNK_SIZE=integer. (GDAL >= 3.1) Maximum size of chunk (in bytes) to use
1145 : * to split large objects when downloading them from /vsis3/, /vsigs/, /vsiaz/
1146 : * or /vsiadls/ to local file system, or for upload to /vsis3/, /vsiaz/ or
1147 : * /vsiadls/ from local file system. Only used if NUM_THREADS > 1. For upload to
1148 : * /vsis3/, this chunk size must be set at least to 5 MB. The default is 8 MB
1149 : * since GDAL 3.3</li> <li>x-amz-KEY=value. (GDAL >= 3.5) MIME header to pass
1150 : * during creation of a /vsis3/ object.</li> <li>x-goog-KEY=value. (GDAL >= 3.5)
1151 : * MIME header to pass during creation of a /vsigs/ object.</li>
1152 : * <li>x-ms-KEY=value. (GDAL >= 3.5) MIME header to pass during creation of a
1153 : * /vsiaz/ or /vsiadls/ object.</li>
1154 : * </ul>
1155 : * @param pProgressFunc Progress callback, or NULL.
1156 : * @param pProgressData User data of progress callback, or NULL.
1157 : * @param ppapszOutputs Unused. Should be set to NULL for now.
1158 : *
1159 : * @return TRUE on success or FALSE on an error.
1160 : */
1161 :
1162 49 : int VSISync(const char *pszSource, const char *pszTarget,
1163 : const char *const *papszOptions, GDALProgressFunc pProgressFunc,
1164 : void *pProgressData, char ***ppapszOutputs)
1165 :
1166 : {
1167 49 : if (pszSource[0] == '\0' || pszTarget[0] == '\0')
1168 : {
1169 0 : return FALSE;
1170 : }
1171 :
1172 : VSIFilesystemHandler *poFSHandlerSource =
1173 49 : VSIFileManager::GetHandler(pszSource);
1174 : VSIFilesystemHandler *poFSHandlerTarget =
1175 49 : VSIFileManager::GetHandler(pszTarget);
1176 49 : VSIFilesystemHandler *poFSHandlerLocal = VSIFileManager::GetHandler("");
1177 : VSIFilesystemHandler *poFSHandlerMem =
1178 49 : VSIFileManager::GetHandler("/vsimem/");
1179 49 : VSIFilesystemHandler *poFSHandler = poFSHandlerSource;
1180 49 : if (poFSHandlerTarget != poFSHandlerLocal &&
1181 : poFSHandlerTarget != poFSHandlerMem)
1182 : {
1183 22 : poFSHandler = poFSHandlerTarget;
1184 : }
1185 :
1186 98 : return poFSHandler->Sync(pszSource, pszTarget, papszOptions, pProgressFunc,
1187 49 : pProgressData, ppapszOutputs)
1188 49 : ? TRUE
1189 49 : : FALSE;
1190 : }
1191 :
1192 : /************************************************************************/
1193 : /* VSIMultipartUploadGetCapabilities() */
1194 : /************************************************************************/
1195 :
1196 : /**
1197 : * \brief Return capabilities for multiple part file upload.
1198 : *
1199 : * @param pszFilename Filename, or virtual file system prefix, onto which
1200 : * capabilities should apply.
1201 : * @param[out] pbNonSequentialUploadSupported If not null,
1202 : * the pointed value is set if parts can be uploaded in a non-sequential way.
1203 : * @param[out] pbParallelUploadSupported If not null,
1204 : * the pointed value is set if parts can be uploaded in a parallel way.
1205 : * (implies *pbNonSequentialUploadSupported = true)
1206 : * @param[out] pbAbortSupported If not null,
1207 : * the pointed value is set if VSIMultipartUploadAbort() is implemented.
1208 : * @param[out] pnMinPartSize If not null, the pointed value is set to the minimum
1209 : * size of parts (but the last one), in MiB.
1210 : * @param[out] pnMaxPartSize If not null, the pointed value is set to the maximum
1211 : * size of parts, in MiB.
1212 : * @param[out] pnMaxPartCount If not null, the pointed value is set to the
1213 : * maximum number of parts that can be uploaded.
1214 : *
1215 : * @return TRUE in case of success, FALSE otherwise.
1216 : *
1217 : * @since 3.10
1218 : */
1219 7 : int VSIMultipartUploadGetCapabilities(
1220 : const char *pszFilename, int *pbNonSequentialUploadSupported,
1221 : int *pbParallelUploadSupported, int *pbAbortSupported,
1222 : size_t *pnMinPartSize, size_t *pnMaxPartSize, int *pnMaxPartCount)
1223 : {
1224 7 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
1225 :
1226 14 : return poFSHandler->MultipartUploadGetCapabilities(
1227 : pbNonSequentialUploadSupported, pbParallelUploadSupported,
1228 7 : pbAbortSupported, pnMinPartSize, pnMaxPartSize, pnMaxPartCount);
1229 : }
1230 :
1231 : /************************************************************************/
1232 : /* VSIMultipartUploadStart() */
1233 : /************************************************************************/
1234 :
1235 : /**
1236 : * \brief Initiates the upload a (big) file in a piece-wise way.
1237 : *
1238 : * Using this API directly is generally not needed, but in very advanced cases,
1239 : * as VSIFOpenL(..., "wb") + VSIFWriteL(), VSISync(), VSICopyFile() or
1240 : * VSICopyFileRestartable() may be able to leverage it when needed.
1241 : *
1242 : * This is only implemented for the /vsis3/, /vsigs/, /vsiaz/, /vsiadls/ and
1243 : * /vsioss/ virtual file systems.
1244 : *
1245 : * The typical workflow is to do :
1246 : * - VSIMultipartUploadStart()
1247 : * - VSIMultipartUploadAddPart(): several times
1248 : * - VSIMultipartUploadEnd()
1249 : *
1250 : * If VSIMultipartUploadAbort() is supported by the filesystem (VSIMultipartUploadGetCapabilities()
1251 : * can be used to determine it), this function should be called to cancel an
1252 : * upload. This can be needed to avoid extra billing for some cloud storage
1253 : * providers.
1254 : *
1255 : * The following options are supported:
1256 : * <ul>
1257 : * <li>MIME headers such as Content-Type and Content-Encoding
1258 : * are supported for the /vsis3/, /vsigs/, /vsiaz/, /vsiadls/ file systems.</li>
1259 : * </ul>
1260 : *
1261 : * @param pszFilename Filename to create
1262 : * @param papszOptions NULL or null-terminated list of options.
1263 : * @return an upload ID to pass to other VSIMultipartUploadXXXXX() functions,
1264 : * and to free with CPLFree() once done, or nullptr in case of error.
1265 : *
1266 : * @since 3.10
1267 : */
1268 4 : char *VSIMultipartUploadStart(const char *pszFilename,
1269 : CSLConstList papszOptions)
1270 : {
1271 4 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
1272 :
1273 4 : return poFSHandler->MultipartUploadStart(pszFilename, papszOptions);
1274 : }
1275 :
1276 : /************************************************************************/
1277 : /* VSIMultipartUploadAddPart() */
1278 : /************************************************************************/
1279 :
1280 : /**
1281 : * \brief Uploads a new part to a multi-part uploaded file.
1282 : *
1283 : * Cf VSIMultipartUploadStart().
1284 : *
1285 : * VSIMultipartUploadGetCapabilities() returns hints on the constraints that
1286 : * apply to the upload, in terms of minimum/maximum size of each part, maximum
1287 : * number of parts, and whether non-sequential or parallel uploads are
1288 : * supported.
1289 : *
1290 : * @param pszFilename Filename to which to append the new part. Should be the
1291 : * same as the one used for VSIMultipartUploadStart()
1292 : * @param pszUploadId Value returned by VSIMultipartUploadStart()
1293 : * @param nPartNumber Part number, starting at 1.
1294 : * @param nFileOffset Offset within the file at which (starts at 0) the passed
1295 : * data starts.
1296 : * @param pData Pointer to an array of nDataLength bytes.
1297 : * @param nDataLength Size in bytes of pData.
1298 : * @param papszOptions Unused. Should be nullptr.
1299 : *
1300 : * @return a part identifier that must be passed into the apszPartIds[] array of
1301 : * VSIMultipartUploadEnd(), and to free with CPLFree() once done, or nullptr in
1302 : * case of error.
1303 : *
1304 : * @since 3.10
1305 : */
1306 5 : char *VSIMultipartUploadAddPart(const char *pszFilename,
1307 : const char *pszUploadId, int nPartNumber,
1308 : vsi_l_offset nFileOffset, const void *pData,
1309 : size_t nDataLength, CSLConstList papszOptions)
1310 : {
1311 5 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
1312 :
1313 5 : return poFSHandler->MultipartUploadAddPart(pszFilename, pszUploadId,
1314 : nPartNumber, nFileOffset, pData,
1315 5 : nDataLength, papszOptions);
1316 : }
1317 :
1318 : /************************************************************************/
1319 : /* VSIMultipartUploadEnd() */
1320 : /************************************************************************/
1321 :
1322 : /**
1323 : * \brief Completes a multi-part file upload.
1324 : *
1325 : * Cf VSIMultipartUploadStart().
1326 : *
1327 : * @param pszFilename Filename for which multipart upload should be completed.
1328 : * Should be the same as the one used for
1329 : * VSIMultipartUploadStart()
1330 : * @param pszUploadId Value returned by VSIMultipartUploadStart()
1331 : * @param nPartIdsCount Number of parts, and size of apszPartIds
1332 : * @param apszPartIds Array of part identifiers (as returned by
1333 : * VSIMultipartUploadAddPart()), that must be ordered in
1334 : * the sequential order of parts, and of size nPartIdsCount.
1335 : * @param nTotalSize Total size of the file in bytes (must be equal to the sum
1336 : * of nDataLength passed to VSIMultipartUploadAddPart())
1337 : * @param papszOptions Unused. Should be nullptr.
1338 : *
1339 : * @return TRUE in case of success, FALSE in case of failure.
1340 : *
1341 : * @since 3.10
1342 : */
1343 5 : int VSIMultipartUploadEnd(const char *pszFilename, const char *pszUploadId,
1344 : size_t nPartIdsCount, const char *const *apszPartIds,
1345 : vsi_l_offset nTotalSize, CSLConstList papszOptions)
1346 : {
1347 5 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
1348 :
1349 10 : return poFSHandler->MultipartUploadEnd(pszFilename, pszUploadId,
1350 : nPartIdsCount, apszPartIds,
1351 5 : nTotalSize, papszOptions);
1352 : }
1353 :
1354 : /************************************************************************/
1355 : /* VSIMultipartUploadAbort() */
1356 : /************************************************************************/
1357 :
1358 : /**
1359 : * \brief Aborts a multi-part file upload.
1360 : *
1361 : * Cf VSIMultipartUploadStart().
1362 : *
1363 : * This function is not implemented for all virtual file systems.
1364 : * Use VSIMultipartUploadGetCapabilities() to determine if it is supported.
1365 : *
1366 : * This can be needed to avoid extra billing for some cloud storage providers.
1367 : *
1368 : * @param pszFilename Filename for which multipart upload should be completed.
1369 : * Should be the same as the one used for
1370 : * VSIMultipartUploadStart()
1371 : * @param pszUploadId Value returned by VSIMultipartUploadStart()
1372 : * @param papszOptions Unused. Should be nullptr.
1373 : *
1374 : * @return TRUE in case of success, FALSE in case of failure.
1375 : *
1376 : * @since 3.10
1377 : */
1378 6 : int VSIMultipartUploadAbort(const char *pszFilename, const char *pszUploadId,
1379 : CSLConstList papszOptions)
1380 : {
1381 6 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
1382 :
1383 12 : return poFSHandler->MultipartUploadAbort(pszFilename, pszUploadId,
1384 6 : papszOptions);
1385 : }
1386 :
1387 : #ifndef DOXYGEN_SKIP
1388 :
1389 : /************************************************************************/
1390 : /* MultipartUploadGetCapabilities() */
1391 : /************************************************************************/
1392 :
1393 2 : bool VSIFilesystemHandler::MultipartUploadGetCapabilities(int *, int *, int *,
1394 : size_t *, size_t *,
1395 : int *)
1396 : {
1397 2 : CPLError(
1398 : CE_Failure, CPLE_NotSupported,
1399 : "MultipartUploadGetCapabilities() not supported by this file system");
1400 2 : return false;
1401 : }
1402 :
1403 : /************************************************************************/
1404 : /* MultipartUploadStart() */
1405 : /************************************************************************/
1406 :
1407 1 : char *VSIFilesystemHandler::MultipartUploadStart(const char *, CSLConstList)
1408 : {
1409 1 : CPLError(CE_Failure, CPLE_NotSupported,
1410 : "MultipartUploadStart() not supported by this file system");
1411 1 : return nullptr;
1412 : }
1413 :
1414 : /************************************************************************/
1415 : /* MultipartUploadAddPart() */
1416 : /************************************************************************/
1417 :
1418 1 : char *VSIFilesystemHandler::MultipartUploadAddPart(const char *, const char *,
1419 : int, vsi_l_offset,
1420 : const void *, size_t,
1421 : CSLConstList)
1422 : {
1423 1 : CPLError(CE_Failure, CPLE_NotSupported,
1424 : "MultipartUploadAddPart() not supported by this file system");
1425 1 : return nullptr;
1426 : }
1427 :
1428 : /************************************************************************/
1429 : /* MultipartUploadEnd() */
1430 : /************************************************************************/
1431 :
1432 1 : bool VSIFilesystemHandler::MultipartUploadEnd(const char *, const char *,
1433 : size_t, const char *const *,
1434 : vsi_l_offset, CSLConstList)
1435 : {
1436 1 : CPLError(CE_Failure, CPLE_NotSupported,
1437 : "MultipartUploadEnd() not supported by this file system");
1438 1 : return FALSE;
1439 : }
1440 :
1441 : /************************************************************************/
1442 : /* MultipartUploadAbort() */
1443 : /************************************************************************/
1444 :
1445 1 : bool VSIFilesystemHandler::MultipartUploadAbort(const char *, const char *,
1446 : CSLConstList)
1447 : {
1448 1 : CPLError(CE_Failure, CPLE_NotSupported,
1449 : "MultipartUploadAbort() not supported by this file system");
1450 1 : return FALSE;
1451 : }
1452 :
1453 : #endif
1454 :
1455 : /************************************************************************/
1456 : /* VSIAbortPendingUploads() */
1457 : /************************************************************************/
1458 :
1459 : /**
1460 : * \brief Abort all ongoing multi-part uploads.
1461 : *
1462 : * Abort ongoing multi-part uploads on AWS S3 and Google Cloud Storage. This
1463 : * can be used in case a process doing such uploads was killed in a unclean way.
1464 : *
1465 : * This can be needed to avoid extra billing for some cloud storage providers.
1466 : *
1467 : * Without effect on other virtual file systems.
1468 : *
1469 : * VSIMultipartUploadAbort() can also be used to cancel a given upload, if the
1470 : * upload ID is known.
1471 : *
1472 : * @param pszFilename filename or prefix of a directory into which multipart
1473 : * uploads must be aborted. This can be the root directory of a bucket. UTF-8
1474 : * encoded.
1475 : *
1476 : * @return TRUE on success or FALSE on an error.
1477 : * @since GDAL 3.4
1478 : */
1479 :
1480 1 : int VSIAbortPendingUploads(const char *pszFilename)
1481 : {
1482 1 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
1483 :
1484 1 : return poFSHandler->AbortPendingUploads(pszFilename);
1485 : }
1486 :
1487 : /************************************************************************/
1488 : /* VSIRmdir() */
1489 : /************************************************************************/
1490 :
1491 : /**
1492 : * \brief Delete a directory.
1493 : *
1494 : * Deletes a directory object from the file system. On some systems
1495 : * the directory must be empty before it can be deleted.
1496 : *
1497 : * This method goes through the VSIFileHandler virtualization and may
1498 : * work on unusual filesystems such as in memory.
1499 : *
1500 : * Analog of the POSIX rmdir() function.
1501 : *
1502 : * @param pszDirname the path of the directory to be deleted. UTF-8 encoded.
1503 : *
1504 : * @return 0 on success or -1 on an error.
1505 : */
1506 :
1507 170 : int VSIRmdir(const char *pszDirname)
1508 :
1509 : {
1510 170 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszDirname);
1511 :
1512 170 : return poFSHandler->Rmdir(pszDirname);
1513 : }
1514 :
1515 : /************************************************************************/
1516 : /* VSIRmdirRecursive() */
1517 : /************************************************************************/
1518 :
1519 : /**
1520 : * \brief Delete a directory recursively
1521 : *
1522 : * Deletes a directory object and its content from the file system.
1523 : *
1524 : * Starting with GDAL 3.1, /vsis3/ has an efficient implementation of this
1525 : * function.
1526 : * Starting with GDAL 3.4, /vsigs/ has an efficient implementation of this
1527 : * function, provided that OAuth2 authentication is used.
1528 : *
1529 : * @return 0 on success or -1 on an error.
1530 : */
1531 :
1532 5647 : int VSIRmdirRecursive(const char *pszDirname)
1533 : {
1534 5647 : if (pszDirname == nullptr || pszDirname[0] == '\0' ||
1535 5647 : strncmp("/", pszDirname, 2) == 0)
1536 : {
1537 0 : return -1;
1538 : }
1539 5647 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszDirname);
1540 5647 : return poFSHandler->RmdirRecursive(pszDirname);
1541 : }
1542 :
1543 : /************************************************************************/
1544 : /* VSIStatL() */
1545 : /************************************************************************/
1546 :
1547 : /**
1548 : * \brief Get filesystem object info.
1549 : *
1550 : * Fetches status information about a filesystem object (file, directory, etc).
1551 : * The returned information is placed in the VSIStatBufL structure. For
1552 : * portability, only use the st_size (size in bytes) and st_mode (file type).
1553 : * This method is similar to VSIStat(), but will work on large files on
1554 : * systems where this requires special calls.
1555 : *
1556 : * This method goes through the VSIFileHandler virtualization and may
1557 : * work on unusual filesystems such as in memory.
1558 : *
1559 : * Analog of the POSIX stat() function.
1560 : *
1561 : * @param pszFilename the path of the filesystem object to be queried.
1562 : * UTF-8 encoded.
1563 : * @param psStatBuf the structure to load with information.
1564 : *
1565 : * @return 0 on success or -1 on an error.
1566 : */
1567 :
1568 428008 : int VSIStatL(const char *pszFilename, VSIStatBufL *psStatBuf)
1569 :
1570 : {
1571 428008 : return VSIStatExL(pszFilename, psStatBuf, 0);
1572 : }
1573 :
1574 : /************************************************************************/
1575 : /* VSIStatExL() */
1576 : /************************************************************************/
1577 :
1578 : /**
1579 : * \brief Get filesystem object info.
1580 : *
1581 : * Fetches status information about a filesystem object (file, directory, etc).
1582 : * The returned information is placed in the VSIStatBufL structure. For
1583 : * portability, only use the st_size (size in bytes) and st_mode (file type).
1584 : * This method is similar to VSIStat(), but will work on large files on
1585 : * systems where this requires special calls.
1586 : *
1587 : * This method goes through the VSIFileHandler virtualization and may
1588 : * work on unusual filesystems such as in memory.
1589 : *
1590 : * Analog of the POSIX stat() function, with an extra parameter to
1591 : * specify which information is needed, which offers a potential for
1592 : * speed optimizations on specialized and potentially slow virtual
1593 : * filesystem objects (/vsigzip/, /vsicurl/)
1594 : *
1595 : * @param pszFilename the path of the filesystem object to be queried.
1596 : * UTF-8 encoded.
1597 : * @param psStatBuf the structure to load with information.
1598 : * @param nFlags 0 to get all information, or VSI_STAT_EXISTS_FLAG,
1599 : * VSI_STAT_NATURE_FLAG, VSI_STAT_SIZE_FLAG,
1600 : * VSI_STAT_SET_ERROR_FLAG, VSI_STAT_CACHE_ONLY or a combination of those to get
1601 : * partial info.
1602 : *
1603 : * @return 0 on success or -1 on an error.
1604 : *
1605 : */
1606 :
1607 845546 : int VSIStatExL(const char *pszFilename, VSIStatBufL *psStatBuf, int nFlags)
1608 :
1609 : {
1610 845546 : char szAltPath[4] = {'\0'};
1611 :
1612 : // Enable to work on "C:" as if it were "C:\".
1613 845546 : if (pszFilename[0] != '\0' && pszFilename[1] == ':' &&
1614 20 : pszFilename[2] == '\0')
1615 : {
1616 0 : szAltPath[0] = pszFilename[0];
1617 0 : szAltPath[1] = pszFilename[1];
1618 0 : szAltPath[2] = '\\';
1619 0 : szAltPath[3] = '\0';
1620 :
1621 0 : pszFilename = szAltPath;
1622 : }
1623 :
1624 845546 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
1625 :
1626 845542 : if (nFlags == 0)
1627 432015 : nFlags =
1628 : VSI_STAT_EXISTS_FLAG | VSI_STAT_NATURE_FLAG | VSI_STAT_SIZE_FLAG;
1629 :
1630 1691090 : return poFSHandler->Stat(pszFilename, psStatBuf, nFlags);
1631 : }
1632 :
1633 : /************************************************************************/
1634 : /* VSIGetFileMetadata() */
1635 : /************************************************************************/
1636 :
1637 : /**
1638 : * \brief Get metadata on files.
1639 : *
1640 : * Implemented currently only for network-like filesystems, or starting
1641 : * with GDAL 3.7 for /vsizip/
1642 : *
1643 : * Starting with GDAL 3.11, calling it with pszFilename being the root of a
1644 : * /vsigs/ bucket and pszDomain == nullptr, and when authenticated through
1645 : * OAuth2, will result in returning the result of a "Buckets: get"
1646 : * operation (https://cloud.google.com/storage/docs/json_api/v1/buckets/get),
1647 : * with the keys of the top-level JSON document as keys of the key=value pairs
1648 : * returned by this function.
1649 : *
1650 : * @param pszFilename the path of the filesystem object to be queried.
1651 : * UTF-8 encoded.
1652 : * @param pszDomain Metadata domain to query. Depends on the file system.
1653 : * The following ones are supported:
1654 : * <ul>
1655 : * <li>HEADERS: to get HTTP headers for network-like filesystems (/vsicurl/,
1656 : * /vsis3/, /vsgis/, etc)</li>
1657 : * <li>TAGS:
1658 : * <ul>
1659 : * <li>/vsis3/: to get S3 Object tagging information</li>
1660 : * <li>/vsiaz/: to get blob tags. Refer to
1661 : * https://docs.microsoft.com/en-us/rest/api/storageservices/get-blob-tags
1662 : * </li>
1663 : * </ul>
1664 : * </li>
1665 : * <li>STATUS: specific to /vsiadls/: returns all system defined properties for
1666 : * a path (seems in practice to be a subset of HEADERS)</li> <li>ACL: specific
1667 : * to /vsiadls/ and /vsigs/: returns the access control list for a path. For
1668 : * /vsigs/, a single XML=xml_content string is returned. Refer to
1669 : * https://cloud.google.com/storage/docs/xml-api/get-object-acls
1670 : * </li>
1671 : * <li>METADATA: specific to /vsiaz/: to get blob metadata. Refer to
1672 : * https://docs.microsoft.com/en-us/rest/api/storageservices/get-blob-metadata.
1673 : * Note: this will be a subset of what pszDomain=HEADERS returns</li>
1674 : * <li>ZIP: specific to /vsizip/: to obtain ZIP specific metadata, in particular
1675 : * if a file is SOZIP-enabled (SOZIP_VALID=YES)</li>
1676 : * </ul>
1677 : * @param papszOptions Unused. Should be set to NULL.
1678 : *
1679 : * @return a NULL-terminated list of key=value strings, to be freed with
1680 : * CSLDestroy() or NULL in case of error / empty list.
1681 : *
1682 : * @since GDAL 3.1.0
1683 : */
1684 :
1685 133 : char **VSIGetFileMetadata(const char *pszFilename, const char *pszDomain,
1686 : CSLConstList papszOptions)
1687 : {
1688 133 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
1689 133 : return poFSHandler->GetFileMetadata(pszFilename, pszDomain, papszOptions);
1690 : }
1691 :
1692 : /************************************************************************/
1693 : /* VSISetFileMetadata() */
1694 : /************************************************************************/
1695 :
1696 : /**
1697 : * \brief Set metadata on files.
1698 : *
1699 : * Implemented currently only for /vsis3/, /vsigs/, /vsiaz/ and /vsiadls/
1700 : *
1701 : * @param pszFilename the path of the filesystem object to be set.
1702 : * UTF-8 encoded.
1703 : * @param papszMetadata NULL-terminated list of key=value strings.
1704 : * @param pszDomain Metadata domain to set. Depends on the file system.
1705 : * The following are supported:
1706 : * <ul>
1707 : * <li>HEADERS: specific to /vsis3/ and /vsigs/: to set HTTP headers, such as
1708 : * "Content-Type", or other file system specific header.
1709 : * For /vsigs/, this also includes: x-goog-meta-{key}={value}. Note that you
1710 : * should specify all metadata to be set, as existing metadata will be
1711 : * overridden.
1712 : * </li>
1713 : * <li>TAGS: Content of papszMetadata should be KEY=VALUE pairs.
1714 : * <ul>
1715 : * <li>/vsis3/: to set S3 Object tagging information</li>
1716 : * <li>/vsiaz/: to set blob tags. Refer to
1717 : * https://docs.microsoft.com/en-us/rest/api/storageservices/set-blob-tags.
1718 : * Note: storageV2 must be enabled on the account</li>
1719 : * </ul>
1720 : * </li>
1721 : * <li>PROPERTIES:
1722 : * <ul>
1723 : * <li>to /vsiaz/: to set properties. Refer to
1724 : * https://docs.microsoft.com/en-us/rest/api/storageservices/set-blob-properties.</li>
1725 : * <li>to /vsiadls/: to set properties. Refer to
1726 : * https://docs.microsoft.com/en-us/rest/api/storageservices/datalakestoragegen2/path/update
1727 : * for headers valid for action=setProperties.</li>
1728 : * </ul>
1729 : * </li>
1730 : * <li>ACL: specific to /vsiadls/ and /vsigs/: to set access control list.
1731 : * For /vsiadls/, refer to
1732 : * https://docs.microsoft.com/en-us/rest/api/storageservices/datalakestoragegen2/path/update
1733 : * for headers valid for action=setAccessControl or setAccessControlRecursive.
1734 : * In setAccessControlRecursive, x-ms-acl must be specified in papszMetadata.
1735 : * For /vsigs/, refer to
1736 : * https://cloud.google.com/storage/docs/xml-api/put-object-acls. A single
1737 : * XML=xml_content string should be specified as in papszMetadata.
1738 : * </li>
1739 : * <li>METADATA: specific to /vsiaz/: to set blob metadata. Refer to
1740 : * https://docs.microsoft.com/en-us/rest/api/storageservices/set-blob-metadata.
1741 : * Content of papszMetadata should be strings in the form
1742 : * x-ms-meta-name=value</li>
1743 : * </ul>
1744 : * @param papszOptions NULL or NULL terminated list of options.
1745 : * For /vsiadls/ and pszDomain=ACL, "RECURSIVE=TRUE" can be
1746 : * set to set the access control list recursively. When
1747 : * RECURSIVE=TRUE is set, MODE should also be set to one of
1748 : * "set", "modify" or "remove".
1749 : *
1750 : * @return TRUE in case of success.
1751 : *
1752 : * @since GDAL 3.1.0
1753 : */
1754 :
1755 17 : int VSISetFileMetadata(const char *pszFilename, CSLConstList papszMetadata,
1756 : const char *pszDomain, CSLConstList papszOptions)
1757 : {
1758 17 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
1759 34 : return poFSHandler->SetFileMetadata(pszFilename, papszMetadata, pszDomain,
1760 17 : papszOptions)
1761 17 : ? 1
1762 17 : : 0;
1763 : }
1764 :
1765 : /************************************************************************/
1766 : /* VSIIsCaseSensitiveFS() */
1767 : /************************************************************************/
1768 :
1769 : /**
1770 : * \brief Returns if the filenames of the filesystem are case sensitive.
1771 : *
1772 : * This method retrieves to which filesystem belongs the passed filename
1773 : * and return TRUE if the filenames of that filesystem are case sensitive.
1774 : *
1775 : * Currently, this will return FALSE only for Windows real filenames. Other
1776 : * VSI virtual filesystems are case sensitive.
1777 : *
1778 : * This methods avoid ugly \#ifndef _WIN32 / \#endif code, that is wrong when
1779 : * dealing with virtual filenames.
1780 : *
1781 : * @param pszFilename the path of the filesystem object to be tested.
1782 : * UTF-8 encoded.
1783 : *
1784 : * @return TRUE if the filenames of the filesystem are case sensitive.
1785 : *
1786 : */
1787 :
1788 29890 : int VSIIsCaseSensitiveFS(const char *pszFilename)
1789 : {
1790 29890 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
1791 :
1792 29890 : return poFSHandler->IsCaseSensitive(pszFilename);
1793 : }
1794 :
1795 : /************************************************************************/
1796 : /* VSISupportsSparseFiles() */
1797 : /************************************************************************/
1798 :
1799 : /**
1800 : * \brief Returns if the filesystem supports sparse files.
1801 : *
1802 : * Only supported on Linux (and no other Unix derivatives) and
1803 : * Windows. On Linux, the answer depends on a few hardcoded
1804 : * signatures for common filesystems. Other filesystems will be
1805 : * considered as not supporting sparse files.
1806 : *
1807 : * @param pszPath the path of the filesystem object to be tested.
1808 : * UTF-8 encoded.
1809 : *
1810 : * @return TRUE if the file system is known to support sparse files. FALSE may
1811 : * be returned both in cases where it is known to not support them,
1812 : * or when it is unknown.
1813 : *
1814 : */
1815 :
1816 2 : int VSISupportsSparseFiles(const char *pszPath)
1817 : {
1818 2 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszPath);
1819 :
1820 2 : return poFSHandler->SupportsSparseFiles(pszPath);
1821 : }
1822 :
1823 : /************************************************************************/
1824 : /* VSIIsLocal() */
1825 : /************************************************************************/
1826 :
1827 : /**
1828 : * \brief Returns if the file/filesystem is "local".
1829 : *
1830 : * The concept of local is mostly by opposition with a network / remote
1831 : * file system whose access time can be long.
1832 : *
1833 : * /vsimem/ is considered to be a local file system, although a non-persistent
1834 : * one.
1835 : *
1836 : * @param pszPath the path of the filesystem object to be tested.
1837 : * UTF-8 encoded.
1838 : *
1839 : * @return TRUE or FALSE
1840 : *
1841 : * @since GDAL 3.6
1842 : */
1843 :
1844 3790 : bool VSIIsLocal(const char *pszPath)
1845 : {
1846 3790 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszPath);
1847 :
1848 3790 : return poFSHandler->IsLocal(pszPath);
1849 : }
1850 :
1851 : /************************************************************************/
1852 : /* VSIGetCanonicalFilename() */
1853 : /************************************************************************/
1854 :
1855 : /**
1856 : * \brief Returns the canonical filename.
1857 : *
1858 : * May be implemented by case-insensitive filesystems
1859 : * (currently Win32 and MacOSX) to return the filename with its actual case
1860 : * (i.e. the one that would be used when listing the content of the directory).
1861 : *
1862 : * @param pszPath UTF-8 encoded path
1863 : *
1864 : * @return UTF-8 encoded string, to free with VSIFree()
1865 : *
1866 : * @since GDAL 3.8
1867 : */
1868 :
1869 256 : char *VSIGetCanonicalFilename(const char *pszPath)
1870 : {
1871 256 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszPath);
1872 :
1873 256 : return CPLStrdup(poFSHandler->GetCanonicalFilename(pszPath).c_str());
1874 : }
1875 :
1876 : /************************************************************************/
1877 : /* VSISupportsSequentialWrite() */
1878 : /************************************************************************/
1879 :
1880 : /**
1881 : * \brief Returns if the filesystem supports sequential write.
1882 : *
1883 : * @param pszPath the path of the filesystem object to be tested.
1884 : * UTF-8 encoded.
1885 : * @param bAllowLocalTempFile whether the file system is allowed to use a
1886 : * local temporary file before uploading to the target location.
1887 : *
1888 : * @return TRUE or FALSE
1889 : *
1890 : * @since GDAL 3.6
1891 : */
1892 :
1893 122 : bool VSISupportsSequentialWrite(const char *pszPath, bool bAllowLocalTempFile)
1894 : {
1895 122 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszPath);
1896 :
1897 122 : return poFSHandler->SupportsSequentialWrite(pszPath, bAllowLocalTempFile);
1898 : }
1899 :
1900 : /************************************************************************/
1901 : /* VSISupportsRandomWrite() */
1902 : /************************************************************************/
1903 :
1904 : /**
1905 : * \brief Returns if the filesystem supports random write.
1906 : *
1907 : * @param pszPath the path of the filesystem object to be tested.
1908 : * UTF-8 encoded.
1909 : * @param bAllowLocalTempFile whether the file system is allowed to use a
1910 : * local temporary file before uploading to the target location.
1911 : *
1912 : * @return TRUE or FALSE
1913 : *
1914 : * @since GDAL 3.6
1915 : */
1916 :
1917 369 : bool VSISupportsRandomWrite(const char *pszPath, bool bAllowLocalTempFile)
1918 : {
1919 369 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszPath);
1920 :
1921 369 : return poFSHandler->SupportsRandomWrite(pszPath, bAllowLocalTempFile);
1922 : }
1923 :
1924 : /************************************************************************/
1925 : /* VSIHasOptimizedReadMultiRange() */
1926 : /************************************************************************/
1927 :
1928 : /**
1929 : * \brief Returns if the filesystem supports efficient multi-range reading.
1930 : *
1931 : * Currently only returns TRUE for /vsicurl/ and derived file systems.
1932 : *
1933 : * @param pszPath the path of the filesystem object to be tested.
1934 : * UTF-8 encoded.
1935 : *
1936 : * @return TRUE if the file system is known to have an efficient multi-range
1937 : * reading.
1938 : *
1939 : */
1940 :
1941 15665 : int VSIHasOptimizedReadMultiRange(const char *pszPath)
1942 : {
1943 15665 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszPath);
1944 :
1945 15665 : return poFSHandler->HasOptimizedReadMultiRange(pszPath);
1946 : }
1947 :
1948 : /************************************************************************/
1949 : /* VSIGetActualURL() */
1950 : /************************************************************************/
1951 :
1952 : /**
1953 : * \brief Returns the actual URL of a supplied filename.
1954 : *
1955 : * Currently only returns a non-NULL value for network-based virtual file
1956 : * systems. For example "/vsis3/bucket/filename" will be expanded as
1957 : * "https://bucket.s3.amazon.com/filename"
1958 : *
1959 : * Note that the lifetime of the returned string, is short, and may be
1960 : * invalidated by any following GDAL functions.
1961 : *
1962 : * @param pszFilename the path of the filesystem object. UTF-8 encoded.
1963 : *
1964 : * @return the actual URL corresponding to the supplied filename, or NULL.
1965 : * Should not be freed.
1966 : *
1967 : */
1968 :
1969 9 : const char *VSIGetActualURL(const char *pszFilename)
1970 : {
1971 9 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
1972 :
1973 9 : return poFSHandler->GetActualURL(pszFilename);
1974 : }
1975 :
1976 : /************************************************************************/
1977 : /* VSIGetSignedURL() */
1978 : /************************************************************************/
1979 :
1980 : /**
1981 : * \brief Returns a signed URL of a supplied filename.
1982 : *
1983 : * Currently only returns a non-NULL value for /vsis3/, /vsigs/, /vsiaz/ and
1984 : * /vsioss/ For example "/vsis3/bucket/filename" will be expanded as
1985 : * "https://bucket.s3.amazon.com/filename?X-Amz-Algorithm=AWS4-HMAC-SHA256..."
1986 : * Configuration options that apply for file opening (typically to provide
1987 : * credentials), and are returned by VSIGetFileSystemOptions(), are also valid
1988 : * in that context.
1989 : *
1990 : * @param pszFilename the path of the filesystem object. UTF-8 encoded.
1991 : * @param papszOptions list of options, or NULL. Depend on file system handler.
1992 : * For /vsis3/, /vsigs/, /vsiaz/ and /vsioss/, the following options are
1993 : * supported: <ul> <li>START_DATE=YYMMDDTHHMMSSZ: date and time in UTC following
1994 : * ISO 8601 standard, corresponding to the start of validity of the URL. If not
1995 : * specified, current date time.</li> <li>EXPIRATION_DELAY=number_of_seconds:
1996 : * number between 1 and 604800 (seven days) for the validity of the signed URL.
1997 : * Defaults to 3600 (one hour)</li> <li>VERB=GET/HEAD/DELETE/PUT/POST: HTTP VERB
1998 : * for which the request will be used. Default to GET.</li>
1999 : * </ul>
2000 : *
2001 : * /vsiaz/ supports additional options:
2002 : * <ul>
2003 : * <li>SIGNEDIDENTIFIER=value: to relate the given shared access signature
2004 : * to a corresponding stored access policy.</li>
2005 : * <li>SIGNEDPERMISSIONS=r|w: permissions associated with the shared access
2006 : * signature. Normally deduced from VERB.</li>
2007 : * </ul>
2008 : *
2009 : * @return a signed URL, or NULL. Should be freed with CPLFree().
2010 : */
2011 :
2012 25 : char *VSIGetSignedURL(const char *pszFilename, CSLConstList papszOptions)
2013 : {
2014 25 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
2015 :
2016 25 : return poFSHandler->GetSignedURL(pszFilename, papszOptions);
2017 : }
2018 :
2019 : /************************************************************************/
2020 : /* VSIFOpenL() */
2021 : /************************************************************************/
2022 :
2023 : /**
2024 : * \brief Open file.
2025 : *
2026 : * This function opens a file with the desired access. Large files (larger
2027 : * than 2GB) should be supported. Binary access is always implied and
2028 : * the "b" does not need to be included in the pszAccess string.
2029 : *
2030 : * *NOT* a standard C library FILE *, and cannot be used with any functions
2031 : * other than the "VSI*L" family of functions. They aren't "real" FILE objects.
2032 : *
2033 : * On windows it is possible to define the configuration option
2034 : * GDAL_FILE_IS_UTF8 to have pszFilename treated as being in the local
2035 : * encoding instead of UTF-8, restoring the pre-1.8.0 behavior of VSIFOpenL().
2036 : *
2037 : * This method goes through the VSIFileHandler virtualization and may
2038 : * work on unusual filesystems such as in memory.
2039 : *
2040 : * Analog of the POSIX fopen() function.
2041 : *
2042 : * @param pszFilename the file to open. UTF-8 encoded.
2043 : * @param pszAccess access requested (i.e. "r", "r+", "w")
2044 : *
2045 : * @return NULL on failure, or the file handle.
2046 : */
2047 :
2048 262752 : VSILFILE *VSIFOpenL(const char *pszFilename, const char *pszAccess)
2049 :
2050 : {
2051 262752 : return VSIFOpenExL(pszFilename, pszAccess, false);
2052 : }
2053 :
2054 : /************************************************************************/
2055 : /* Open() */
2056 : /************************************************************************/
2057 :
2058 : #ifndef DOXYGEN_SKIP
2059 :
2060 : VSIVirtualHandleUniquePtr
2061 29755 : VSIFilesystemHandler::OpenStatic(const char *pszFilename, const char *pszAccess,
2062 : bool bSetError, CSLConstList papszOptions)
2063 : {
2064 29755 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
2065 :
2066 29755 : return poFSHandler->Open(pszFilename, pszAccess, bSetError, papszOptions);
2067 : }
2068 :
2069 : /************************************************************************/
2070 : /* CopyFile() */
2071 : /************************************************************************/
2072 :
2073 2307 : int VSIFilesystemHandler::CopyFile(const char *pszSource, const char *pszTarget,
2074 : VSILFILE *fpSource, vsi_l_offset nSourceSize,
2075 : CSLConstList papszOptions,
2076 : GDALProgressFunc pProgressFunc,
2077 : void *pProgressData)
2078 : {
2079 2307 : VSIVirtualHandleUniquePtr poFileHandleAutoClose;
2080 2307 : if (!fpSource)
2081 : {
2082 2290 : CPLAssert(pszSource);
2083 2290 : fpSource = VSIFOpenExL(pszSource, "rb", TRUE);
2084 2290 : if (!fpSource)
2085 : {
2086 1 : CPLError(CE_Failure, CPLE_FileIO, "Cannot open %s", pszSource);
2087 1 : return -1;
2088 : }
2089 2289 : poFileHandleAutoClose.reset(fpSource);
2090 : }
2091 2306 : if (nSourceSize == static_cast<vsi_l_offset>(-1) &&
2092 3 : pProgressFunc != nullptr && pszSource != nullptr)
2093 : {
2094 : VSIStatBufL sStat;
2095 3 : if (VSIStatL(pszSource, &sStat) == 0)
2096 : {
2097 3 : nSourceSize = sStat.st_size;
2098 : }
2099 : }
2100 :
2101 2306 : VSILFILE *fpOut = VSIFOpenEx2L(pszTarget, "wb", TRUE, papszOptions);
2102 2306 : if (!fpOut)
2103 : {
2104 3 : CPLError(CE_Failure, CPLE_FileIO, "Cannot create %s", pszTarget);
2105 3 : return -1;
2106 : }
2107 :
2108 4606 : CPLString osMsg;
2109 2303 : if (pszSource)
2110 2301 : osMsg.Printf("Copying of %s", pszSource);
2111 : else
2112 2 : pszSource = "(unknown filename)";
2113 :
2114 2303 : int ret = 0;
2115 2303 : constexpr size_t nBufferSize = 10 * 4096;
2116 2303 : std::vector<GByte> abyBuffer(nBufferSize, 0);
2117 2303 : GUIntBig nOffset = 0;
2118 : while (true)
2119 : {
2120 2619 : const size_t nRead = VSIFReadL(&abyBuffer[0], 1, nBufferSize, fpSource);
2121 2619 : if (nRead < nBufferSize && VSIFErrorL(fpSource))
2122 : {
2123 7 : CPLError(
2124 : CE_Failure, CPLE_FileIO,
2125 : "Copying of %s to %s failed: error while reading source file",
2126 : pszSource, pszTarget);
2127 7 : ret = -1;
2128 7 : break;
2129 : }
2130 2612 : if (nRead > 0)
2131 : {
2132 2611 : const size_t nWritten = VSIFWriteL(&abyBuffer[0], 1, nRead, fpOut);
2133 2611 : if (nWritten != nRead)
2134 : {
2135 20 : CPLError(CE_Failure, CPLE_FileIO,
2136 : "Copying of %s to %s failed: error while writing into "
2137 : "target file",
2138 : pszSource, pszTarget);
2139 20 : ret = -1;
2140 20 : break;
2141 : }
2142 2591 : nOffset += nRead;
2143 2640 : if (pProgressFunc &&
2144 98 : !pProgressFunc(
2145 : nSourceSize == 0 ? 1.0
2146 49 : : nSourceSize > 0 &&
2147 : nSourceSize != static_cast<vsi_l_offset>(-1)
2148 98 : ? double(nOffset) / nSourceSize
2149 : : 0.0,
2150 49 : !osMsg.empty() ? osMsg.c_str() : nullptr, pProgressData))
2151 : {
2152 1 : ret = -1;
2153 1 : break;
2154 : }
2155 : }
2156 2591 : if (nRead < nBufferSize)
2157 : {
2158 2275 : break;
2159 : }
2160 316 : }
2161 :
2162 2303 : if (nSourceSize != static_cast<vsi_l_offset>(-1) && nOffset != nSourceSize)
2163 : {
2164 2 : CPLError(CE_Failure, CPLE_FileIO,
2165 : "Copying of %s to %s failed: %" PRIu64 " bytes were copied "
2166 : "whereas %" PRIu64 " were expected",
2167 : pszSource, pszTarget, static_cast<uint64_t>(nOffset),
2168 : static_cast<uint64_t>(nSourceSize));
2169 2 : ret = -1;
2170 : }
2171 :
2172 2303 : if (VSIFCloseL(fpOut) != 0)
2173 : {
2174 1 : ret = -1;
2175 : }
2176 :
2177 2303 : if (ret != 0)
2178 29 : VSIUnlink(pszTarget);
2179 :
2180 2303 : return ret;
2181 : }
2182 :
2183 : /************************************************************************/
2184 : /* CopyFileRestartable() */
2185 : /************************************************************************/
2186 :
2187 2 : int VSIFilesystemHandler::CopyFileRestartable(
2188 : const char *pszSource, const char *pszTarget,
2189 : const char * /* pszInputPayload */, char **ppszOutputPayload,
2190 : CSLConstList papszOptions, GDALProgressFunc pProgressFunc,
2191 : void *pProgressData)
2192 : {
2193 2 : *ppszOutputPayload = nullptr;
2194 2 : return CopyFile(pszSource, pszTarget, nullptr,
2195 : static_cast<vsi_l_offset>(-1), papszOptions, pProgressFunc,
2196 2 : pProgressData);
2197 : }
2198 :
2199 : /************************************************************************/
2200 : /* Sync() */
2201 : /************************************************************************/
2202 :
2203 31 : bool VSIFilesystemHandler::Sync(const char *pszSource, const char *pszTarget,
2204 : const char *const *papszOptions,
2205 : GDALProgressFunc pProgressFunc,
2206 : void *pProgressData, char ***ppapszOutputs)
2207 : {
2208 31 : const char SOURCE_SEP = VSIGetDirectorySeparator(pszSource)[0];
2209 :
2210 31 : if (ppapszOutputs)
2211 : {
2212 0 : *ppapszOutputs = nullptr;
2213 : }
2214 :
2215 : VSIStatBufL sSource;
2216 62 : CPLString osSource(pszSource);
2217 62 : CPLString osSourceWithoutSlash(pszSource);
2218 40 : if (osSourceWithoutSlash.back() == '/' ||
2219 9 : osSourceWithoutSlash.back() == '\\')
2220 : {
2221 22 : osSourceWithoutSlash.pop_back();
2222 : }
2223 31 : if (VSIStatL(osSourceWithoutSlash, &sSource) < 0)
2224 : {
2225 2 : CPLError(CE_Failure, CPLE_FileIO, "%s does not exist", pszSource);
2226 2 : return false;
2227 : }
2228 :
2229 29 : if (VSI_ISDIR(sSource.st_mode))
2230 : {
2231 22 : std::string osTargetDir(pszTarget);
2232 11 : if (osSource.back() != '/' && osSource.back() != '\\')
2233 : {
2234 2 : osTargetDir = CPLFormFilenameSafe(
2235 1 : osTargetDir.c_str(), CPLGetFilename(pszSource), nullptr);
2236 : }
2237 :
2238 : VSIStatBufL sTarget;
2239 11 : bool ret = true;
2240 11 : if (VSIStatL(osTargetDir.c_str(), &sTarget) < 0)
2241 : {
2242 9 : if (VSIMkdirRecursive(osTargetDir.c_str(), 0755) < 0)
2243 : {
2244 1 : CPLError(CE_Failure, CPLE_FileIO, "Cannot create directory %s",
2245 : osTargetDir.c_str());
2246 1 : return false;
2247 : }
2248 : }
2249 :
2250 10 : if (!CPLFetchBool(papszOptions, "STOP_ON_DIR", false))
2251 : {
2252 20 : CPLStringList aosChildOptions(CSLDuplicate(papszOptions));
2253 10 : if (!CPLFetchBool(papszOptions, "RECURSIVE", true))
2254 : {
2255 0 : aosChildOptions.SetNameValue("RECURSIVE", nullptr);
2256 0 : aosChildOptions.AddString("STOP_ON_DIR=TRUE");
2257 : }
2258 :
2259 10 : char **papszSrcFiles = VSIReadDir(osSourceWithoutSlash);
2260 10 : int nFileCount = 0;
2261 27 : for (auto iter = papszSrcFiles; iter && *iter; ++iter)
2262 : {
2263 17 : if (strcmp(*iter, ".") != 0 && strcmp(*iter, "..") != 0)
2264 : {
2265 17 : nFileCount++;
2266 : }
2267 : }
2268 10 : int iFile = 0;
2269 10 : const int nDenom = std::max(1, nFileCount);
2270 27 : for (auto iter = papszSrcFiles; iter && *iter; ++iter, ++iFile)
2271 : {
2272 17 : if (strcmp(*iter, ".") == 0 || strcmp(*iter, "..") == 0)
2273 : {
2274 0 : continue;
2275 : }
2276 : const std::string osSubSource(CPLFormFilenameSafe(
2277 17 : osSourceWithoutSlash.c_str(), *iter, nullptr));
2278 : const std::string osSubTarget(
2279 17 : CPLFormFilenameSafe(osTargetDir.c_str(), *iter, nullptr));
2280 34 : void *pScaledProgress = GDALCreateScaledProgress(
2281 17 : double(iFile) / nDenom, double(iFile + 1) / nDenom,
2282 : pProgressFunc, pProgressData);
2283 17 : ret = Sync((osSubSource + SOURCE_SEP).c_str(),
2284 17 : osSubTarget.c_str(), aosChildOptions.List(),
2285 17 : GDALScaledProgress, pScaledProgress, nullptr);
2286 17 : GDALDestroyScaledProgress(pScaledProgress);
2287 17 : if (!ret)
2288 : {
2289 0 : break;
2290 : }
2291 : }
2292 10 : CSLDestroy(papszSrcFiles);
2293 : }
2294 10 : return ret;
2295 : }
2296 :
2297 : VSIStatBufL sTarget;
2298 36 : std::string osTarget(pszTarget);
2299 18 : if (VSIStatL(osTarget.c_str(), &sTarget) == 0)
2300 : {
2301 4 : bool bTargetIsFile = true;
2302 4 : if (VSI_ISDIR(sTarget.st_mode))
2303 : {
2304 4 : osTarget = CPLFormFilenameSafe(osTarget.c_str(),
2305 2 : CPLGetFilename(pszSource), nullptr);
2306 3 : bTargetIsFile = VSIStatL(osTarget.c_str(), &sTarget) == 0 &&
2307 1 : !CPL_TO_BOOL(VSI_ISDIR(sTarget.st_mode));
2308 : }
2309 4 : if (bTargetIsFile)
2310 : {
2311 3 : if (sSource.st_size == sTarget.st_size &&
2312 3 : sSource.st_mtime == sTarget.st_mtime && sSource.st_mtime != 0)
2313 : {
2314 2 : CPLDebug("VSI",
2315 : "%s and %s have same size and modification "
2316 : "date. Skipping copying",
2317 : osSourceWithoutSlash.c_str(), osTarget.c_str());
2318 2 : return true;
2319 : }
2320 : }
2321 : }
2322 :
2323 16 : VSILFILE *fpIn = VSIFOpenExL(osSourceWithoutSlash, "rb", TRUE);
2324 16 : if (fpIn == nullptr)
2325 : {
2326 0 : CPLError(CE_Failure, CPLE_FileIO, "Cannot open %s",
2327 : osSourceWithoutSlash.c_str());
2328 0 : return false;
2329 : }
2330 :
2331 16 : VSILFILE *fpOut = VSIFOpenExL(osTarget.c_str(), "wb", TRUE);
2332 16 : if (fpOut == nullptr)
2333 : {
2334 2 : CPLError(CE_Failure, CPLE_FileIO, "Cannot create %s", osTarget.c_str());
2335 2 : VSIFCloseL(fpIn);
2336 2 : return false;
2337 : }
2338 :
2339 14 : bool ret = true;
2340 14 : constexpr size_t nBufferSize = 10 * 4096;
2341 28 : std::vector<GByte> abyBuffer(nBufferSize, 0);
2342 14 : GUIntBig nOffset = 0;
2343 14 : CPLString osMsg;
2344 14 : osMsg.Printf("Copying of %s", osSourceWithoutSlash.c_str());
2345 : while (true)
2346 : {
2347 17 : size_t nRead = VSIFReadL(&abyBuffer[0], 1, nBufferSize, fpIn);
2348 17 : size_t nWritten = VSIFWriteL(&abyBuffer[0], 1, nRead, fpOut);
2349 17 : if (nWritten != nRead)
2350 : {
2351 0 : CPLError(CE_Failure, CPLE_FileIO, "Copying of %s to %s failed",
2352 : osSourceWithoutSlash.c_str(), osTarget.c_str());
2353 0 : ret = false;
2354 0 : break;
2355 : }
2356 17 : nOffset += nRead;
2357 17 : if (pProgressFunc && !pProgressFunc(double(nOffset) / sSource.st_size,
2358 : osMsg.c_str(), pProgressData))
2359 : {
2360 0 : ret = false;
2361 0 : break;
2362 : }
2363 17 : if (nRead < nBufferSize)
2364 : {
2365 14 : break;
2366 : }
2367 3 : }
2368 :
2369 14 : VSIFCloseL(fpIn);
2370 14 : if (VSIFCloseL(fpOut) != 0)
2371 : {
2372 0 : ret = false;
2373 : }
2374 14 : return ret;
2375 : }
2376 :
2377 : /************************************************************************/
2378 : /* VSIVirtualHandleOnlyVisibleAtCloseTime() */
2379 : /************************************************************************/
2380 :
2381 : class VSIVirtualHandleOnlyVisibleAtCloseTime final : public VSIProxyFileHandle
2382 : {
2383 : const std::string m_osTargetName;
2384 : const std::string m_osTmpName;
2385 : bool m_bAlreadyClosed = false;
2386 : bool m_bCancelCreation = false;
2387 :
2388 : public:
2389 224 : VSIVirtualHandleOnlyVisibleAtCloseTime(
2390 : VSIVirtualHandleUniquePtr &&nativeHandle,
2391 : const std::string &osTargetName, const std::string &osTmpName)
2392 224 : : VSIProxyFileHandle(std::move(nativeHandle)),
2393 224 : m_osTargetName(osTargetName), m_osTmpName(osTmpName)
2394 : {
2395 224 : }
2396 :
2397 448 : ~VSIVirtualHandleOnlyVisibleAtCloseTime() override
2398 224 : {
2399 224 : VSIVirtualHandleOnlyVisibleAtCloseTime::Close();
2400 448 : }
2401 :
2402 14 : void CancelCreation() override
2403 : {
2404 14 : VSIProxyFileHandle::CancelCreation();
2405 14 : m_bCancelCreation = true;
2406 14 : }
2407 :
2408 : int Close() override;
2409 : };
2410 :
2411 638 : int VSIVirtualHandleOnlyVisibleAtCloseTime::Close()
2412 : {
2413 638 : if (m_bAlreadyClosed)
2414 414 : return 0;
2415 224 : m_bAlreadyClosed = true;
2416 224 : int ret = VSIProxyFileHandle::Close();
2417 224 : if (ret == 0)
2418 : {
2419 224 : if (m_bCancelCreation)
2420 : {
2421 8 : ret = VSIUnlink(m_osTmpName.c_str());
2422 : VSIStatBufL sStatBuf;
2423 8 : if (ret != 0 && VSIStatL(m_osTmpName.c_str(), &sStatBuf) != 0)
2424 4 : ret = 0;
2425 : }
2426 : else
2427 : {
2428 216 : ret = VSIRename(m_osTmpName.c_str(), m_osTargetName.c_str());
2429 : }
2430 : }
2431 224 : return ret;
2432 : }
2433 :
2434 : /************************************************************************/
2435 : /* CreateOnlyVisibleAtCloseTime() */
2436 : /************************************************************************/
2437 :
2438 224 : VSIVirtualHandleUniquePtr VSIFilesystemHandler::CreateOnlyVisibleAtCloseTime(
2439 : const char *pszFilename, bool bEmulationAllowed, CSLConstList papszOptions)
2440 : {
2441 224 : if (!bEmulationAllowed)
2442 0 : return nullptr;
2443 :
2444 672 : const std::string tmpName = std::string(pszFilename).append(".tmp");
2445 : VSIVirtualHandleUniquePtr nativeHandle(
2446 448 : Open(tmpName.c_str(), "wb+", true, papszOptions));
2447 224 : if (!nativeHandle)
2448 0 : return nullptr;
2449 : return VSIVirtualHandleUniquePtr(
2450 448 : std::make_unique<VSIVirtualHandleOnlyVisibleAtCloseTime>(
2451 224 : std::move(nativeHandle), pszFilename, tmpName)
2452 224 : .release());
2453 : }
2454 :
2455 : /************************************************************************/
2456 : /* VSIDIREntry() */
2457 : /************************************************************************/
2458 :
2459 4165 : VSIDIREntry::VSIDIREntry()
2460 : : pszName(nullptr), nMode(0), nSize(0), nMTime(0), bModeKnown(false),
2461 4165 : bSizeKnown(false), bMTimeKnown(false), papszExtra(nullptr)
2462 : {
2463 4165 : }
2464 :
2465 : /************************************************************************/
2466 : /* VSIDIREntry() */
2467 : /************************************************************************/
2468 :
2469 6 : VSIDIREntry::VSIDIREntry(const VSIDIREntry &other)
2470 6 : : pszName(VSIStrdup(other.pszName)), nMode(other.nMode), nSize(other.nSize),
2471 6 : nMTime(other.nMTime), bModeKnown(other.bModeKnown),
2472 6 : bSizeKnown(other.bSizeKnown), bMTimeKnown(other.bMTimeKnown),
2473 6 : papszExtra(CSLDuplicate(other.papszExtra))
2474 : {
2475 6 : }
2476 :
2477 : /************************************************************************/
2478 : /* ~VSIDIREntry() */
2479 : /************************************************************************/
2480 :
2481 8342 : VSIDIREntry::~VSIDIREntry()
2482 : {
2483 4171 : CPLFree(pszName);
2484 4171 : CSLDestroy(papszExtra);
2485 4171 : }
2486 :
2487 : /************************************************************************/
2488 : /* ~VSIDIR() */
2489 : /************************************************************************/
2490 :
2491 3774 : VSIDIR::~VSIDIR()
2492 : {
2493 3774 : }
2494 :
2495 : /************************************************************************/
2496 : /* VSIDIRGeneric */
2497 : /************************************************************************/
2498 :
2499 : namespace
2500 : {
2501 : struct VSIDIRGeneric : public VSIDIR
2502 : {
2503 : CPLString osRootPath{};
2504 : CPLString osBasePath{};
2505 : char **papszContent = nullptr;
2506 : int nRecurseDepth = 0;
2507 : int nPos = 0;
2508 : VSIDIREntry entry{};
2509 : std::vector<VSIDIRGeneric *> aoStackSubDir{};
2510 : VSIFilesystemHandler *poFS = nullptr;
2511 : std::string m_osFilterPrefix{};
2512 :
2513 2721 : explicit VSIDIRGeneric(VSIFilesystemHandler *poFSIn) : poFS(poFSIn)
2514 : {
2515 2721 : }
2516 :
2517 : ~VSIDIRGeneric() override;
2518 :
2519 : const VSIDIREntry *NextDirEntry() override;
2520 :
2521 : VSIDIRGeneric(const VSIDIRGeneric &) = delete;
2522 : VSIDIRGeneric &operator=(const VSIDIRGeneric &) = delete;
2523 : };
2524 :
2525 : /************************************************************************/
2526 : /* ~VSIDIRGeneric() */
2527 : /************************************************************************/
2528 :
2529 8163 : VSIDIRGeneric::~VSIDIRGeneric()
2530 : {
2531 2728 : while (!aoStackSubDir.empty())
2532 : {
2533 7 : delete aoStackSubDir.back();
2534 7 : aoStackSubDir.pop_back();
2535 : }
2536 2721 : CSLDestroy(papszContent);
2537 5442 : }
2538 :
2539 : } // namespace
2540 :
2541 : /************************************************************************/
2542 : /* OpenDir() */
2543 : /************************************************************************/
2544 :
2545 2737 : VSIDIR *VSIFilesystemHandler::OpenDir(const char *pszPath, int nRecurseDepth,
2546 : const char *const *papszOptions)
2547 : {
2548 2737 : char **papszContent = VSIReadDir(pszPath);
2549 : VSIStatBufL sStatL;
2550 3284 : if (papszContent == nullptr &&
2551 547 : (VSIStatL(pszPath, &sStatL) != 0 || !VSI_ISDIR(sStatL.st_mode)))
2552 : {
2553 16 : return nullptr;
2554 : }
2555 2721 : VSIDIRGeneric *dir = new VSIDIRGeneric(this);
2556 2721 : dir->osRootPath = pszPath;
2557 5442 : if (!dir->osRootPath.empty() &&
2558 2721 : (dir->osRootPath.back() == '/' || dir->osRootPath.back() == '\\'))
2559 17 : dir->osRootPath.pop_back();
2560 2721 : dir->nRecurseDepth = nRecurseDepth;
2561 2721 : dir->papszContent = papszContent;
2562 2721 : dir->m_osFilterPrefix = CSLFetchNameValueDef(papszOptions, "PREFIX", "");
2563 2721 : return dir;
2564 : }
2565 :
2566 : /************************************************************************/
2567 : /* NextDirEntry() */
2568 : /************************************************************************/
2569 :
2570 508538 : const VSIDIREntry *VSIDIRGeneric::NextDirEntry()
2571 : {
2572 508538 : const char SEP = VSIGetDirectorySeparator(osRootPath.c_str())[0];
2573 :
2574 508539 : begin:
2575 508539 : if (VSI_ISDIR(entry.nMode) && nRecurseDepth != 0)
2576 : {
2577 1372 : CPLString osCurFile(osRootPath);
2578 1372 : if (!osCurFile.empty())
2579 1372 : osCurFile += SEP;
2580 1372 : osCurFile += entry.pszName;
2581 : auto subdir =
2582 1372 : static_cast<VSIDIRGeneric *>(poFS->VSIFilesystemHandler::OpenDir(
2583 1372 : osCurFile, nRecurseDepth - 1, nullptr));
2584 1372 : if (subdir)
2585 : {
2586 1372 : subdir->osRootPath = osRootPath;
2587 1372 : subdir->osBasePath = entry.pszName;
2588 1372 : subdir->m_osFilterPrefix = m_osFilterPrefix;
2589 1372 : aoStackSubDir.push_back(subdir);
2590 : }
2591 1372 : entry.nMode = 0;
2592 : }
2593 :
2594 509904 : while (!aoStackSubDir.empty())
2595 : {
2596 503898 : auto l_entry = aoStackSubDir.back()->NextDirEntry();
2597 503898 : if (l_entry)
2598 : {
2599 502533 : return l_entry;
2600 : }
2601 1365 : delete aoStackSubDir.back();
2602 1365 : aoStackSubDir.pop_back();
2603 : }
2604 :
2605 6006 : if (papszContent == nullptr)
2606 : {
2607 531 : return nullptr;
2608 : }
2609 :
2610 : while (true)
2611 : {
2612 5503 : if (!papszContent[nPos])
2613 : {
2614 2173 : return nullptr;
2615 : }
2616 : // Skip . and ..entries
2617 3330 : if (papszContent[nPos][0] == '.' &&
2618 115 : (papszContent[nPos][1] == '\0' ||
2619 115 : (papszContent[nPos][1] == '.' && papszContent[nPos][2] == '\0')))
2620 : {
2621 23 : nPos++;
2622 : }
2623 : else
2624 : {
2625 3307 : CPLFree(entry.pszName);
2626 3307 : CPLString osName(osBasePath);
2627 3307 : if (!osName.empty())
2628 2082 : osName += SEP;
2629 3307 : osName += papszContent[nPos];
2630 3307 : nPos++;
2631 :
2632 3307 : entry.pszName = CPLStrdup(osName);
2633 3307 : entry.nMode = 0;
2634 3307 : CPLString osCurFile(osRootPath);
2635 3307 : if (!osCurFile.empty())
2636 3307 : osCurFile += SEP;
2637 3307 : osCurFile += entry.pszName;
2638 :
2639 6604 : const auto StatFile = [&osCurFile, this]()
2640 : {
2641 : VSIStatBufL sStatL;
2642 3302 : if (VSIStatL(osCurFile, &sStatL) == 0)
2643 : {
2644 3300 : entry.nMode = sStatL.st_mode;
2645 3300 : entry.nSize = sStatL.st_size;
2646 3300 : entry.nMTime = sStatL.st_mtime;
2647 3300 : entry.bModeKnown = true;
2648 3300 : entry.bSizeKnown = true;
2649 3300 : entry.bMTimeKnown = true;
2650 : }
2651 : else
2652 : {
2653 2 : entry.nMode = 0;
2654 2 : entry.nSize = 0;
2655 2 : entry.nMTime = 0;
2656 2 : entry.bModeKnown = false;
2657 2 : entry.bSizeKnown = false;
2658 2 : entry.bMTimeKnown = false;
2659 : }
2660 3302 : };
2661 :
2662 3319 : if (!m_osFilterPrefix.empty() &&
2663 12 : m_osFilterPrefix.size() > osName.size())
2664 : {
2665 6 : if (STARTS_WITH(m_osFilterPrefix.c_str(), osName.c_str()) &&
2666 2 : m_osFilterPrefix[osName.size()] == SEP)
2667 : {
2668 1 : StatFile();
2669 1 : if (VSI_ISDIR(entry.nMode))
2670 : {
2671 1 : goto begin;
2672 : }
2673 : }
2674 3 : continue;
2675 : }
2676 3311 : if (!m_osFilterPrefix.empty() &&
2677 8 : !STARTS_WITH(osName.c_str(), m_osFilterPrefix.c_str()))
2678 : {
2679 2 : continue;
2680 : }
2681 :
2682 3301 : StatFile();
2683 :
2684 3301 : break;
2685 : }
2686 28 : }
2687 :
2688 3301 : return &(entry);
2689 : }
2690 :
2691 : /************************************************************************/
2692 : /* UnlinkBatch() */
2693 : /************************************************************************/
2694 :
2695 1 : int *VSIFilesystemHandler::UnlinkBatch(CSLConstList papszFiles)
2696 : {
2697 : int *panRet =
2698 1 : static_cast<int *>(CPLMalloc(sizeof(int) * CSLCount(papszFiles)));
2699 3 : for (int i = 0; papszFiles && papszFiles[i]; ++i)
2700 : {
2701 2 : panRet[i] = VSIUnlink(papszFiles[i]) == 0;
2702 : }
2703 1 : return panRet;
2704 : }
2705 :
2706 : /************************************************************************/
2707 : /* RmdirRecursive() */
2708 : /************************************************************************/
2709 :
2710 34 : int VSIFilesystemHandler::RmdirRecursive(const char *pszDirname)
2711 : {
2712 68 : CPLString osDirnameWithoutEndSlash(pszDirname);
2713 68 : if (!osDirnameWithoutEndSlash.empty() &&
2714 34 : (osDirnameWithoutEndSlash.back() == '/' ||
2715 34 : osDirnameWithoutEndSlash.back() == '\\'))
2716 : {
2717 0 : osDirnameWithoutEndSlash.pop_back();
2718 : }
2719 :
2720 34 : const char SEP = VSIGetDirectorySeparator(pszDirname)[0];
2721 :
2722 68 : CPLStringList aosOptions;
2723 : auto poDir =
2724 68 : std::unique_ptr<VSIDIR>(OpenDir(pszDirname, -1, aosOptions.List()));
2725 34 : if (!poDir)
2726 8 : return -1;
2727 52 : std::vector<std::string> aosDirs;
2728 : while (true)
2729 : {
2730 141 : auto entry = poDir->NextDirEntry();
2731 141 : if (!entry)
2732 26 : break;
2733 :
2734 230 : CPLString osFilename(osDirnameWithoutEndSlash + SEP + entry->pszName);
2735 115 : if ((entry->nMode & S_IFDIR))
2736 : {
2737 10 : aosDirs.push_back(std::move(osFilename));
2738 : }
2739 : else
2740 : {
2741 105 : if (VSIUnlink(osFilename) != 0)
2742 0 : return -1;
2743 : }
2744 115 : }
2745 :
2746 : // Sort in reverse order, so that inner-most directories are deleted first
2747 26 : std::sort(aosDirs.begin(), aosDirs.end(),
2748 2 : [](const std::string &a, const std::string &b) { return a > b; });
2749 :
2750 36 : for (const auto &osDir : aosDirs)
2751 : {
2752 10 : if (VSIRmdir(osDir.c_str()) != 0)
2753 0 : return -1;
2754 : }
2755 :
2756 26 : return VSIRmdir(pszDirname);
2757 : }
2758 :
2759 : /************************************************************************/
2760 : /* GetFileMetadata() */
2761 : /************************************************************************/
2762 :
2763 15 : char **VSIFilesystemHandler::GetFileMetadata(const char * /* pszFilename*/,
2764 : const char * /*pszDomain*/,
2765 : CSLConstList /*papszOptions*/)
2766 : {
2767 15 : return nullptr;
2768 : }
2769 :
2770 : /************************************************************************/
2771 : /* SetFileMetadata() */
2772 : /************************************************************************/
2773 :
2774 0 : bool VSIFilesystemHandler::SetFileMetadata(const char * /* pszFilename*/,
2775 : CSLConstList /* papszMetadata */,
2776 : const char * /* pszDomain */,
2777 : CSLConstList /* papszOptions */)
2778 : {
2779 0 : CPLError(CE_Failure, CPLE_NotSupported, "SetFileMetadata() not supported");
2780 0 : return false;
2781 : }
2782 :
2783 : #endif
2784 :
2785 : /************************************************************************/
2786 : /* VSIFOpenExL() */
2787 : /************************************************************************/
2788 :
2789 : /**
2790 : * \brief Open/create file.
2791 : *
2792 : * This function opens (or creates) a file with the desired access.
2793 : * Binary access is always implied and
2794 : * the "b" does not need to be included in the pszAccess string.
2795 : *
2796 : * Note that the "VSILFILE *" returned by this function is
2797 : * *NOT* a standard C library FILE *, and cannot be used with any functions
2798 : * other than the "VSI*L" family of functions. They aren't "real" FILE objects.
2799 : *
2800 : * On windows it is possible to define the configuration option
2801 : * GDAL_FILE_IS_UTF8 to have pszFilename treated as being in the local
2802 : * encoding instead of UTF-8, restoring the pre-1.8.0 behavior of VSIFOpenL().
2803 : *
2804 : * This method goes through the VSIFileHandler virtualization and may
2805 : * work on unusual filesystems such as in memory.
2806 : *
2807 : * Analog of the POSIX fopen() function.
2808 : *
2809 : * @param pszFilename the file to open. UTF-8 encoded.
2810 : * @param pszAccess access requested (i.e. "r", "r+", "w")
2811 : * @param bSetError flag determining whether or not this open call
2812 : * should set VSIErrors on failure.
2813 : *
2814 : * @return NULL on failure, or the file handle.
2815 : *
2816 : */
2817 :
2818 436856 : VSILFILE *VSIFOpenExL(const char *pszFilename, const char *pszAccess,
2819 : int bSetError)
2820 :
2821 : {
2822 436856 : return VSIFOpenEx2L(pszFilename, pszAccess, bSetError, nullptr);
2823 : }
2824 :
2825 : /************************************************************************/
2826 : /* VSIFOpenEx2L() */
2827 : /************************************************************************/
2828 :
2829 : /**
2830 : * \brief Open/create file.
2831 : *
2832 : * This function opens (or creates) a file with the desired access.
2833 : * Binary access is always implied and
2834 : * the "b" does not need to be included in the pszAccess string.
2835 : *
2836 : * Note that the "VSILFILE *" returned by this function is
2837 : * *NOT* a standard C library FILE *, and cannot be used with any functions
2838 : * other than the "VSI*L" family of functions. They aren't "real" FILE objects.
2839 : *
2840 : * On windows it is possible to define the configuration option
2841 : * GDAL_FILE_IS_UTF8 to have pszFilename treated as being in the local
2842 : * encoding instead of UTF-8, restoring the pre-1.8.0 behavior of VSIFOpenL().
2843 : *
2844 : * This method goes through the VSIFileHandler virtualization and may
2845 : * work on unusual filesystems such as in memory.
2846 : *
2847 : * The following options are supported:
2848 : * <ul>
2849 : * <li>MIME headers such as Content-Type and Content-Encoding
2850 : * are supported for the /vsis3/, /vsigs/, /vsiaz/, /vsiadls/ file systems.</li>
2851 : * <li>DISABLE_READDIR_ON_OPEN=YES/NO (GDAL >= 3.6) for /vsicurl/ and other
2852 : * network-based file systems. By default, directory file listing is done,
2853 : * unless YES is specified.</li>
2854 : * <li>WRITE_THROUGH=YES (GDAL >= 3.8) for the Windows regular files to
2855 : * set the FILE_FLAG_WRITE_THROUGH flag to the CreateFile() function. In that
2856 : * mode, the data is written to the system cache but is flushed to disk without
2857 : * delay.</li>
2858 : * </ul>
2859 : *
2860 : * Options specifics to /vsis3/, /vsigs/, /vsioss/ and /vsiaz/:
2861 : * <ul>
2862 : * <li>CACHE=YES/NO. (GDAL >= 3.10) Whether file metadata and content that is
2863 : * read from the network should be kept cached in memory, after file handle
2864 : * closing. Default is YES, unless the filename is set in CPL_VSIL_CURL_NON_CACHED.
2865 : * </li>
2866 : * <li>CHUNK_SIZE=val in MiB. (GDAL >= 3.10) For "w" mode. Size of a block. Default is 50 MiB.
2867 : * For /vsis3/, /vsigz/, /vsioss/, it can be up to 5000 MiB.
2868 : * For /vsiaz/, only taken into account when BLOB_TYPE=BLOCK. It can be up to 4000 MiB.
2869 : * </li>
2870 : * </ul>
2871 : *
2872 : * Options specifics to /vsiaz/ in "w" mode:
2873 : * <ul>
2874 : * <li>BLOB_TYPE=APPEND/BLOCK. (GDAL >= 3.10) Type of blob. Defaults to APPEND.
2875 : * Append blocks are limited to 195 GiB
2876 : * (however if the file size is below 4 MiB, a block blob will be created in a
2877 : * single PUT operation)
2878 : * </li>
2879 : * </ul>
2880 : *
2881 : * Analog of the POSIX fopen() function.
2882 : *
2883 : * @param pszFilename the file to open. UTF-8 encoded.
2884 : * @param pszAccess access requested (i.e. "r", "r+", "w")
2885 : * @param bSetError flag determining whether or not this open call
2886 : * should set VSIErrors on failure.
2887 : * @param papszOptions NULL or NULL-terminated list of strings. The content is
2888 : * highly file system dependent.
2889 : *
2890 : *
2891 : * @return NULL on failure, or the file handle.
2892 : *
2893 : * @since GDAL 3.3
2894 : */
2895 :
2896 477360 : VSILFILE *VSIFOpenEx2L(const char *pszFilename, const char *pszAccess,
2897 : int bSetError, CSLConstList papszOptions)
2898 :
2899 : {
2900 : // Too long filenames can cause excessive memory allocation due to
2901 : // recursion in some filesystem handlers
2902 477360 : constexpr size_t knMaxPath = 8192;
2903 477360 : if (CPLStrnlen(pszFilename, knMaxPath) == knMaxPath)
2904 5 : return nullptr;
2905 :
2906 477466 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
2907 :
2908 477490 : auto fp = poFSHandler->Open(pszFilename, pszAccess, CPL_TO_BOOL(bSetError),
2909 954062 : papszOptions);
2910 :
2911 : VSIDebug4("VSIFOpenEx2L(%s,%s,%d) = %p", pszFilename, pszAccess, bSetError,
2912 : fp.get());
2913 :
2914 476942 : return fp.release();
2915 : }
2916 :
2917 : /************************************************************************/
2918 : /* VSIFCloseL() */
2919 : /************************************************************************/
2920 :
2921 : /**
2922 : * \fn VSIVirtualHandle::Close()
2923 : * \brief Close file.
2924 : *
2925 : * This function closes the indicated file.
2926 : *
2927 : * This method goes through the VSIFileHandler virtualization and may
2928 : * work on unusual filesystems such as in memory.
2929 : *
2930 : * Analog of the POSIX fclose() function.
2931 : *
2932 : * @return 0 on success or -1 on failure.
2933 : */
2934 :
2935 : /**
2936 : * \brief Close file.
2937 : *
2938 : * This function closes the indicated file.
2939 : *
2940 : * This method goes through the VSIFileHandler virtualization and may
2941 : * work on unusual filesystems such as in memory.
2942 : *
2943 : * Analog of the POSIX fclose() function.
2944 : *
2945 : * @param fp file handle opened with VSIFOpenL(). Passing a nullptr produces
2946 : * undefined behavior.
2947 : *
2948 : * @return 0 on success or -1 on failure.
2949 : */
2950 :
2951 302188 : int VSIFCloseL(VSILFILE *fp)
2952 :
2953 : {
2954 : VSIDebug1("VSIFCloseL(%p)", fp);
2955 :
2956 302188 : const int nResult = fp->Close();
2957 :
2958 301838 : delete fp;
2959 :
2960 302019 : return nResult;
2961 : }
2962 :
2963 : /************************************************************************/
2964 : /* VSIFSeekL() */
2965 : /************************************************************************/
2966 :
2967 : /**
2968 : * \fn int VSIVirtualHandle::Seek( vsi_l_offset nOffset, int nWhence )
2969 : * \brief Seek to requested offset.
2970 : *
2971 : * Seek to the desired offset (nOffset) in the indicated file.
2972 : *
2973 : * This method goes through the VSIFileHandler virtualization and may
2974 : * work on unusual filesystems such as in memory.
2975 : *
2976 : * Analog of the POSIX fseek() call.
2977 : *
2978 : * Caution: vsi_l_offset is a unsigned type, so SEEK_CUR can only be used
2979 : * for positive seek. If negative seek is needed, use
2980 : * handle->Seek( handle->Tell() + negative_offset, SEEK_SET ).
2981 : *
2982 : * @param nOffset offset in bytes.
2983 : * @param nWhence one of SEEK_SET, SEEK_CUR or SEEK_END.
2984 : *
2985 : * @return 0 on success or -1 one failure.
2986 : */
2987 :
2988 : /**
2989 : * \brief Seek to requested offset.
2990 : *
2991 : * Seek to the desired offset (nOffset) in the indicated file.
2992 : *
2993 : * This method goes through the VSIFileHandler virtualization and may
2994 : * work on unusual filesystems such as in memory.
2995 : *
2996 : * Analog of the POSIX fseek() call.
2997 : *
2998 : * Caution: vsi_l_offset is an unsigned type, so SEEK_CUR can only be used
2999 : * for positive seek. If negative seek is needed, use
3000 : * VSIFSeekL( fp, VSIFTellL(fp) + negative_offset, SEEK_SET ).
3001 : *
3002 : * @param fp file handle opened with VSIFOpenL().
3003 : * @param nOffset offset in bytes.
3004 : * @param nWhence one of SEEK_SET, SEEK_CUR or SEEK_END.
3005 : *
3006 : * @return 0 on success or -1 on failure.
3007 : */
3008 :
3009 9956620 : int VSIFSeekL(VSILFILE *fp, vsi_l_offset nOffset, int nWhence)
3010 :
3011 : {
3012 9956620 : return fp->Seek(nOffset, nWhence);
3013 : }
3014 :
3015 : /************************************************************************/
3016 : /* VSIFTellL() */
3017 : /************************************************************************/
3018 :
3019 : /**
3020 : * \fn VSIVirtualHandle::Tell()
3021 : * \brief Tell current file offset.
3022 : *
3023 : * Returns the current file read/write offset in bytes from the beginning of
3024 : * the file.
3025 : *
3026 : * This method goes through the VSIFileHandler virtualization and may
3027 : * work on unusual filesystems such as in memory.
3028 : *
3029 : * Analog of the POSIX ftell() call.
3030 : *
3031 : * @return file offset in bytes.
3032 : */
3033 :
3034 : /**
3035 : * \brief Tell current file offset.
3036 : *
3037 : * Returns the current file read/write offset in bytes from the beginning of
3038 : * the file.
3039 : *
3040 : * This method goes through the VSIFileHandler virtualization and may
3041 : * work on unusual filesystems such as in memory.
3042 : *
3043 : * Analog of the POSIX ftell() call.
3044 : *
3045 : * @param fp file handle opened with VSIFOpenL().
3046 : *
3047 : * @return file offset in bytes.
3048 : */
3049 :
3050 7167640 : vsi_l_offset VSIFTellL(VSILFILE *fp)
3051 :
3052 : {
3053 7167640 : return fp->Tell();
3054 : }
3055 :
3056 : /************************************************************************/
3057 : /* VSIRewindL() */
3058 : /************************************************************************/
3059 :
3060 : /**
3061 : * \brief Rewind the file pointer to the beginning of the file.
3062 : *
3063 : * This is equivalent to VSIFSeekL( fp, 0, SEEK_SET )
3064 : *
3065 : * Analog of the POSIX rewind() call.
3066 : *
3067 : * @param fp file handle opened with VSIFOpenL().
3068 : */
3069 :
3070 97320 : void VSIRewindL(VSILFILE *fp)
3071 :
3072 : {
3073 97320 : CPL_IGNORE_RET_VAL(VSIFSeekL(fp, 0, SEEK_SET));
3074 97316 : }
3075 :
3076 : /************************************************************************/
3077 : /* VSIFFlushL() */
3078 : /************************************************************************/
3079 :
3080 : /**
3081 : * \fn VSIVirtualHandle::Flush()
3082 : * \brief Flush pending writes to disk.
3083 : *
3084 : * For files in write or update mode and on filesystem types where it is
3085 : * applicable, all pending output on the file is flushed to the physical disk.
3086 : *
3087 : * This method goes through the VSIFileHandler virtualization and may
3088 : * work on unusual filesystems such as in memory.
3089 : *
3090 : * Analog of the POSIX fflush() call.
3091 : *
3092 : * On Windows regular files, this method does nothing, unless the
3093 : * VSI_FLUSH configuration option is set to YES (and only when the file has
3094 : * *not* been opened with the WRITE_THROUGH option).
3095 : *
3096 : * @return 0 on success or -1 on error.
3097 : */
3098 :
3099 : /**
3100 : * \brief Flush pending writes to disk.
3101 : *
3102 : * For files in write or update mode and on filesystem types where it is
3103 : * applicable, all pending output on the file is flushed to the physical disk.
3104 : *
3105 : * This method goes through the VSIFileHandler virtualization and may
3106 : * work on unusual filesystems such as in memory.
3107 : *
3108 : * Analog of the POSIX fflush() call.
3109 : *
3110 : * On Windows regular files, this method does nothing, unless the
3111 : * VSI_FLUSH configuration option is set to YES (and only when the file has
3112 : * *not* been opened with the WRITE_THROUGH option).
3113 : *
3114 : * @param fp file handle opened with VSIFOpenL().
3115 : *
3116 : * @return 0 on success or -1 on error.
3117 : */
3118 :
3119 91271 : int VSIFFlushL(VSILFILE *fp)
3120 :
3121 : {
3122 91271 : return fp->Flush();
3123 : }
3124 :
3125 : /************************************************************************/
3126 : /* VSIFReadL() */
3127 : /************************************************************************/
3128 :
3129 : /**
3130 : * \brief Read bytes from file.
3131 : *
3132 : * Reads nCount objects of nSize bytes from the indicated file at the
3133 : * current offset into the indicated buffer.
3134 : *
3135 : * This method goes through the VSIFileHandler virtualization and may
3136 : * work on unusual filesystems such as in memory.
3137 : *
3138 : * Analog of the POSIX fread() call.
3139 : *
3140 : * @param pBuffer the buffer into which the data should be read (at least
3141 : * nCount * nSize bytes in size.
3142 : * @param nSize size of objects to read in bytes.
3143 : * @param nCount number of objects to read.
3144 : * @param fp file handle opened with VSIFOpenL().
3145 : *
3146 : * @return number of objects successfully read. If that number is less than
3147 : * nCount, VSIFEofL() or VSIFErrorL() can be used to determine the reason for
3148 : * the short read.
3149 : */
3150 :
3151 17352000 : size_t VSIFReadL(void *pBuffer, size_t nSize, size_t nCount, VSILFILE *fp)
3152 :
3153 : {
3154 17352000 : return fp->Read(pBuffer, nSize, nCount);
3155 : }
3156 :
3157 : /************************************************************************/
3158 : /* VSIFReadMultiRangeL() */
3159 : /************************************************************************/
3160 :
3161 : /**
3162 : * \fn VSIVirtualHandle::ReadMultiRange( int nRanges, void ** ppData,
3163 : * const vsi_l_offset* panOffsets,
3164 : * const size_t* panSizes )
3165 : * \brief Read several ranges of bytes from file.
3166 : *
3167 : * Reads nRanges objects of panSizes[i] bytes from the indicated file at the
3168 : * offset panOffsets[i] into the buffer ppData[i].
3169 : *
3170 : * Ranges must be sorted in ascending start offset, and must not overlap each
3171 : * other.
3172 : *
3173 : * This method goes through the VSIFileHandler virtualization and may
3174 : * work on unusual filesystems such as in memory or /vsicurl/.
3175 : *
3176 : * @param nRanges number of ranges to read.
3177 : * @param ppData array of nRanges buffer into which the data should be read
3178 : * (ppData[i] must be at list panSizes[i] bytes).
3179 : * @param panOffsets array of nRanges offsets at which the data should be read.
3180 : * @param panSizes array of nRanges sizes of objects to read (in bytes).
3181 : *
3182 : * @return 0 in case of success, -1 otherwise.
3183 : */
3184 :
3185 : /**
3186 : * \brief Read several ranges of bytes from file.
3187 : *
3188 : * Reads nRanges objects of panSizes[i] bytes from the indicated file at the
3189 : * offset panOffsets[i] into the buffer ppData[i].
3190 : *
3191 : * Ranges must be sorted in ascending start offset, and must not overlap each
3192 : * other.
3193 : *
3194 : * This method goes through the VSIFileHandler virtualization and may
3195 : * work on unusual filesystems such as in memory or /vsicurl/.
3196 : *
3197 : * @param nRanges number of ranges to read.
3198 : * @param ppData array of nRanges buffer into which the data should be read
3199 : * (ppData[i] must be at list panSizes[i] bytes).
3200 : * @param panOffsets array of nRanges offsets at which the data should be read.
3201 : * @param panSizes array of nRanges sizes of objects to read (in bytes).
3202 : * @param fp file handle opened with VSIFOpenL().
3203 : *
3204 : * @return 0 in case of success, -1 otherwise.
3205 : */
3206 :
3207 764 : int VSIFReadMultiRangeL(int nRanges, void **ppData,
3208 : const vsi_l_offset *panOffsets, const size_t *panSizes,
3209 : VSILFILE *fp)
3210 : {
3211 764 : return fp->ReadMultiRange(nRanges, ppData, panOffsets, panSizes);
3212 : }
3213 :
3214 : /************************************************************************/
3215 : /* VSIFWriteL() */
3216 : /************************************************************************/
3217 :
3218 : /**
3219 : * \brief Write bytes to file.
3220 : *
3221 : * Writes nCount objects of nSize bytes to the indicated file at the
3222 : * current offset into the indicated buffer.
3223 : *
3224 : * This method goes through the VSIFileHandler virtualization and may
3225 : * work on unusual filesystems such as in memory.
3226 : *
3227 : * Analog of the POSIX fwrite() call.
3228 : *
3229 : * @param pBuffer the buffer from which the data should be written (at least
3230 : * nCount * nSize bytes in size.
3231 : * @param nSize size of objects to write in bytes.
3232 : * @param nCount number of objects to write.
3233 : * @param fp file handle opened with VSIFOpenL().
3234 : *
3235 : * @return number of objects successfully written.
3236 : */
3237 :
3238 4377780 : size_t VSIFWriteL(const void *pBuffer, size_t nSize, size_t nCount,
3239 : VSILFILE *fp)
3240 :
3241 : {
3242 4377780 : return fp->Write(pBuffer, nSize, nCount);
3243 : }
3244 :
3245 : /************************************************************************/
3246 : /* VSIFEofL() */
3247 : /************************************************************************/
3248 :
3249 : /**
3250 : * \fn VSIVirtualHandle::Eof()
3251 : * \brief Test for end of file.
3252 : *
3253 : * Returns TRUE (non-zero) if an end-of-file condition occurred during the
3254 : * previous read operation. The end-of-file flag is cleared by a successful
3255 : * VSIFSeekL() call, or a call to VSIFClearErrL().
3256 : *
3257 : * This method goes through the VSIFileHandler virtualization and may
3258 : * work on unusual filesystems such as in memory.
3259 : *
3260 : * Analog of the POSIX feof() call.
3261 : *
3262 : * @return TRUE if at EOF, else FALSE.
3263 : */
3264 :
3265 : /**
3266 : * \brief Test for end of file.
3267 : *
3268 : * Returns TRUE (non-zero) if an end-of-file condition occurred during the
3269 : * previous read operation. The end-of-file flag is cleared by a successful
3270 : * VSIFSeekL() call, or a call to VSIFClearErrL().
3271 : *
3272 : * This method goes through the VSIFileHandler virtualization and may
3273 : * work on unusual filesystems such as in memory.
3274 : *
3275 : * Analog of the POSIX feof() call.
3276 : *
3277 : * @param fp file handle opened with VSIFOpenL().
3278 : *
3279 : * @return TRUE if at EOF, else FALSE.
3280 : */
3281 :
3282 282565 : int VSIFEofL(VSILFILE *fp)
3283 :
3284 : {
3285 282565 : return fp->Eof();
3286 : }
3287 :
3288 : /************************************************************************/
3289 : /* VSIFErrorL() */
3290 : /************************************************************************/
3291 :
3292 : /**
3293 : * \fn VSIVirtualHandle::Error()
3294 : * \brief Test the error indicator.
3295 : *
3296 : * Returns TRUE (non-zero) if an error condition occurred during the
3297 : * previous read operation. The error indicator is cleared by a call to
3298 : * VSIFClearErrL(). Note that a end-of-file situation, reported by VSIFEofL(),
3299 : * is *not* an error reported by VSIFErrorL().
3300 : *
3301 : * This method goes through the VSIFileHandler virtualization and may
3302 : * work on unusual filesystems such as in memory.
3303 : *
3304 : * Analog of the POSIX ferror() call.
3305 : *
3306 : * @return TRUE if the error indicator is set, else FALSE.
3307 : * @since 3.10
3308 : */
3309 :
3310 : /**
3311 : * \brief Test the error indicator.
3312 : *
3313 : * Returns TRUE (non-zero) if an error condition occurred during the
3314 : * previous read operation. The error indicator is cleared by a call to
3315 : * VSIFClearErrL(). Note that a end-of-file situation, reported by VSIFEofL(),
3316 : * is *not* an error reported by VSIFErrorL().
3317 : *
3318 : * This method goes through the VSIFileHandler virtualization and may
3319 : * work on unusual filesystems such as in memory.
3320 : *
3321 : * Analog of the POSIX feof() call.
3322 : *
3323 : * @param fp file handle opened with VSIFOpenL().
3324 : *
3325 : * @return TRUE if the error indicator is set, else FALSE.
3326 : * @since 3.10
3327 : */
3328 :
3329 91555 : int VSIFErrorL(VSILFILE *fp)
3330 :
3331 : {
3332 91555 : return fp->Error();
3333 : }
3334 :
3335 : /************************************************************************/
3336 : /* VSIFClearErrL() */
3337 : /************************************************************************/
3338 :
3339 : /**
3340 : * \fn VSIVirtualHandle::ClearErr()
3341 : * \brief Reset the error and end-of-file indicators.
3342 : *
3343 : * This method goes through the VSIFileHandler virtualization and may
3344 : * work on unusual filesystems such as in memory.
3345 : *
3346 : * Analog of the POSIX clearerr() call.
3347 : *
3348 : * @since 3.10
3349 : */
3350 :
3351 : /**
3352 : * \brief Reset the error and end-of-file indicators.
3353 : *
3354 : * This method goes through the VSIFileHandler virtualization and may
3355 : * work on unusual filesystems such as in memory.
3356 : *
3357 : * Analog of the POSIX clearerr() call.
3358 : *
3359 : * @param fp file handle opened with VSIFOpenL().
3360 : *
3361 : * @since 3.10
3362 : */
3363 :
3364 31224 : void VSIFClearErrL(VSILFILE *fp)
3365 :
3366 : {
3367 31224 : fp->ClearErr();
3368 31224 : }
3369 :
3370 : /************************************************************************/
3371 : /* VSIFTruncateL() */
3372 : /************************************************************************/
3373 :
3374 : /**
3375 : * \fn VSIVirtualHandle::Truncate( vsi_l_offset nNewSize )
3376 : * \brief Truncate/expand the file to the specified size
3377 :
3378 : * This method goes through the VSIFileHandler virtualization and may
3379 : * work on unusual filesystems such as in memory.
3380 : *
3381 : * Analog of the POSIX ftruncate() call.
3382 : *
3383 : * @param nNewSize new size in bytes.
3384 : *
3385 : * @return 0 on success
3386 : */
3387 :
3388 : /**
3389 : * \brief Truncate/expand the file to the specified size
3390 :
3391 : * This method goes through the VSIFileHandler virtualization and may
3392 : * work on unusual filesystems such as in memory.
3393 : *
3394 : * Analog of the POSIX ftruncate() call.
3395 : *
3396 : * @param fp file handle opened with VSIFOpenL().
3397 : * @param nNewSize new size in bytes.
3398 : *
3399 : * @return 0 on success
3400 : */
3401 :
3402 1435 : int VSIFTruncateL(VSILFILE *fp, vsi_l_offset nNewSize)
3403 :
3404 : {
3405 1435 : return fp->Truncate(nNewSize);
3406 : }
3407 :
3408 : /************************************************************************/
3409 : /* VSIFPrintfL() */
3410 : /************************************************************************/
3411 :
3412 : /**
3413 : * \brief Formatted write to file.
3414 : *
3415 : * Provides fprintf() style formatted output to a VSI*L file. This formats
3416 : * an internal buffer which is written using VSIFWriteL().
3417 : *
3418 : * Analog of the POSIX fprintf() call.
3419 : *
3420 : * @param fp file handle opened with VSIFOpenL().
3421 : * @param pszFormat the printf() style format string.
3422 : *
3423 : * @return the number of bytes written or -1 on an error.
3424 : */
3425 :
3426 93129 : int VSIFPrintfL(VSILFILE *fp, CPL_FORMAT_STRING(const char *pszFormat), ...)
3427 :
3428 : {
3429 : va_list args;
3430 :
3431 93129 : va_start(args, pszFormat);
3432 93129 : CPLString osResult;
3433 93129 : osResult.vPrintf(pszFormat, args);
3434 93129 : va_end(args);
3435 :
3436 : return static_cast<int>(
3437 186258 : VSIFWriteL(osResult.c_str(), 1, osResult.length(), fp));
3438 : }
3439 :
3440 : /************************************************************************/
3441 : /* VSIVirtualHandle::Read() */
3442 : /************************************************************************/
3443 :
3444 : /**
3445 : * \fn VSIVirtualHandle::Read( void *pBuffer, size_t nBytes )
3446 : * \brief Read bytes from file.
3447 : *
3448 : * Reads nBytes bytes from the indicated file at the current offset into the
3449 : * indicated buffer.
3450 : *
3451 : * This method goes through the VSIFileHandler virtualization and may
3452 : * work on unusual filesystems such as in memory.
3453 : *
3454 : * Similar to the POSIX read() call, except it returns a size_t unsigned value.
3455 : *
3456 : * @param pBuffer the buffer into which the data should be read (at least
3457 : * nCount * nSize bytes in size.
3458 : * @param nBytes number of bytes to read.
3459 : *
3460 : * @return number of bytes successfully read. If that number is less than
3461 : * nCount, Eof() or Error() can be used to determine the reason for
3462 : * the short read.
3463 : *
3464 : * @since 3.13
3465 : */
3466 :
3467 : /************************************************************************/
3468 : /* VSIVirtualHandle::Read() */
3469 : /************************************************************************/
3470 :
3471 : /**
3472 : * \brief Read bytes from file.
3473 : *
3474 : * Reads nCount objects of nSize bytes from the indicated file at the
3475 : * current offset into the indicated buffer.
3476 : *
3477 : * This method goes through the VSIFileHandler virtualization and may
3478 : * work on unusual filesystems such as in memory.
3479 : *
3480 : * Analog of the POSIX fread() call.
3481 : *
3482 : * @param pBuffer the buffer into which the data should be read (at least
3483 : * nCount * nSize bytes in size.
3484 : * @param nSize size of objects to read in bytes.
3485 : * @param nCount number of objects to read.
3486 : *
3487 : * @return number of objects successfully read. If that number is less than
3488 : * nCount, VSIFEofL() or VSIFErrorL() can be used to determine the reason for
3489 : * the short read.
3490 : */
3491 17603000 : size_t VSIVirtualHandle::Read(void *pBuffer, size_t nSize, size_t nCount)
3492 : {
3493 17603000 : const size_t nBytes = nSize * nCount;
3494 17603000 : if (nSize == 0)
3495 0 : return 0;
3496 17603000 : const size_t nBytesRead = Read(pBuffer, nBytes);
3497 17602700 : return nBytesRead / nSize;
3498 : }
3499 :
3500 : /************************************************************************/
3501 : /* VSIVirtualHandle::Write() */
3502 : /************************************************************************/
3503 :
3504 : /**
3505 : * \fn VSIVirtualHandle::Write( const void *pBuffer, size_t nBytes )
3506 : * \brief Write bytes to file.
3507 : *
3508 : * Writes nBytes bytes to the indicated file at the current offset into the
3509 : * indicated buffer.
3510 : *
3511 : * This method goes through the VSIFileHandler virtualization and may
3512 : * work on unusual filesystems such as in memory.
3513 : *
3514 : * Analog of the POSIX write() call, except it returns a size_t unsigned value.
3515 : *
3516 : * @param pBuffer the buffer from which the data should be written (at least
3517 : * nCount * nSize bytes in size.
3518 : * @param nBytes number of bytes to write.
3519 : *
3520 : * @return number of bytes successfully written.
3521 : *
3522 : * @since 3.13
3523 : */
3524 :
3525 : /************************************************************************/
3526 : /* VSIVirtualHandle::Write() */
3527 : /************************************************************************/
3528 :
3529 : /**
3530 : * \brief Write bytes to file.
3531 : *
3532 : * Writes nCount objects of nSize bytes to the indicated file at the
3533 : * current offset into the indicated buffer.
3534 : *
3535 : * This method goes through the VSIFileHandler virtualization and may
3536 : * work on unusual filesystems such as in memory.
3537 : *
3538 : * Analog of the POSIX fwrite() call.
3539 : *
3540 : * @param pBuffer the buffer from which the data should be written (at least
3541 : * nCount * nSize bytes in size.
3542 : * @param nSize size of objects to write in bytes.
3543 : * @param nCount number of objects to write.
3544 : *
3545 : * @return number of objects successfully written.
3546 : */
3547 :
3548 4751810 : size_t VSIVirtualHandle::Write(const void *pBuffer, size_t nSize, size_t nCount)
3549 : {
3550 4751810 : const size_t nBytes = nSize * nCount;
3551 4751810 : if (nSize == 0)
3552 7 : return 0;
3553 4751800 : const size_t nBytesWritten = Write(pBuffer, nBytes);
3554 4752450 : return nBytesWritten / nSize;
3555 : }
3556 :
3557 : /************************************************************************/
3558 : /* VSIVirtualHandle::Printf() */
3559 : /************************************************************************/
3560 :
3561 : /**
3562 : * \brief Formatted write to file.
3563 : *
3564 : * Provides fprintf() style formatted output to a VSI*L file. This formats
3565 : * an internal buffer which is written using VSIFWriteL().
3566 : *
3567 : * Analog of the POSIX fprintf() call.
3568 : *
3569 : * @param pszFormat the printf() style format string.
3570 : *
3571 : * @return the number of bytes written or -1 on an error.
3572 : */
3573 :
3574 2601 : int VSIVirtualHandle::Printf(CPL_FORMAT_STRING(const char *pszFormat), ...)
3575 : {
3576 : va_list args;
3577 :
3578 2601 : va_start(args, pszFormat);
3579 2601 : CPLString osResult;
3580 2601 : osResult.vPrintf(pszFormat, args);
3581 2601 : va_end(args);
3582 :
3583 5202 : return static_cast<int>(Write(osResult.c_str(), 1, osResult.length()));
3584 : }
3585 :
3586 : /************************************************************************/
3587 : /* VSIFPutcL() */
3588 : /************************************************************************/
3589 :
3590 : // TODO: should we put in conformance with POSIX regarding the return
3591 : // value. As of today (2015-08-29), no code in GDAL sources actually
3592 : // check the return value.
3593 :
3594 : /**
3595 : * \brief Write a single byte to the file
3596 : *
3597 : * Writes the character nChar, cast to an unsigned char, to file.
3598 : *
3599 : * Almost an analog of the POSIX fputc() call, except that it returns
3600 : * the number of character written (1 or 0), and not the (cast)
3601 : * character itself or EOF.
3602 : *
3603 : * @param nChar character to write.
3604 : * @param fp file handle opened with VSIFOpenL().
3605 : *
3606 : * @return 1 in case of success, 0 on error.
3607 : */
3608 :
3609 562 : int VSIFPutcL(int nChar, VSILFILE *fp)
3610 :
3611 : {
3612 562 : const unsigned char cChar = static_cast<unsigned char>(nChar);
3613 562 : return static_cast<int>(VSIFWriteL(&cChar, 1, 1, fp));
3614 : }
3615 :
3616 : /************************************************************************/
3617 : /* VSIFGetRangeStatusL() */
3618 : /************************************************************************/
3619 :
3620 : /**
3621 : * \fn VSIVirtualHandle::GetRangeStatus( vsi_l_offset nOffset,
3622 : * vsi_l_offset nLength )
3623 : * \brief Return if a given file range contains data or holes filled with zeroes
3624 : *
3625 : * This uses the filesystem capabilities of querying which regions of
3626 : * a sparse file are allocated or not. This is currently only
3627 : * implemented for Linux (and no other Unix derivatives) and Windows.
3628 : *
3629 : * Note: A return of VSI_RANGE_STATUS_DATA doesn't exclude that the
3630 : * extent is filled with zeroes! It must be interpreted as "may
3631 : * contain non-zero data".
3632 : *
3633 : * @param nOffset offset of the start of the extent.
3634 : * @param nLength extent length.
3635 : *
3636 : * @return extent status: VSI_RANGE_STATUS_UNKNOWN, VSI_RANGE_STATUS_DATA or
3637 : * VSI_RANGE_STATUS_HOLE
3638 : */
3639 :
3640 : /**
3641 : * \brief Return if a given file range contains data or holes filled with zeroes
3642 : *
3643 : * This uses the filesystem capabilities of querying which regions of
3644 : * a sparse file are allocated or not. This is currently only
3645 : * implemented for Linux (and no other Unix derivatives) and Windows.
3646 : *
3647 : * Note: A return of VSI_RANGE_STATUS_DATA doesn't exclude that the
3648 : * extent is filled with zeroes! It must be interpreted as "may
3649 : * contain non-zero data".
3650 : *
3651 : * @param fp file handle opened with VSIFOpenL().
3652 : * @param nOffset offset of the start of the extent.
3653 : * @param nLength extent length.
3654 : *
3655 : * @return extent status: VSI_RANGE_STATUS_UNKNOWN, VSI_RANGE_STATUS_DATA or
3656 : * VSI_RANGE_STATUS_HOLE
3657 : */
3658 :
3659 706 : VSIRangeStatus VSIFGetRangeStatusL(VSILFILE *fp, vsi_l_offset nOffset,
3660 : vsi_l_offset nLength)
3661 : {
3662 706 : return fp->GetRangeStatus(nOffset, nLength);
3663 : }
3664 :
3665 : /************************************************************************/
3666 : /* VSIIngestFile() */
3667 : /************************************************************************/
3668 :
3669 : /**
3670 : * \brief Ingest a file into memory.
3671 : *
3672 : * Read the whole content of a file into a memory buffer.
3673 : *
3674 : * Either fp or pszFilename can be NULL, but not both at the same time.
3675 : *
3676 : * If fp is passed non-NULL, it is the responsibility of the caller to
3677 : * close it.
3678 : *
3679 : * If non-NULL, the returned buffer is guaranteed to be NUL-terminated.
3680 : *
3681 : * @param fp file handle opened with VSIFOpenL().
3682 : * @param pszFilename filename.
3683 : * @param ppabyRet pointer to the target buffer. *ppabyRet must be freed with
3684 : * VSIFree()
3685 : * @param pnSize pointer to variable to store the file size. May be NULL.
3686 : * @param nMaxSize maximum size of file allowed. If no limit, set to a negative
3687 : * value.
3688 : *
3689 : * @return TRUE in case of success.
3690 : *
3691 : */
3692 :
3693 15213 : int VSIIngestFile(VSILFILE *fp, const char *pszFilename, GByte **ppabyRet,
3694 : vsi_l_offset *pnSize, GIntBig nMaxSize)
3695 : {
3696 15213 : if (fp == nullptr && pszFilename == nullptr)
3697 0 : return FALSE;
3698 15213 : if (ppabyRet == nullptr)
3699 0 : return FALSE;
3700 :
3701 15213 : *ppabyRet = nullptr;
3702 15213 : if (pnSize != nullptr)
3703 8274 : *pnSize = 0;
3704 :
3705 15213 : bool bFreeFP = false;
3706 15213 : if (nullptr == fp)
3707 : {
3708 13918 : fp = VSIFOpenL(pszFilename, "rb");
3709 13918 : if (nullptr == fp)
3710 : {
3711 1828 : CPLError(CE_Failure, CPLE_FileIO, "Cannot open file '%s'",
3712 : pszFilename);
3713 1828 : return FALSE;
3714 : }
3715 12090 : bFreeFP = true;
3716 : }
3717 : else
3718 : {
3719 1295 : if (VSIFSeekL(fp, 0, SEEK_SET) != 0)
3720 0 : return FALSE;
3721 : }
3722 :
3723 13385 : vsi_l_offset nDataLen = 0;
3724 :
3725 13385 : if (pszFilename == nullptr || strcmp(pszFilename, "/vsistdin/") == 0)
3726 : {
3727 102 : vsi_l_offset nDataAlloc = 0;
3728 102 : if (VSIFSeekL(fp, 0, SEEK_SET) != 0)
3729 : {
3730 0 : if (bFreeFP)
3731 0 : CPL_IGNORE_RET_VAL(VSIFCloseL(fp));
3732 0 : return FALSE;
3733 : }
3734 102 : constexpr size_t CHUNK_SIZE = 32768;
3735 : while (true)
3736 : {
3737 268 : if (nDataLen + CHUNK_SIZE + 1 > nDataAlloc)
3738 : {
3739 213 : nDataAlloc = (nDataAlloc * 4) / 3 + CHUNK_SIZE + 1;
3740 : if (nDataAlloc >
3741 : static_cast<vsi_l_offset>(static_cast<size_t>(nDataAlloc)))
3742 : {
3743 : CPLError(CE_Failure, CPLE_AppDefined,
3744 : "Input file too large to be opened");
3745 : VSIFree(*ppabyRet);
3746 : *ppabyRet = nullptr;
3747 : if (bFreeFP)
3748 : CPL_IGNORE_RET_VAL(VSIFCloseL(fp));
3749 : return FALSE;
3750 : }
3751 : GByte *pabyNew = static_cast<GByte *>(
3752 213 : VSIRealloc(*ppabyRet, static_cast<size_t>(nDataAlloc)));
3753 213 : if (pabyNew == nullptr)
3754 : {
3755 0 : CPLError(CE_Failure, CPLE_OutOfMemory,
3756 : "Cannot allocate " CPL_FRMT_GIB " bytes",
3757 : nDataAlloc);
3758 0 : VSIFree(*ppabyRet);
3759 0 : *ppabyRet = nullptr;
3760 0 : if (bFreeFP)
3761 0 : CPL_IGNORE_RET_VAL(VSIFCloseL(fp));
3762 0 : return FALSE;
3763 : }
3764 213 : *ppabyRet = pabyNew;
3765 : }
3766 : const int nRead = static_cast<int>(
3767 268 : VSIFReadL(*ppabyRet + nDataLen, 1, CHUNK_SIZE, fp));
3768 268 : nDataLen += nRead;
3769 :
3770 268 : if (nMaxSize >= 0 && nDataLen > static_cast<vsi_l_offset>(nMaxSize))
3771 : {
3772 0 : CPLError(CE_Failure, CPLE_AppDefined,
3773 : "Input file too large to be opened");
3774 0 : VSIFree(*ppabyRet);
3775 0 : *ppabyRet = nullptr;
3776 0 : if (pnSize != nullptr)
3777 0 : *pnSize = 0;
3778 0 : if (bFreeFP)
3779 0 : CPL_IGNORE_RET_VAL(VSIFCloseL(fp));
3780 0 : return FALSE;
3781 : }
3782 :
3783 268 : if (pnSize != nullptr)
3784 194 : *pnSize += nRead;
3785 268 : (*ppabyRet)[nDataLen] = '\0';
3786 268 : if (nRead == 0)
3787 102 : break;
3788 268 : }
3789 : }
3790 : else
3791 : {
3792 13283 : if (VSIFSeekL(fp, 0, SEEK_END) != 0)
3793 : {
3794 0 : if (bFreeFP)
3795 0 : CPL_IGNORE_RET_VAL(VSIFCloseL(fp));
3796 0 : return FALSE;
3797 : }
3798 13283 : nDataLen = VSIFTellL(fp);
3799 :
3800 : // With "large" VSI I/O API we can read data chunks larger than
3801 : // VSIMalloc could allocate. Catch it here.
3802 : // Opening a directory returns nDataLen = INT_MAX (on 32bit) or
3803 : // INT64_MAX (on 64bit)
3804 21283 : if (nDataLen > std::numeric_limits<size_t>::max() / 2 - 1 ||
3805 8000 : (nMaxSize >= 0 && nDataLen > static_cast<vsi_l_offset>(nMaxSize)))
3806 : {
3807 0 : CPLError(CE_Failure, CPLE_AppDefined,
3808 : "Input file too large to be opened");
3809 0 : if (bFreeFP)
3810 0 : CPL_IGNORE_RET_VAL(VSIFCloseL(fp));
3811 0 : return FALSE;
3812 : }
3813 :
3814 13283 : if (VSIFSeekL(fp, 0, SEEK_SET) != 0)
3815 : {
3816 0 : if (bFreeFP)
3817 0 : CPL_IGNORE_RET_VAL(VSIFCloseL(fp));
3818 0 : return FALSE;
3819 : }
3820 :
3821 13283 : *ppabyRet =
3822 13283 : static_cast<GByte *>(VSIMalloc(static_cast<size_t>(nDataLen + 1)));
3823 13283 : if (nullptr == *ppabyRet)
3824 : {
3825 0 : CPLError(CE_Failure, CPLE_OutOfMemory,
3826 : "Cannot allocate " CPL_FRMT_GIB " bytes", nDataLen + 1);
3827 0 : if (bFreeFP)
3828 0 : CPL_IGNORE_RET_VAL(VSIFCloseL(fp));
3829 0 : return FALSE;
3830 : }
3831 :
3832 13283 : (*ppabyRet)[nDataLen] = '\0';
3833 13283 : if (nDataLen !=
3834 13283 : VSIFReadL(*ppabyRet, 1, static_cast<size_t>(nDataLen), fp))
3835 : {
3836 0 : CPLError(CE_Failure, CPLE_FileIO,
3837 : "Cannot read " CPL_FRMT_GIB " bytes", nDataLen);
3838 0 : VSIFree(*ppabyRet);
3839 0 : *ppabyRet = nullptr;
3840 0 : if (bFreeFP)
3841 0 : CPL_IGNORE_RET_VAL(VSIFCloseL(fp));
3842 0 : return FALSE;
3843 : }
3844 13283 : if (pnSize != nullptr)
3845 6466 : *pnSize = nDataLen;
3846 : }
3847 13385 : if (bFreeFP)
3848 12090 : CPL_IGNORE_RET_VAL(VSIFCloseL(fp));
3849 13385 : return TRUE;
3850 : }
3851 :
3852 : /************************************************************************/
3853 : /* VSIOverwriteFile() */
3854 : /************************************************************************/
3855 :
3856 : /**
3857 : * \brief Overwrite an existing file with content from another one
3858 : *
3859 : * @param fpTarget file handle opened with VSIFOpenL() with "rb+" flag.
3860 : * @param pszSourceFilename source filename
3861 : *
3862 : * @return TRUE in case of success.
3863 : *
3864 : * @since GDAL 3.1
3865 : */
3866 :
3867 4 : int VSIOverwriteFile(VSILFILE *fpTarget, const char *pszSourceFilename)
3868 : {
3869 4 : VSILFILE *fpSource = VSIFOpenL(pszSourceFilename, "rb");
3870 4 : if (fpSource == nullptr)
3871 : {
3872 0 : CPLError(CE_Failure, CPLE_FileIO, "Cannot open %s", pszSourceFilename);
3873 0 : return false;
3874 : }
3875 :
3876 4 : const size_t nBufferSize = 4096;
3877 4 : void *pBuffer = CPLMalloc(nBufferSize);
3878 4 : VSIFSeekL(fpTarget, 0, SEEK_SET);
3879 4 : bool bRet = true;
3880 : while (true)
3881 : {
3882 4 : size_t nRead = VSIFReadL(pBuffer, 1, nBufferSize, fpSource);
3883 4 : size_t nWritten = VSIFWriteL(pBuffer, 1, nRead, fpTarget);
3884 4 : if (nWritten != nRead)
3885 : {
3886 0 : bRet = false;
3887 0 : break;
3888 : }
3889 4 : if (nRead < nBufferSize)
3890 4 : break;
3891 0 : }
3892 :
3893 4 : if (bRet)
3894 : {
3895 4 : bRet = VSIFTruncateL(fpTarget, VSIFTellL(fpTarget)) == 0;
3896 4 : if (!bRet)
3897 : {
3898 0 : CPLError(CE_Failure, CPLE_FileIO, "Truncation failed");
3899 : }
3900 : }
3901 :
3902 4 : CPLFree(pBuffer);
3903 4 : VSIFCloseL(fpSource);
3904 4 : return bRet;
3905 : }
3906 :
3907 : /************************************************************************/
3908 : /* VSIFGetNativeFileDescriptorL() */
3909 : /************************************************************************/
3910 :
3911 : /**
3912 : * \fn VSIVirtualHandle::GetNativeFileDescriptor()
3913 : * \brief Returns the "native" file descriptor for the virtual handle.
3914 : *
3915 : * This will only return a non-NULL value for "real" files handled by the
3916 : * operating system (to be opposed to GDAL virtual file systems).
3917 : *
3918 : * On POSIX systems, this will be a integer value ("fd") cast as a void*.
3919 : * On Windows systems, this will be the HANDLE.
3920 : *
3921 : * @return the native file descriptor, or NULL.
3922 : */
3923 :
3924 : /**
3925 : * \brief Returns the "native" file descriptor for the virtual handle.
3926 : *
3927 : * This will only return a non-NULL value for "real" files handled by the
3928 : * operating system (to be opposed to GDAL virtual file systems).
3929 : *
3930 : * On POSIX systems, this will be a integer value ("fd") cast as a void*.
3931 : * On Windows systems, this will be the HANDLE.
3932 : *
3933 : * @param fp file handle opened with VSIFOpenL().
3934 : *
3935 : * @return the native file descriptor, or NULL.
3936 : */
3937 :
3938 64 : void *VSIFGetNativeFileDescriptorL(VSILFILE *fp)
3939 : {
3940 64 : return fp->GetNativeFileDescriptor();
3941 : }
3942 :
3943 : /************************************************************************/
3944 : /* VSIGetDiskFreeSpace() */
3945 : /************************************************************************/
3946 :
3947 : /**
3948 : * \brief Return free disk space available on the filesystem.
3949 : *
3950 : * This function returns the free disk space available on the filesystem.
3951 : *
3952 : * @param pszDirname a directory of the filesystem to query.
3953 : * @return The free space in bytes. Or -1 in case of error.
3954 : */
3955 :
3956 75 : GIntBig VSIGetDiskFreeSpace(const char *pszDirname)
3957 : {
3958 75 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszDirname);
3959 :
3960 75 : return poFSHandler->GetDiskFreeSpace(pszDirname);
3961 : }
3962 :
3963 : /************************************************************************/
3964 : /* VSIGetFileSystemsPrefixes() */
3965 : /************************************************************************/
3966 :
3967 : /**
3968 : * \brief Return the list of prefixes for virtual file system handlers
3969 : * currently registered.
3970 : *
3971 : * Typically: "", "/vsimem/", "/vsicurl/", etc
3972 : *
3973 : * @return a NULL terminated list of prefixes. Must be freed with CSLDestroy()
3974 : */
3975 :
3976 9 : char **VSIGetFileSystemsPrefixes(void)
3977 : {
3978 9 : return VSIFileManager::GetPrefixes();
3979 : }
3980 :
3981 : /************************************************************************/
3982 : /* VSIGetFileSystemOptions() */
3983 : /************************************************************************/
3984 :
3985 : /**
3986 : * \brief Return the list of options associated with a virtual file system
3987 : * handler as a serialized XML string.
3988 : *
3989 : * Those options may be set as configuration options with CPLSetConfigOption().
3990 : *
3991 : * @param pszFilename a filename, or prefix of a virtual file system handler.
3992 : * @return a string, which must not be freed, or NULL if no options is declared.
3993 : */
3994 :
3995 36 : const char *VSIGetFileSystemOptions(const char *pszFilename)
3996 : {
3997 36 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
3998 :
3999 36 : return poFSHandler->GetOptions();
4000 : }
4001 :
4002 : /************************************************************************/
4003 : /* VSISetPathSpecificOption() */
4004 : /************************************************************************/
4005 :
4006 : static std::mutex oMutexPathSpecificOptions;
4007 :
4008 : // key is a path prefix
4009 : // value is a map of key, value pair
4010 : static std::map<std::string, std::map<std::string, std::string>>
4011 : oMapPathSpecificOptions;
4012 :
4013 : /**
4014 : * \brief Set a credential (or more generally an option related to a
4015 : * virtual file system) for a given path prefix.
4016 : * @deprecated in GDAL 3.6 for the better named VSISetPathSpecificOption()
4017 : * @see VSISetPathSpecificOption()
4018 : */
4019 0 : void VSISetCredential(const char *pszPathPrefix, const char *pszKey,
4020 : const char *pszValue)
4021 : {
4022 0 : VSISetPathSpecificOption(pszPathPrefix, pszKey, pszValue);
4023 0 : }
4024 :
4025 : /**
4026 : * \brief Set a path specific option for a given path prefix.
4027 : *
4028 : * Such option is typically, but not limited to, a credential setting for a
4029 : * virtual file system.
4030 : *
4031 : * That option may also be set as a configuration option with
4032 : * CPLSetConfigOption(), but this function allows to specify them with a
4033 : * granularity at the level of a file path, which makes it easier if using the
4034 : * same virtual file system but with different credentials (e.g. different
4035 : * credentials for bucket "/vsis3/foo" and "/vsis3/bar")
4036 : *
4037 : * This is supported for the following virtual file systems:
4038 : * /vsis3/, /vsigs/, /vsiaz/, /vsioss/, /vsiwebhdfs, /vsiswift.
4039 : * Note: setting them for a path starting with /vsiXXX/ will also apply for
4040 : * /vsiXXX_streaming/ requests.
4041 : *
4042 : * Note that no particular care is taken to store them in RAM in a secure way.
4043 : * So they might accidentally hit persistent storage if swapping occurs, or
4044 : * someone with access to the memory allocated by the process may be able to
4045 : * read them.
4046 : *
4047 : * @param pszPathPrefix a path prefix of a virtual file system handler.
4048 : * Typically of the form "/vsiXXX/bucket". Must NOT be
4049 : * NULL. Should not include trailing slashes.
4050 : * @param pszKey Option name. Must NOT be NULL.
4051 : * @param pszValue Option value. May be NULL to erase it.
4052 : *
4053 : * @since GDAL 3.6
4054 : */
4055 :
4056 92 : void VSISetPathSpecificOption(const char *pszPathPrefix, const char *pszKey,
4057 : const char *pszValue)
4058 : {
4059 184 : std::lock_guard<std::mutex> oLock(oMutexPathSpecificOptions);
4060 92 : auto oIter = oMapPathSpecificOptions.find(pszPathPrefix);
4061 184 : CPLString osKey(pszKey);
4062 92 : osKey.toupper();
4063 92 : if (oIter == oMapPathSpecificOptions.end())
4064 : {
4065 25 : if (pszValue != nullptr)
4066 25 : oMapPathSpecificOptions[pszPathPrefix][osKey] = pszValue;
4067 : }
4068 67 : else if (pszValue != nullptr)
4069 59 : oIter->second[osKey] = pszValue;
4070 : else
4071 8 : oIter->second.erase(osKey);
4072 92 : }
4073 :
4074 : /************************************************************************/
4075 : /* VSIClearPathSpecificOptions() */
4076 : /************************************************************************/
4077 :
4078 : /**
4079 : * \brief Clear path specific options set with VSISetPathSpecificOption()
4080 : * @deprecated in GDAL 3.6 for the better named VSIClearPathSpecificOptions()
4081 : * @see VSIClearPathSpecificOptions()
4082 : */
4083 0 : void VSIClearCredentials(const char *pszPathPrefix)
4084 : {
4085 0 : return VSIClearPathSpecificOptions(pszPathPrefix);
4086 : }
4087 :
4088 : /**
4089 : * \brief Clear path specific options set with VSISetPathSpecificOption()
4090 : *
4091 : * Note that no particular care is taken to remove them from RAM in a secure
4092 : * way.
4093 : *
4094 : * @param pszPathPrefix If set to NULL, all path specific options are cleared.
4095 : * If set to not-NULL, only those set with
4096 : * VSISetPathSpecificOption(pszPathPrefix, ...) will be
4097 : * cleared.
4098 : *
4099 : * @since GDAL 3.6
4100 : */
4101 20 : void VSIClearPathSpecificOptions(const char *pszPathPrefix)
4102 : {
4103 40 : std::lock_guard<std::mutex> oLock(oMutexPathSpecificOptions);
4104 20 : if (pszPathPrefix == nullptr)
4105 : {
4106 3 : oMapPathSpecificOptions.clear();
4107 : }
4108 : else
4109 : {
4110 17 : oMapPathSpecificOptions.erase(pszPathPrefix);
4111 : }
4112 20 : }
4113 :
4114 : /************************************************************************/
4115 : /* VSIGetPathSpecificOption() */
4116 : /************************************************************************/
4117 :
4118 : /**
4119 : * \brief Get the value of a credential (or more generally an option related to
4120 : * a virtual file system) for a given path.
4121 : * @deprecated in GDAL 3.6 for the better named VSIGetPathSpecificOption()
4122 : * @see VSIGetPathSpecificOption()
4123 : */
4124 0 : const char *VSIGetCredential(const char *pszPath, const char *pszKey,
4125 : const char *pszDefault)
4126 : {
4127 0 : return VSIGetPathSpecificOption(pszPath, pszKey, pszDefault);
4128 : }
4129 :
4130 : /**
4131 : * \brief Get the value a path specific option.
4132 : *
4133 : * Such option is typically, but not limited to, a credential setting for a
4134 : * virtual file system.
4135 : *
4136 : * If no match occurs, CPLGetConfigOption(pszKey, pszDefault) is returned.
4137 : *
4138 : * Mostly to be used by virtual file system implementations.
4139 : *
4140 : * @since GDAL 3.6
4141 : * @see VSISetPathSpecificOption()
4142 : */
4143 153584 : const char *VSIGetPathSpecificOption(const char *pszPath, const char *pszKey,
4144 : const char *pszDefault)
4145 : {
4146 : {
4147 153584 : std::lock_guard<std::mutex> oLock(oMutexPathSpecificOptions);
4148 193055 : for (auto it = oMapPathSpecificOptions.rbegin();
4149 232523 : it != oMapPathSpecificOptions.rend(); ++it)
4150 : {
4151 39739 : if (STARTS_WITH(pszPath, it->first.c_str()))
4152 : {
4153 3930 : auto oIter = it->second.find(CPLString(pszKey).toupper());
4154 3930 : if (oIter != it->second.end())
4155 271 : return oIter->second.c_str();
4156 : }
4157 : }
4158 : }
4159 153315 : return CPLGetConfigOption(pszKey, pszDefault);
4160 : }
4161 :
4162 : /************************************************************************/
4163 : /* VSIDuplicateFileSystemHandler() */
4164 : /************************************************************************/
4165 :
4166 : /**
4167 : * \brief Duplicate an existing file system handler.
4168 : *
4169 : * A number of virtual file system for remote object stores use protocols
4170 : * identical or close to popular ones (typically AWS S3), but with slightly
4171 : * different settings (at the very least the endpoint).
4172 : *
4173 : * This functions allows to duplicate the source virtual file system handler
4174 : * as a new one with a different prefix (when the source virtual file system
4175 : * handler supports the duplication operation).
4176 : *
4177 : * VSISetPathSpecificOption() will typically be called afterwards to change
4178 : * configurable settings on the cloned file system handler (e.g. AWS_S3_ENDPOINT
4179 : * for a clone of /vsis3/).
4180 : *
4181 : * @since GDAL 3.7
4182 : */
4183 4 : bool VSIDuplicateFileSystemHandler(const char *pszSourceFSName,
4184 : const char *pszNewFSName)
4185 : {
4186 : VSIFilesystemHandler *poTargetFSHandler =
4187 4 : VSIFileManager::GetHandler(pszNewFSName);
4188 4 : if (poTargetFSHandler != VSIFileManager::GetHandler("/"))
4189 : {
4190 1 : CPLError(CE_Failure, CPLE_AppDefined,
4191 : "%s is already a known virtual file system", pszNewFSName);
4192 1 : return false;
4193 : }
4194 :
4195 : VSIFilesystemHandler *poSourceFSHandler =
4196 3 : VSIFileManager::GetHandler(pszSourceFSName);
4197 3 : if (!poSourceFSHandler)
4198 : {
4199 0 : CPLError(CE_Failure, CPLE_AppDefined,
4200 : "%s is not a known virtual file system", pszSourceFSName);
4201 0 : return false;
4202 : }
4203 :
4204 : auto poClonedFSHandler = std::shared_ptr<VSIFilesystemHandler>(
4205 6 : poSourceFSHandler->Duplicate(pszNewFSName));
4206 3 : if (!poClonedFSHandler)
4207 2 : return false;
4208 :
4209 1 : VSIFileManager::InstallHandler(pszNewFSName, poClonedFSHandler);
4210 1 : return true;
4211 : }
4212 :
4213 : /************************************************************************/
4214 : /* ==================================================================== */
4215 : /* VSIFileManager() */
4216 : /* ==================================================================== */
4217 : /************************************************************************/
4218 :
4219 : #ifndef DOXYGEN_SKIP
4220 :
4221 : /*
4222 : ** Notes on Multithreading:
4223 : **
4224 : ** The VSIFileManager maintains a list of file type handlers (mem, large
4225 : ** file, etc). It should be thread safe.
4226 : **/
4227 :
4228 : /************************************************************************/
4229 : /* VSIFileManager() */
4230 : /************************************************************************/
4231 :
4232 : VSIFileManager::VSIFileManager() = default;
4233 :
4234 : /************************************************************************/
4235 : /* ~VSIFileManager() */
4236 : /************************************************************************/
4237 :
4238 : VSIFileManager::~VSIFileManager() = default;
4239 :
4240 : /************************************************************************/
4241 : /* Get() */
4242 : /************************************************************************/
4243 :
4244 : static VSIFileManager *poManager = nullptr;
4245 : static CPLMutex *hVSIFileManagerMutex = nullptr;
4246 :
4247 2996690 : VSIFileManager *VSIFileManager::Get()
4248 : {
4249 5993570 : CPLMutexHolder oHolder(&hVSIFileManagerMutex);
4250 2996870 : if (poManager != nullptr)
4251 : {
4252 2995080 : return poManager;
4253 : }
4254 :
4255 1793 : poManager = new VSIFileManager;
4256 1793 : VSIInstallLargeFileHandler();
4257 1793 : VSIInstallSubFileHandler();
4258 1793 : VSIInstallMemFileHandler();
4259 : #ifdef HAVE_LIBZ
4260 1793 : VSIInstallGZipFileHandler();
4261 1793 : VSIInstallZipFileHandler();
4262 : #endif
4263 : #ifdef HAVE_LIBARCHIVE
4264 : VSIInstall7zFileHandler();
4265 : VSIInstallRarFileHandler();
4266 : #endif
4267 : #ifdef HAVE_CURL
4268 1793 : VSIInstallCurlFileHandler();
4269 1793 : VSIInstallCurlStreamingFileHandler();
4270 1793 : VSIInstallS3FileHandler();
4271 1793 : VSIInstallS3StreamingFileHandler();
4272 1793 : VSIInstallGSFileHandler();
4273 1793 : VSIInstallGSStreamingFileHandler();
4274 1793 : VSIInstallAzureFileHandler();
4275 1793 : VSIInstallAzureStreamingFileHandler();
4276 1793 : VSIInstallADLSFileHandler();
4277 1793 : VSIInstallOSSFileHandler();
4278 1793 : VSIInstallOSSStreamingFileHandler();
4279 1793 : VSIInstallSwiftFileHandler();
4280 1793 : VSIInstallSwiftStreamingFileHandler();
4281 1793 : VSIInstallWebHdfsHandler();
4282 : #endif
4283 1793 : VSIInstallStdinHandler();
4284 1793 : VSIInstallHdfsHandler();
4285 1793 : VSIInstallStdoutHandler();
4286 1793 : VSIInstallSparseFileHandler();
4287 1793 : VSIInstallTarFileHandler();
4288 1793 : VSIInstallCachedFileHandler();
4289 1793 : VSIInstallCryptFileHandler();
4290 :
4291 1793 : return poManager;
4292 : }
4293 :
4294 : /************************************************************************/
4295 : /* GetPrefixes() */
4296 : /************************************************************************/
4297 :
4298 743 : char **VSIFileManager::GetPrefixes()
4299 : {
4300 1486 : CPLMutexHolder oHolder(&hVSIFileManagerMutex);
4301 1486 : CPLStringList aosList;
4302 23776 : for (const auto &oIter : Get()->m_apoHandlers)
4303 : {
4304 23033 : if (oIter.first != "/vsicurl?")
4305 : {
4306 22290 : aosList.AddString(oIter.first.c_str());
4307 : }
4308 : }
4309 1486 : return aosList.StealList();
4310 : }
4311 :
4312 : /************************************************************************/
4313 : /* GetHandler() */
4314 : /************************************************************************/
4315 :
4316 2938720 : VSIFilesystemHandler *VSIFileManager::GetHandler(const char *pszPath)
4317 :
4318 : {
4319 2938720 : VSIFileManager *poThis = Get();
4320 2938800 : const size_t nPathLen = strlen(pszPath);
4321 :
4322 67077900 : for (auto &[key, handler] : poThis->m_apoHandlers)
4323 : {
4324 66160200 : const char *pszIterKey = key.c_str();
4325 66164400 : const size_t nIterKeyLen = key.size();
4326 66175500 : if (strncmp(pszPath, pszIterKey, nIterKeyLen) == 0)
4327 2036390 : return handler.get();
4328 :
4329 : // "/vsimem\foo" should be handled as "/vsimem/foo".
4330 64248800 : if (nIterKeyLen && nPathLen > nIterKeyLen &&
4331 56884900 : pszIterKey[nIterKeyLen - 1] == '/' &&
4332 50202300 : pszPath[nIterKeyLen - 1] == '\\' &&
4333 30 : strncmp(pszPath, pszIterKey, nIterKeyLen - 1) == 0)
4334 0 : return handler.get();
4335 :
4336 : // /vsimem should be treated as a match for /vsimem/.
4337 64248800 : if (nPathLen + 1 == nIterKeyLen &&
4338 606671 : strncmp(pszPath, pszIterKey, nPathLen) == 0)
4339 109704 : return handler.get();
4340 : }
4341 :
4342 899640 : return poThis->m_poDefaultHandler.get();
4343 : }
4344 :
4345 : /************************************************************************/
4346 : /* InstallHandler() */
4347 : /************************************************************************/
4348 :
4349 0 : void VSIFileManager::InstallHandler(const std::string &osPrefix,
4350 : VSIFilesystemHandler *poHandler)
4351 :
4352 : {
4353 0 : InstallHandler(osPrefix, std::shared_ptr<VSIFilesystemHandler>(poHandler));
4354 0 : }
4355 :
4356 : /************************************************************************/
4357 : /* InstallHandler() */
4358 : /************************************************************************/
4359 :
4360 57327 : void VSIFileManager::InstallHandler(
4361 : const std::string &osPrefix,
4362 : const std::shared_ptr<VSIFilesystemHandler> &poHandler)
4363 :
4364 : {
4365 57327 : if (osPrefix == "")
4366 1793 : Get()->m_poDefaultHandler = poHandler;
4367 : else
4368 55534 : Get()->m_apoHandlers[osPrefix] = poHandler;
4369 57327 : }
4370 :
4371 : /************************************************************************/
4372 : /* RemoveHandler() */
4373 : /************************************************************************/
4374 :
4375 3 : void VSIFileManager::RemoveHandler(const std::string &osPrefix)
4376 : {
4377 3 : if (osPrefix == "")
4378 0 : Get()->m_poDefaultHandler.reset();
4379 : else
4380 3 : Get()->m_apoHandlers.erase(osPrefix);
4381 3 : }
4382 :
4383 : /************************************************************************/
4384 : /* VSICleanupFileManager() */
4385 : /************************************************************************/
4386 :
4387 1130 : void VSICleanupFileManager()
4388 :
4389 : {
4390 1130 : if (poManager)
4391 : {
4392 1130 : delete poManager;
4393 1130 : poManager = nullptr;
4394 : }
4395 :
4396 1130 : if (hVSIFileManagerMutex != nullptr)
4397 : {
4398 1130 : CPLDestroyMutex(hVSIFileManagerMutex);
4399 1130 : hVSIFileManagerMutex = nullptr;
4400 : }
4401 :
4402 : #ifdef HAVE_CURL
4403 1130 : VSICURLDestroyCacheFileProp();
4404 : #endif
4405 1130 : }
4406 :
4407 : /************************************************************************/
4408 : /* Truncate() */
4409 : /************************************************************************/
4410 :
4411 2 : int VSIVirtualHandle::Truncate(vsi_l_offset nNewSize)
4412 : {
4413 2 : const vsi_l_offset nOriginalPos = Tell();
4414 2 : if (Seek(0, SEEK_END) == 0 && nNewSize >= Tell())
4415 : {
4416 : // Fill with zeroes
4417 2 : std::vector<GByte> aoBytes(4096, 0);
4418 1 : vsi_l_offset nCurOffset = nOriginalPos;
4419 3 : while (nCurOffset < nNewSize)
4420 : {
4421 2 : constexpr vsi_l_offset nMaxOffset = 4096;
4422 : const int nSize =
4423 2 : static_cast<int>(std::min(nMaxOffset, nNewSize - nCurOffset));
4424 2 : if (Write(&aoBytes[0], nSize, 1) != 1)
4425 : {
4426 0 : Seek(nOriginalPos, SEEK_SET);
4427 0 : return -1;
4428 : }
4429 2 : nCurOffset += nSize;
4430 : }
4431 1 : return Seek(nOriginalPos, SEEK_SET) == 0 ? 0 : -1;
4432 : }
4433 :
4434 1 : CPLDebug("VSI", "Truncation is not supported in generic implementation "
4435 : "of Truncate()");
4436 1 : Seek(nOriginalPos, SEEK_SET);
4437 1 : return -1;
4438 : }
4439 :
4440 : /************************************************************************/
4441 : /* ReadMultiRange() */
4442 : /************************************************************************/
4443 :
4444 4785 : int VSIVirtualHandle::ReadMultiRange(int nRanges, void **ppData,
4445 : const vsi_l_offset *panOffsets,
4446 : const size_t *panSizes)
4447 : {
4448 4785 : int nRet = 0;
4449 4785 : const vsi_l_offset nCurOffset = Tell();
4450 74892 : for (int i = 0; i < nRanges; i++)
4451 : {
4452 70126 : if (Seek(panOffsets[i], SEEK_SET) < 0)
4453 : {
4454 0 : nRet = -1;
4455 0 : break;
4456 : }
4457 :
4458 70126 : const size_t nRead = Read(ppData[i], 1, panSizes[i]);
4459 70126 : if (panSizes[i] != nRead)
4460 : {
4461 19 : nRet = -1;
4462 19 : break;
4463 : }
4464 : }
4465 :
4466 4785 : Seek(nCurOffset, SEEK_SET);
4467 :
4468 4785 : return nRet;
4469 : }
4470 :
4471 : #endif // #ifndef DOXYGEN_SKIP
4472 :
4473 : /************************************************************************/
4474 : /* HasPRead() */
4475 : /************************************************************************/
4476 :
4477 : /** Returns whether this file handle supports the PRead() method.
4478 : *
4479 : * @since GDAL 3.6
4480 : */
4481 0 : bool VSIVirtualHandle::HasPRead() const
4482 : {
4483 0 : return false;
4484 : }
4485 :
4486 : /************************************************************************/
4487 : /* PRead() */
4488 : /************************************************************************/
4489 :
4490 : /** Do a parallel-compatible read operation.
4491 : *
4492 : * This methods reads into pBuffer up to nSize bytes starting at offset nOffset
4493 : * in the file. The current file offset is not affected by this method.
4494 : *
4495 : * The implementation is thread-safe: several threads can issue PRead()
4496 : * concurrently on the same VSIVirtualHandle object.
4497 : *
4498 : * This method has the same semantics as pread() Linux operation. It is only
4499 : * available if HasPRead() returns true.
4500 : *
4501 : * @param pBuffer output buffer (must be at least nSize bytes large).
4502 : * @param nSize number of bytes to read in the file.
4503 : * @param nOffset file offset from which to read.
4504 : * @return number of bytes read.
4505 : * @since GDAL 3.6
4506 : */
4507 0 : size_t VSIVirtualHandle::PRead(CPL_UNUSED void *pBuffer,
4508 : CPL_UNUSED size_t nSize,
4509 : CPL_UNUSED vsi_l_offset nOffset) const
4510 : {
4511 0 : return 0;
4512 : }
4513 :
4514 : #ifndef DOXYGEN_SKIP
4515 : /************************************************************************/
4516 : /* VSIProxyFileHandle::CancelCreation() */
4517 : /************************************************************************/
4518 :
4519 14 : void VSIProxyFileHandle::CancelCreation()
4520 : {
4521 14 : m_nativeHandle->CancelCreation();
4522 14 : }
4523 : #endif
4524 :
4525 : /************************************************************************/
4526 : /* VSIURIToVSIPath() */
4527 : /************************************************************************/
4528 :
4529 : /** Return a VSI compatible path from a URI / URL
4530 : *
4531 : * Substitute URI / URLs starting with s3://, gs://, etc. by their VSI
4532 : * prefix equivalent. If no known substitution is found, the input string is
4533 : * returned unmodified.
4534 : *
4535 : * @since GDAL 3.12
4536 : */
4537 108 : std::string VSIURIToVSIPath(const std::string &osURI)
4538 : {
4539 : static const struct
4540 : {
4541 : const char *pszFSSpecPrefix;
4542 : const char *pszVSIPrefix;
4543 : } substitutions[] = {
4544 : {"s3://", "/vsis3/"},
4545 : {"gs://", "/vsigs/"},
4546 : {"gcs://", "/vsigs/"},
4547 : {"az://", "/vsiaz/"},
4548 : {"azure://", "/vsiaz/"},
4549 : {"http://", "/vsicurl/http://"},
4550 : {"https://", "/vsicurl/https://"},
4551 : {"file://", ""},
4552 : };
4553 :
4554 956 : for (const auto &substitution : substitutions)
4555 : {
4556 858 : if (STARTS_WITH(osURI.c_str(), substitution.pszFSSpecPrefix))
4557 : {
4558 20 : return std::string(substitution.pszVSIPrefix)
4559 10 : .append(osURI.c_str() + strlen(substitution.pszFSSpecPrefix));
4560 : }
4561 : }
4562 :
4563 98 : return osURI;
4564 : }
|
