Line data Source code
1 : /******************************************************************************
2 : *
3 : * Project: GDAL Core
4 : * Purpose: Python interface
5 : * Author: Even Rouault, <even dot rouault at spatialys dot com>
6 : *
7 : ******************************************************************************
8 : * Copyright (c) 2017-2019, Even Rouault, <even dot rouault at spatialys dot
9 : *com>
10 : *
11 : * Permission is hereby granted, free of charge, to any person obtaining a
12 : * copy of this software and associated documentation files (the "Software"),
13 : * to deal in the Software without restriction, including without limitation
14 : * the rights to use, copy, modify, merge, publish, distribute, sublicense,
15 : * and/or sell copies of the Software, and to permit persons to whom the
16 : * Software is furnished to do so, subject to the following conditions:
17 : *
18 : * The above copyright notice and this permission notice shall be included
19 : * in all copies or substantial portions of the Software.
20 : *
21 : * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
22 : * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
23 : * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
24 : * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
25 : * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
26 : * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
27 : * DEALINGS IN THE SOFTWARE.
28 : ****************************************************************************/
29 :
30 : #include "cpl_conv.h"
31 : #include "cpl_error.h"
32 : #include "cpl_string.h"
33 : #include "cpl_spawn.h"
34 : #include "gdalpython.h"
35 :
36 : #include <algorithm>
37 : #include <mutex>
38 : #include <set>
39 : #include <vector>
40 :
41 : using namespace GDALPy;
42 :
43 : typedef struct PyThreadState_t PyThreadState;
44 :
45 : static PyThreadState *(*PyEval_SaveThread)(void) = nullptr;
46 : static void (*PyEval_RestoreThread)(PyThreadState *) = nullptr;
47 : static void (*Py_Finalize)(void) = nullptr;
48 : static void (*Py_InitializeEx)(int) = nullptr;
49 : static void (*PyEval_InitThreads)(void) = nullptr;
50 : static PyObject *(*Py_CompileStringExFlags)(const char *, const char *, int,
51 : void *, int) = nullptr;
52 :
53 : static std::mutex gMutexGDALPython;
54 : static bool gbHasInitializedPython = false;
55 : static PyThreadState *gphThreadState = nullptr;
56 :
57 : // Emulate Py_CompileString with Py_CompileStringExFlags
58 : // Probably just a temporary measure for a bug of Python 3.8.0 on Windows
59 : // https://bugs.python.org/issue37633
60 0 : static PyObject *GDAL_Py_CompileString(const char *str, const char *filename,
61 : int start)
62 : {
63 0 : return Py_CompileStringExFlags(str, filename, start, nullptr, -1);
64 : }
65 :
66 : namespace GDALPy
67 : {
68 : int (*Py_IsInitialized)(void) = nullptr;
69 : PyGILState_STATE (*PyGILState_Ensure)(void) = nullptr;
70 : void (*PyGILState_Release)(PyGILState_STATE) = nullptr;
71 : void (*Py_SetProgramName)(const wchar_t *) = nullptr;
72 : void (*Py_SetPythonHome)(const wchar_t *) = nullptr;
73 : PyObject *(*PyObject_Type)(PyObject *) = nullptr;
74 : int (*PyObject_IsInstance)(PyObject *, PyObject *) = nullptr;
75 : PyObject *(*PyTuple_New)(size_t) = nullptr;
76 : PyObject *(*PyBool_FromLong)(long) = nullptr;
77 : PyObject *(*PyLong_FromLong)(long) = nullptr;
78 : long (*PyLong_AsLong)(PyObject *) = nullptr;
79 : PyObject *(*PyLong_FromLongLong)(GIntBig) = nullptr;
80 : GIntBig (*PyLong_AsLongLong)(PyObject *) = nullptr;
81 : PyObject *(*PyFloat_FromDouble)(double) = nullptr;
82 : double (*PyFloat_AsDouble)(PyObject *) = nullptr;
83 : PyObject *(*PyObject_Call)(PyObject *, PyObject *, PyObject *) = nullptr;
84 : PyObject *(*PyObject_GetIter)(PyObject *) = nullptr;
85 : PyObject *(*PyIter_Next)(PyObject *) = nullptr;
86 : void (*Py_IncRef)(PyObject *) = nullptr;
87 : void (*Py_DecRef)(PyObject *) = nullptr;
88 : PyObject *(*PyErr_Occurred)(void) = nullptr;
89 : void (*PyErr_Print)(void) = nullptr;
90 :
91 : PyObject *(*Py_CompileString)(const char *, const char *, int) = nullptr;
92 : PyObject *(*PyImport_ExecCodeModule)(const char *, PyObject *) = nullptr;
93 : int (*PyObject_HasAttrString)(PyObject *, const char *) = nullptr;
94 : PyObject *(*PyObject_GetAttrString)(PyObject *, const char *) = nullptr;
95 : int (*PyObject_SetAttrString)(PyObject *, const char *, PyObject *) = nullptr;
96 : int (*PyTuple_SetItem)(PyObject *, size_t, PyObject *) = nullptr;
97 : void (*PyObject_Print)(PyObject *, FILE *, int) = nullptr;
98 : Py_ssize_t (*PyBytes_Size)(PyObject *) = nullptr;
99 : const char *(*PyBytes_AsString)(PyObject *) = nullptr;
100 : int *(*PyBytes_AsStringAndSize)(PyObject *, char **, size_t *) = nullptr;
101 : PyObject *(*PyBytes_FromObject)(PyObject *) = nullptr;
102 : PyObject *(*PyBytes_FromStringAndSize)(const void *, size_t) = nullptr;
103 : PyObject *(*PyUnicode_FromString)(const char *) = nullptr;
104 : PyObject *(*PyUnicode_AsUTF8String)(PyObject *) = nullptr;
105 : PyObject *(*PyImport_ImportModule)(const char *) = nullptr;
106 : int (*PyCallable_Check)(PyObject *) = nullptr;
107 : PyObject *(*PyDict_New)(void) = nullptr;
108 : int (*PyDict_SetItemString)(PyObject *p, const char *key,
109 : PyObject *val) = nullptr;
110 : int (*PyDict_Next)(PyObject *p, size_t *, PyObject **, PyObject **) = nullptr;
111 : PyObject *(*PyDict_GetItemString)(PyObject *p, const char *key) = nullptr;
112 : PyObject *(*PyList_New)(Py_ssize_t) = nullptr;
113 : int (*PyList_SetItem)(PyObject *, Py_ssize_t, PyObject *) = nullptr;
114 : int (*PyArg_ParseTuple)(PyObject *, const char *, ...) = nullptr;
115 :
116 : int (*PySequence_Check)(PyObject *o) = nullptr;
117 : Py_ssize_t (*PySequence_Size)(PyObject *o) = nullptr;
118 : PyObject *(*PySequence_GetItem)(PyObject *o, Py_ssize_t i) = nullptr;
119 :
120 : void (*PyErr_Fetch)(PyObject **poPyType, PyObject **poPyValue,
121 : PyObject **poPyTraceback) = nullptr;
122 : void (*PyErr_Clear)(void) = nullptr;
123 : const char *(*Py_GetVersion)(void) = nullptr;
124 :
125 : int (*PyBuffer_FillInfo)(Py_buffer *view, PyObject *obj, void *buf, size_t len,
126 : int readonly, int infoflags) = nullptr;
127 : PyObject *(*PyMemoryView_FromBuffer)(Py_buffer *view) = nullptr;
128 :
129 : PyObject *(*PyModule_Create2)(struct PyModuleDef *, int) = nullptr;
130 : } // namespace GDALPy
131 :
132 : /* MinGW32 might define HAVE_DLFCN_H, so skip the unix implementation */
133 : #if defined(HAVE_DLFCN_H) && !defined(_WIN32)
134 :
135 : #include <dlfcn.h>
136 :
137 : typedef void *LibraryHandle;
138 :
139 : #define LOAD_NOCHECK_WITH_NAME(libHandle, x, name) \
140 : do \
141 : { \
142 : void *ptr = dlsym(libHandle, name); \
143 : memcpy(&x, &ptr, sizeof(void *)); \
144 : } while (0)
145 :
146 : #elif defined(_WIN32)
147 :
148 : #include <windows.h>
149 : #include <psapi.h>
150 :
151 : typedef HMODULE LibraryHandle;
152 :
153 : #define LOAD_NOCHECK_WITH_NAME(libHandle, x, name) \
154 : do \
155 : { \
156 : FARPROC ptr = GetProcAddress(libHandle, name); \
157 : memcpy(&x, &ptr, sizeof(void *)); \
158 : } while (0)
159 :
160 : #endif
161 :
162 : #define STRINGIFY(x) #x
163 :
164 : #define LOAD_NOCHECK(libHandle, x) \
165 : LOAD_NOCHECK_WITH_NAME(libHandle, x, STRINGIFY(x))
166 : #define LOAD_WITH_NAME(libHandle, x, name) \
167 : do \
168 : { \
169 : LOAD_NOCHECK_WITH_NAME(libHandle, x, name); \
170 : if (!x) \
171 : { \
172 : CPLError(CE_Failure, CPLE_AppDefined, "Cannot find %s", name); \
173 : return false; \
174 : } \
175 : } while (0)
176 : #define LOAD(libHandle, x) LOAD_WITH_NAME(libHandle, x, STRINGIFY(x))
177 :
178 : /************************************************************************/
179 : /* LoadPythonAPI() */
180 : /************************************************************************/
181 :
182 : #if defined(LOAD_NOCHECK_WITH_NAME) && defined(HAVE_DLFCN_H) && !defined(_WIN32)
183 : static LibraryHandle libHandleStatic = nullptr;
184 : #endif
185 :
186 : /** Load the subset of the Python C API that we need */
187 53 : static bool LoadPythonAPI()
188 : {
189 : static bool bInit = false;
190 53 : if (bInit)
191 46 : return true;
192 :
193 : #ifdef LOAD_NOCHECK_WITH_NAME
194 : // The static here is just to avoid Coverity warning about resource leak.
195 7 : LibraryHandle libHandle = nullptr;
196 :
197 7 : const char *pszPythonSO = CPLGetConfigOption("PYTHONSO", nullptr);
198 : #if defined(HAVE_DLFCN_H) && !defined(_WIN32)
199 :
200 : // First try in the current process in case the python symbols would
201 : // be already loaded
202 : (void)libHandle;
203 7 : libHandle = dlopen(nullptr, RTLD_LAZY);
204 7 : libHandleStatic = libHandle;
205 14 : if (libHandle != nullptr &&
206 7 : dlsym(libHandle, "Py_SetProgramName") != nullptr)
207 : {
208 2 : CPLDebug("GDAL", "Current process has python symbols loaded");
209 : }
210 : else
211 : {
212 5 : libHandle = nullptr;
213 : }
214 :
215 : // Then try the user provided shared object name
216 7 : if (libHandle == nullptr && pszPythonSO != nullptr)
217 : {
218 : // coverity[tainted_string]
219 2 : libHandle = dlopen(pszPythonSO, RTLD_NOW | RTLD_GLOBAL);
220 2 : if (libHandle == nullptr)
221 : {
222 1 : CPLError(CE_Failure, CPLE_AppDefined, "Cannot load %s",
223 : pszPythonSO);
224 1 : return false;
225 : }
226 1 : if (dlsym(libHandle, "Py_SetProgramName") == nullptr)
227 : {
228 1 : CPLError(CE_Failure, CPLE_AppDefined,
229 : "Cannot find Py_SetProgramName symbol in %s", pszPythonSO);
230 1 : return false;
231 : }
232 : }
233 :
234 : // Then try the PYTHONSO_DEFAULT if defined at compile time
235 : #ifdef PYTHONSO_DEFAULT
236 : if (libHandle == nullptr)
237 : {
238 : libHandle = dlopen(PYTHONSO_DEFAULT, RTLD_NOW | RTLD_GLOBAL);
239 : if (!libHandle)
240 : {
241 : CPLDebug("GDAL", "%s found", PYTHONSO_DEFAULT);
242 : }
243 : }
244 : #endif
245 :
246 : #if defined(__MACH__) && defined(__APPLE__)
247 : #define SO_EXT "dylib"
248 : #else
249 : #define IS_SO_EXT
250 : #define SO_EXT "so"
251 : #endif
252 :
253 3 : const auto tryDlopen = [](CPLString osPythonSO)
254 : {
255 3 : CPLDebug("GDAL", "Trying %s", osPythonSO.c_str());
256 3 : auto l_libHandle = dlopen(osPythonSO.c_str(), RTLD_NOW | RTLD_GLOBAL);
257 : #ifdef IS_SO_EXT
258 3 : if (l_libHandle == nullptr)
259 : {
260 0 : osPythonSO += ".1.0";
261 0 : CPLDebug("GDAL", "Trying %s", osPythonSO.c_str());
262 0 : l_libHandle = dlopen(osPythonSO.c_str(), RTLD_NOW | RTLD_GLOBAL);
263 : }
264 : #endif
265 3 : return l_libHandle;
266 : };
267 :
268 : // Then try to find the libpython that corresponds to the python binary
269 : // in the PATH
270 5 : if (libHandle == nullptr)
271 : {
272 6 : CPLString osVersion;
273 3 : char *pszPath = getenv("PATH");
274 3 : if (pszPath != nullptr
275 : #ifdef DEBUG
276 : // For testing purposes
277 3 : && CPLTestBool(CPLGetConfigOption("GDAL_ENABLE_PYTHON_PATH", "YES"))
278 : #endif
279 : )
280 : {
281 : const CPLStringList aosPathParts(
282 6 : CSLTokenizeString2(pszPath, ":", 0));
283 3 : for (int iTry = 0; iTry < 2; ++iTry)
284 : {
285 21 : for (const char *pszPathPart : aosPathParts)
286 : {
287 : struct stat sStat;
288 : CPLString osPythonBinary(
289 21 : CPLFormFilename(pszPathPart, "python", nullptr));
290 21 : if (iTry == 0)
291 21 : osPythonBinary += "3";
292 21 : if (lstat(osPythonBinary, &sStat) != 0)
293 18 : continue;
294 :
295 3 : CPLDebug("GDAL", "Found %s", osPythonBinary.c_str());
296 :
297 6 : if (S_ISLNK(sStat.st_mode)
298 : #ifdef DEBUG
299 : // For testing purposes
300 3 : && CPLTestBool(CPLGetConfigOption(
301 : "GDAL_ENABLE_PYTHON_SYMLINK", "YES"))
302 : #endif
303 : )
304 : {
305 6 : std::set<std::string> oSetAlreadyTriedLinks;
306 : while (true)
307 : {
308 3 : oSetAlreadyTriedLinks.insert(osPythonBinary);
309 :
310 : // If this is a symlink, hopefully the resolved
311 : // name will be like "python3.6"
312 3 : const int nBufSize = 2048;
313 3 : std::vector<char> oFilename(nBufSize);
314 3 : char *szPointerFilename = &oFilename[0];
315 3 : int nBytes = static_cast<int>(readlink(
316 3 : osPythonBinary, szPointerFilename, nBufSize));
317 3 : if (nBytes != -1)
318 : {
319 3 : szPointerFilename[std::min(nBytes,
320 3 : nBufSize - 1)] = 0;
321 : CPLString osFilename(
322 3 : CPLGetFilename(szPointerFilename));
323 3 : CPLDebug("GDAL", "Which is an alias to: %s",
324 : szPointerFilename);
325 :
326 3 : if (STARTS_WITH(osFilename, "python"))
327 : {
328 3 : CPLString osResolvedFullLink;
329 : // If the filename is again a symlink,
330 : // resolve it
331 3 : if (CPLIsFilenameRelative(osFilename))
332 : {
333 : osResolvedFullLink = CPLFormFilename(
334 : CPLGetPath(osPythonBinary),
335 3 : osFilename, nullptr);
336 : }
337 : else
338 : {
339 0 : osResolvedFullLink = osFilename;
340 : }
341 0 : if (oSetAlreadyTriedLinks.find(
342 3 : osResolvedFullLink) ==
343 6 : oSetAlreadyTriedLinks.end() &&
344 3 : lstat(osResolvedFullLink, &sStat) ==
345 6 : 0 &&
346 3 : S_ISLNK(sStat.st_mode))
347 : {
348 : osPythonBinary =
349 0 : std::move(osResolvedFullLink);
350 0 : continue;
351 : }
352 :
353 : osVersion =
354 3 : osFilename.substr(strlen("python"));
355 3 : CPLDebug(
356 : "GDAL",
357 : "Python version from binary name: %s",
358 : osVersion.c_str());
359 : }
360 : }
361 : else
362 : {
363 0 : CPLDebug("GDAL", "realink(%s) failed",
364 : osPythonBinary.c_str());
365 : }
366 3 : break;
367 0 : }
368 : }
369 :
370 : // Otherwise, expensive way: start the binary and ask
371 : // it for its version...
372 3 : if (osVersion.empty())
373 : {
374 0 : const char *pszPrintVersion =
375 : "import sys; print(str(sys.version_info[0]) +"
376 : "'.' + str(sys.version_info[1]))";
377 0 : const char *const apszArgv[] = {osPythonBinary.c_str(),
378 : "-c", pszPrintVersion,
379 0 : nullptr};
380 : const CPLString osTmpFilename(
381 0 : "/vsimem/LoadPythonAPI/out.txt");
382 0 : VSILFILE *fout = VSIFOpenL(osTmpFilename, "wb+");
383 0 : if (CPLSpawn(apszArgv, nullptr, fout, FALSE) == 0)
384 : {
385 : char *pszStr =
386 0 : reinterpret_cast<char *>(VSIGetMemFileBuffer(
387 : osTmpFilename, nullptr, FALSE));
388 0 : osVersion = pszStr;
389 0 : if (!osVersion.empty() && osVersion.back() == '\n')
390 : {
391 0 : osVersion.resize(osVersion.size() - 1);
392 : }
393 0 : CPLDebug("GDAL", "Python version from binary: %s",
394 : osVersion.c_str());
395 : }
396 0 : VSIFCloseL(fout);
397 0 : VSIUnlink(osTmpFilename);
398 : }
399 3 : break;
400 : }
401 3 : if (!osVersion.empty())
402 3 : break;
403 : }
404 : }
405 :
406 3 : if (!osVersion.empty())
407 : {
408 3 : libHandle = tryDlopen("libpython" + osVersion + "." SO_EXT);
409 3 : if (libHandle != nullptr)
410 : {
411 3 : CPLDebug("GDAL", "... success");
412 : }
413 0 : else if (osVersion[0] == '3')
414 : {
415 0 : libHandle = tryDlopen("libpython" + osVersion + "m." SO_EXT);
416 0 : if (libHandle != nullptr)
417 : {
418 0 : CPLDebug("GDAL", "... success");
419 : }
420 : }
421 : }
422 : }
423 :
424 : // Otherwise probe a few known objects.
425 : // Note: update doc/source/drivers/raster/vrt.rst if change
426 5 : if (libHandle == nullptr)
427 : {
428 0 : const char *const apszPythonSO[] = {
429 : "libpython3.8." SO_EXT, "libpython3.9." SO_EXT,
430 : "libpython3.10." SO_EXT, "libpython3.11." SO_EXT,
431 : "libpython3.12." SO_EXT, "libpython3.7m." SO_EXT,
432 : "libpython3.6m." SO_EXT, "libpython3.5m." SO_EXT,
433 : "libpython3.4m." SO_EXT, "libpython3.3." SO_EXT,
434 : "libpython3.2." SO_EXT};
435 0 : for (size_t i = 0;
436 0 : libHandle == nullptr && i < CPL_ARRAYSIZE(apszPythonSO); ++i)
437 : {
438 0 : libHandle = tryDlopen(apszPythonSO[i]);
439 0 : if (libHandle != nullptr)
440 0 : CPLDebug("GDAL", "... success");
441 : }
442 : }
443 :
444 : #elif defined(_WIN32)
445 : CPLString osPythonBinaryUsed;
446 :
447 : // First try in the current process in case the python symbols would
448 : // be already loaded
449 : HANDLE hProcess = GetCurrentProcess();
450 : std::vector<HMODULE> ahModules;
451 :
452 : // 100 is not large enough when GDAL is loaded from QGIS for example
453 : ahModules.resize(1000);
454 : for (int i = 0; i < 2; i++)
455 : {
456 : DWORD nSizeNeeded = 0;
457 : const DWORD nSizeIn =
458 : static_cast<DWORD>(ahModules.size() * sizeof(HMODULE));
459 : EnumProcessModules(hProcess, &ahModules[0], nSizeIn, &nSizeNeeded);
460 : ahModules.resize(static_cast<size_t>(nSizeNeeded) / sizeof(HMODULE));
461 : if (nSizeNeeded <= nSizeIn)
462 : {
463 : break;
464 : }
465 : }
466 :
467 : for (size_t i = 0; i < ahModules.size(); i++)
468 : {
469 : if (GetProcAddress(ahModules[i], "Py_SetProgramName"))
470 : {
471 : libHandle = ahModules[i];
472 : CPLDebug("GDAL", "Current process has python symbols loaded");
473 : break;
474 : }
475 : }
476 :
477 : // Then try the user provided shared object name
478 : if (libHandle == nullptr && pszPythonSO != nullptr)
479 : {
480 : UINT uOldErrorMode;
481 : /* Avoid error boxes to pop up (#5211, #5525) */
482 : uOldErrorMode =
483 : SetErrorMode(SEM_NOOPENFILEERRORBOX | SEM_FAILCRITICALERRORS);
484 :
485 : #if (defined(_WIN32) && _MSC_VER >= 1310) || __MSVCRT_VERSION__ >= 0x0601
486 : if (CPLTestBool(CPLGetConfigOption("GDAL_FILENAME_IS_UTF8", "YES")))
487 : {
488 : wchar_t *pwszFilename =
489 : CPLRecodeToWChar(pszPythonSO, CPL_ENC_UTF8, CPL_ENC_UCS2);
490 : libHandle = LoadLibraryW(pwszFilename);
491 : CPLFree(pwszFilename);
492 : }
493 : else
494 : #endif
495 : {
496 : libHandle = LoadLibrary(pszPythonSO);
497 : }
498 :
499 : SetErrorMode(uOldErrorMode);
500 :
501 : if (libHandle == nullptr)
502 : {
503 : CPLError(CE_Failure, CPLE_AppDefined, "Cannot load %s",
504 : pszPythonSO);
505 : return false;
506 : }
507 : if (GetProcAddress(libHandle, "Py_SetProgramName") == nullptr)
508 : {
509 : CPLError(CE_Failure, CPLE_AppDefined,
510 : "Cannot find Py_SetProgramName symbol in %s", pszPythonSO);
511 : return false;
512 : }
513 : }
514 :
515 : // Then try the PYTHONSO_DEFAULT if defined at compile time
516 : #ifdef PYTHONSO_DEFAULT
517 : if (libHandle == nullptr)
518 : {
519 : UINT uOldErrorMode;
520 : uOldErrorMode =
521 : SetErrorMode(SEM_NOOPENFILEERRORBOX | SEM_FAILCRITICALERRORS);
522 :
523 : libHandle = LoadLibrary(PYTHONSO_DEFAULT);
524 : SetErrorMode(uOldErrorMode);
525 : if (!libHandle)
526 : {
527 : CPLDebug("GDAL", "%s found", PYTHONSO_DEFAULT);
528 : }
529 : }
530 : #endif
531 :
532 : // Then try to find the pythonXY.dll that corresponds to the python binary
533 : // in the PATH
534 : if (libHandle == nullptr)
535 : {
536 : CPLString osDLLName;
537 : char *pszPath = getenv("PATH");
538 : if (pszPath != nullptr
539 : #ifdef DEBUG
540 : // For testing purposes
541 : && CPLTestBool(CPLGetConfigOption("GDAL_ENABLE_PYTHON_PATH", "YES"))
542 : #endif
543 : )
544 : {
545 : const CPLStringList aosPathParts(
546 : CSLTokenizeString2(pszPath, ";", 0));
547 : for (int iTry = 0; iTry < 2; ++iTry)
548 : {
549 : for (const char *pszPathPart : aosPathParts)
550 : {
551 : VSIStatBufL sStat;
552 : CPLString osPythonBinary(
553 : CPLFormFilename(pszPathPart, "python.exe", nullptr));
554 : if (iTry == 1)
555 : osPythonBinary += "3";
556 : if (VSIStatL(osPythonBinary, &sStat) != 0)
557 : continue;
558 :
559 : CPLDebug("GDAL", "Found %s", osPythonBinary.c_str());
560 :
561 : {
562 : // Test when dll is in the same directory as the exe
563 : const CPLStringList aosFiles(VSIReadDir(pszPathPart));
564 : for (const char *pszFilename : aosFiles)
565 : {
566 : if ((STARTS_WITH_CI(pszFilename, "python") ||
567 : // mingw64 uses libpython3.X.dll naming
568 : STARTS_WITH_CI(pszFilename, "libpython3.")) &&
569 : // do not load minimum API dll
570 : !EQUAL(pszFilename, "python3.dll") &&
571 : EQUAL(CPLGetExtension(pszFilename), "dll"))
572 : {
573 : osDLLName = CPLFormFilename(
574 : pszPathPart, pszFilename, nullptr);
575 : osPythonBinaryUsed = osPythonBinary;
576 : break;
577 : }
578 : }
579 : }
580 :
581 : // In python3.2, the dll is in the DLLs subdirectory
582 : if (osDLLName.empty())
583 : {
584 : const CPLString osDLLsDir(
585 : CPLFormFilename(pszPathPart, "DLLs", nullptr));
586 : const CPLStringList aosFiles(VSIReadDir(osDLLsDir));
587 : for (const char *pszFilename : aosFiles)
588 : {
589 : if (STARTS_WITH_CI(pszFilename, "python") &&
590 : EQUAL(CPLGetExtension(pszFilename), "dll"))
591 : {
592 : osDLLName = CPLFormFilename(
593 : osDLLsDir, pszFilename, nullptr);
594 : break;
595 : }
596 : }
597 : }
598 :
599 : break;
600 : }
601 : if (!osDLLName.empty())
602 : break;
603 : }
604 : }
605 :
606 : if (!osDLLName.empty())
607 : {
608 : // CPLDebug("GDAL", "Trying %s", osDLLName.c_str());
609 : UINT uOldErrorMode;
610 : uOldErrorMode =
611 : SetErrorMode(SEM_NOOPENFILEERRORBOX | SEM_FAILCRITICALERRORS);
612 : libHandle = LoadLibrary(osDLLName);
613 : SetErrorMode(uOldErrorMode);
614 : if (libHandle != nullptr)
615 : {
616 : CPLDebug("GDAL", "%s loaded", osDLLName.c_str());
617 : }
618 : }
619 : }
620 :
621 : // Otherwise probe a few known objects
622 : // Note: update doc/source/drivers/raster/vrt.rst if change
623 : if (libHandle == nullptr)
624 : {
625 : const char *const apszPythonSO[] = {
626 : "python38.dll", "python39.dll", "python310.dll", "python311.dll",
627 : "python312.dll", "python37.dll", "python36.dll", "python35.dll",
628 : "python34.dll", "python33.dll", "python32.dll"};
629 : UINT uOldErrorMode;
630 : uOldErrorMode =
631 : SetErrorMode(SEM_NOOPENFILEERRORBOX | SEM_FAILCRITICALERRORS);
632 :
633 : for (size_t i = 0;
634 : libHandle == nullptr && i < CPL_ARRAYSIZE(apszPythonSO); ++i)
635 : {
636 : CPLDebug("GDAL", "Trying %s", apszPythonSO[i]);
637 : libHandle = LoadLibrary(apszPythonSO[i]);
638 : if (libHandle != nullptr)
639 : CPLDebug("GDAL", "... success");
640 : }
641 : SetErrorMode(uOldErrorMode);
642 : }
643 : #endif
644 5 : if (!libHandle)
645 : {
646 0 : CPLError(
647 : CE_Failure, CPLE_AppDefined,
648 : "Cannot find python/libpython. You can set the PYTHONSO "
649 : "configuration option to point to the a python .so/.dll/.dylib");
650 0 : return false;
651 : }
652 :
653 5 : LOAD(libHandle, Py_GetVersion);
654 10 : CPLString osPythonVersion(Py_GetVersion());
655 5 : osPythonVersion.replaceAll("\r\n", ' ');
656 5 : osPythonVersion.replaceAll('\n', ' ');
657 5 : CPLDebug("GDAL", "Python version used: %s", osPythonVersion.c_str());
658 :
659 5 : LOAD(libHandle, Py_SetProgramName);
660 5 : LOAD(libHandle, Py_SetPythonHome);
661 :
662 : #ifdef _WIN32
663 : if (!osPythonBinaryUsed.empty() && getenv("PYTHONHOME") == nullptr)
664 : {
665 : const char *pszPythonHome = CPLGetDirname(osPythonBinaryUsed.c_str());
666 : VSIStatBufL sStat;
667 : bool bOK = false;
668 : // Test Windows Conda layout
669 : const char *pszDirEncodings =
670 : CPLFormFilename(pszPythonHome, "lib/encodings", nullptr);
671 : if (VSIStatL(pszDirEncodings, &sStat) == 0)
672 : {
673 : bOK = true;
674 : }
675 : else
676 : {
677 : // Test mingw64 layout
678 : const CPLStringList aosVersionTokens(
679 : CSLTokenizeString2(osPythonVersion.c_str(), ".", 0));
680 : if (aosVersionTokens.size() >= 3)
681 : {
682 : pszPythonHome = CPLGetDirname(pszPythonHome);
683 : pszDirEncodings = CPLFormFilename(
684 : pszPythonHome,
685 : CPLSPrintf("lib/python%s.%s/encodings", aosVersionTokens[0],
686 : aosVersionTokens[1]),
687 : nullptr);
688 : if (VSIStatL(pszDirEncodings, &sStat) == 0)
689 : {
690 : bOK = true;
691 : }
692 : }
693 : }
694 : if (bOK)
695 : {
696 : static wchar_t wszPythonHome[4096];
697 : wchar_t *pwszPythonHome =
698 : CPLRecodeToWChar(pszPythonHome, CPL_ENC_UTF8, CPL_ENC_UCS2);
699 : const size_t nLength = wcslen(pwszPythonHome) + 1;
700 : if (nLength <= sizeof(wszPythonHome))
701 : {
702 : CPLDebug("GDAL", "Call Py_SetPythonHome(%s)", pszPythonHome);
703 : memcpy(wszPythonHome, pwszPythonHome,
704 : nLength * sizeof(wchar_t));
705 : // The string must reside in static storage
706 : Py_SetPythonHome(wszPythonHome);
707 : }
708 : CPLFree(pwszPythonHome);
709 : }
710 : }
711 : #endif
712 :
713 5 : LOAD(libHandle, PyBuffer_FillInfo);
714 5 : LOAD(libHandle, PyMemoryView_FromBuffer);
715 5 : LOAD(libHandle, PyObject_Type);
716 5 : LOAD(libHandle, PyObject_IsInstance);
717 5 : LOAD(libHandle, PyTuple_New);
718 5 : LOAD(libHandle, PyBool_FromLong);
719 5 : LOAD(libHandle, PyLong_FromLong);
720 5 : LOAD(libHandle, PyLong_AsLong);
721 5 : LOAD(libHandle, PyLong_FromLongLong);
722 5 : LOAD(libHandle, PyLong_AsLongLong);
723 5 : LOAD(libHandle, PyBytes_Size);
724 5 : LOAD(libHandle, PyBytes_AsString);
725 5 : LOAD(libHandle, PyBytes_AsStringAndSize);
726 5 : LOAD(libHandle, PyBytes_FromObject);
727 5 : LOAD(libHandle, PyBytes_FromStringAndSize);
728 :
729 5 : LOAD(libHandle, PyModule_Create2);
730 :
731 5 : LOAD_NOCHECK_WITH_NAME(libHandle, PyUnicode_FromString,
732 : "PyUnicode_FromString");
733 5 : if (PyUnicode_FromString == nullptr)
734 : {
735 0 : LOAD_NOCHECK_WITH_NAME(libHandle, PyUnicode_FromString,
736 : "PyUnicodeUCS2_FromString");
737 : }
738 5 : if (PyUnicode_FromString == nullptr)
739 : {
740 0 : LOAD_WITH_NAME(libHandle, PyUnicode_FromString,
741 : "PyUnicodeUCS4_FromString");
742 : }
743 5 : LOAD_NOCHECK_WITH_NAME(libHandle, PyUnicode_AsUTF8String,
744 : "PyUnicode_AsUTF8String");
745 5 : if (PyUnicode_AsUTF8String == nullptr)
746 : {
747 0 : LOAD_NOCHECK_WITH_NAME(libHandle, PyUnicode_AsUTF8String,
748 : "PyUnicodeUCS2_AsUTF8String");
749 : }
750 5 : if (PyUnicode_AsUTF8String == nullptr)
751 : {
752 0 : LOAD_WITH_NAME(libHandle, PyUnicode_AsUTF8String,
753 : "PyUnicodeUCS4_AsUTF8String");
754 : }
755 :
756 5 : LOAD(libHandle, PyFloat_FromDouble);
757 5 : LOAD(libHandle, PyFloat_AsDouble);
758 5 : LOAD(libHandle, PyObject_Call);
759 5 : LOAD(libHandle, PyObject_GetIter);
760 5 : LOAD(libHandle, PyIter_Next);
761 5 : LOAD(libHandle, Py_IncRef);
762 5 : LOAD(libHandle, Py_DecRef);
763 5 : LOAD(libHandle, PyErr_Occurred);
764 5 : LOAD(libHandle, PyErr_Print);
765 5 : LOAD(libHandle, Py_IsInitialized);
766 5 : LOAD(libHandle, Py_InitializeEx);
767 5 : LOAD(libHandle, PyEval_InitThreads);
768 5 : LOAD(libHandle, PyEval_SaveThread);
769 5 : LOAD(libHandle, PyEval_RestoreThread);
770 5 : LOAD(libHandle, Py_Finalize);
771 5 : LOAD_NOCHECK(libHandle, Py_CompileString);
772 5 : if (Py_CompileString == nullptr)
773 : {
774 : // Probably just a temporary measure for a bug of Python 3.8.0 on
775 : // Windows https://bugs.python.org/issue37633
776 0 : LOAD(libHandle, Py_CompileStringExFlags);
777 0 : Py_CompileString = GDAL_Py_CompileString;
778 : }
779 5 : LOAD(libHandle, PyImport_ExecCodeModule);
780 5 : LOAD(libHandle, PyObject_HasAttrString);
781 5 : LOAD(libHandle, PyObject_GetAttrString);
782 5 : LOAD(libHandle, PyObject_SetAttrString);
783 5 : LOAD(libHandle, PyTuple_SetItem);
784 5 : LOAD(libHandle, PyObject_Print);
785 5 : LOAD(libHandle, PyImport_ImportModule);
786 5 : LOAD(libHandle, PyCallable_Check);
787 5 : LOAD(libHandle, PyDict_New);
788 5 : LOAD(libHandle, PyDict_SetItemString);
789 5 : LOAD(libHandle, PyDict_Next);
790 5 : LOAD(libHandle, PyDict_GetItemString);
791 5 : LOAD(libHandle, PyList_New);
792 5 : LOAD(libHandle, PyList_SetItem);
793 5 : LOAD(libHandle, PySequence_Check);
794 5 : LOAD(libHandle, PySequence_Size);
795 5 : LOAD(libHandle, PySequence_GetItem);
796 5 : LOAD(libHandle, PyArg_ParseTuple);
797 5 : LOAD(libHandle, PyGILState_Ensure);
798 5 : LOAD(libHandle, PyGILState_Release);
799 5 : LOAD(libHandle, PyErr_Fetch);
800 5 : LOAD(libHandle, PyErr_Clear);
801 :
802 : #else // LOAD_NOCHECK_WITH_NAME
803 : CPLError(CE_Failure, CPLE_AppDefined,
804 : "This platform doesn't support dynamic loading of "
805 : "libraries") return false;
806 : #endif // LOAD_NOCHECK_WITH_NAME
807 :
808 5 : bInit = true;
809 5 : return bInit;
810 : }
811 :
812 : //! @cond Doxygen_Suppress
813 :
814 : /************************************************************************/
815 : /* GDALPythonInitialize() */
816 : /************************************************************************/
817 :
818 : /** Call this to initialize the Python environment.
819 : */
820 53 : bool GDALPythonInitialize()
821 : {
822 106 : std::lock_guard<std::mutex> guard(gMutexGDALPython);
823 :
824 53 : if (!LoadPythonAPI())
825 2 : return false;
826 :
827 51 : int bIsInitialized = Py_IsInitialized();
828 51 : if (!bIsInitialized)
829 : {
830 3 : gbHasInitializedPython = true;
831 3 : CPLDebug("GDAL", "Before Py_Initialize()");
832 3 : Py_InitializeEx(0);
833 3 : CPLDebug("GDAL", "Py_Initialize()");
834 3 : PyEval_InitThreads();
835 3 : gphThreadState = PyEval_SaveThread();
836 : }
837 :
838 51 : return true;
839 : }
840 :
841 : /************************************************************************/
842 : /* GDALPythonFinalize() */
843 : /************************************************************************/
844 :
845 : /** To be called by GDALDestroy() */
846 387 : void GDALPythonFinalize()
847 : {
848 387 : if (gbHasInitializedPython)
849 : {
850 3 : CPLDebug("GDAL", "Py_Finalize() = %p", Py_Finalize);
851 3 : PyEval_RestoreThread(gphThreadState);
852 3 : Py_Finalize();
853 3 : gbHasInitializedPython = false;
854 3 : gphThreadState = nullptr;
855 : }
856 387 : }
857 :
858 : namespace GDALPy
859 : {
860 :
861 : /************************************************************************/
862 : /* GIL_Holder() */
863 : /************************************************************************/
864 :
865 571 : GIL_Holder::GIL_Holder(bool bExclusiveLock) : m_bExclusiveLock(bExclusiveLock)
866 : {
867 571 : if (bExclusiveLock)
868 : {
869 0 : gMutexGDALPython.lock();
870 : }
871 571 : m_eState = PyGILState_Ensure();
872 571 : }
873 :
874 : /************************************************************************/
875 : /* ~GIL_Holder() */
876 : /************************************************************************/
877 :
878 1142 : GIL_Holder::~GIL_Holder()
879 : {
880 571 : PyGILState_Release(m_eState);
881 571 : if (m_bExclusiveLock)
882 : {
883 0 : gMutexGDALPython.unlock();
884 : }
885 : else
886 : {
887 : }
888 571 : }
889 :
890 : /************************************************************************/
891 : /* GetString() */
892 : /************************************************************************/
893 :
894 1588 : CPLString GetString(PyObject *obj, bool bEmitError)
895 : {
896 1588 : PyObject *unicode = PyUnicode_AsUTF8String(obj);
897 1588 : if (PyErr_Occurred())
898 : {
899 0 : if (bEmitError)
900 : {
901 0 : CPLError(CE_Failure, CPLE_AppDefined, "%s",
902 0 : GetPyExceptionString().c_str());
903 : }
904 0 : return CPLString();
905 : }
906 :
907 1588 : const char *pszRet = PyBytes_AsString(unicode);
908 3176 : CPLString osRet = pszRet ? pszRet : "";
909 1588 : Py_DecRef(unicode);
910 1588 : return osRet;
911 : }
912 :
913 : /************************************************************************/
914 : /* GetPyExceptionString() */
915 : /************************************************************************/
916 :
917 8 : CPLString GetPyExceptionString()
918 : {
919 8 : PyObject *poPyType = nullptr;
920 8 : PyObject *poPyValue = nullptr;
921 8 : PyObject *poPyTraceback = nullptr;
922 :
923 8 : PyErr_Fetch(&poPyType, &poPyValue, &poPyTraceback);
924 8 : if (poPyType)
925 8 : Py_IncRef(poPyType);
926 8 : if (poPyValue)
927 8 : Py_IncRef(poPyValue);
928 8 : if (poPyTraceback)
929 4 : Py_IncRef(poPyTraceback);
930 :
931 : // This is a mess. traceback.format_exception/format_exception_only
932 : // sometimes throw exceptions themselves !
933 : CPLString osPythonCode(
934 : "import traceback\n"
935 : "\n"
936 : "def GDALFormatException2(etype, value):\n"
937 : " try:\n"
938 : " return ''.join(traceback.format_exception_only(etype, value))\n"
939 : " except:\n"
940 : " return (str(etype) + ', ' + str(value))\n"
941 : "\n"
942 : "def GDALFormatException3(etype, value, tb):\n"
943 : //" print(etype, value, tb)\n"
944 : " try:\n"
945 : " return ''.join(traceback.format_exception(etype, value, tb))\n"
946 : " except:\n"
947 16 : " return (str(etype) + ', ' + str(value))\n");
948 :
949 8 : CPLString osRet("An exception occurred in exception formatting code...");
950 :
951 : static int nCounter = 0;
952 16 : CPLString osModuleName(CPLSPrintf("gdal_exception_%d", nCounter));
953 : PyObject *poCompiledString =
954 8 : Py_CompileString(osPythonCode, osModuleName, Py_file_input);
955 8 : if (poCompiledString == nullptr || PyErr_Occurred())
956 : {
957 0 : PyErr_Print();
958 : }
959 : else
960 : {
961 : PyObject *poModule =
962 8 : PyImport_ExecCodeModule(osModuleName, poCompiledString);
963 8 : CPLAssert(poModule);
964 :
965 8 : Py_DecRef(poCompiledString);
966 :
967 : PyObject *poPyGDALFormatException2 =
968 8 : PyObject_GetAttrString(poModule, "GDALFormatException2");
969 8 : CPLAssert(poPyGDALFormatException2);
970 :
971 : PyObject *poPyGDALFormatException3 =
972 8 : PyObject_GetAttrString(poModule, "GDALFormatException3");
973 8 : CPLAssert(poPyGDALFormatException3);
974 :
975 8 : Py_DecRef(poModule);
976 :
977 8 : PyObject *pyArgs = PyTuple_New(poPyTraceback ? 3 : 2);
978 8 : PyTuple_SetItem(pyArgs, 0, poPyType);
979 8 : PyTuple_SetItem(pyArgs, 1, poPyValue);
980 8 : if (poPyTraceback)
981 4 : PyTuple_SetItem(pyArgs, 2, poPyTraceback);
982 8 : PyObject *poPyRet = PyObject_Call(
983 8 : poPyTraceback ? poPyGDALFormatException3 : poPyGDALFormatException2,
984 : pyArgs, nullptr);
985 8 : Py_DecRef(pyArgs);
986 :
987 8 : if (PyErr_Occurred())
988 : {
989 0 : osRet = "An exception occurred in exception formatting code...";
990 0 : PyErr_Print();
991 : }
992 : else
993 : {
994 8 : osRet = GetString(poPyRet, false);
995 8 : Py_DecRef(poPyRet);
996 : }
997 :
998 8 : Py_DecRef(poPyGDALFormatException2);
999 8 : Py_DecRef(poPyGDALFormatException3);
1000 : }
1001 :
1002 8 : if (poPyType)
1003 8 : Py_DecRef(poPyType);
1004 8 : if (poPyValue)
1005 8 : Py_DecRef(poPyValue);
1006 8 : if (poPyTraceback)
1007 4 : Py_DecRef(poPyTraceback);
1008 :
1009 16 : return osRet;
1010 : }
1011 :
1012 : /************************************************************************/
1013 : /* ErrOccurredEmitCPLError() */
1014 : /************************************************************************/
1015 :
1016 2268 : bool ErrOccurredEmitCPLError()
1017 : {
1018 2268 : if (PyErr_Occurred())
1019 : {
1020 3 : CPLError(CE_Failure, CPLE_AppDefined, "%s",
1021 6 : GetPyExceptionString().c_str());
1022 3 : return true;
1023 : }
1024 2265 : return false;
1025 : }
1026 :
1027 : } // namespace GDALPy
1028 :
1029 : //! @endcond
|
