Line data Source code
1 : /******************************************************************************
2 : *
3 : * Name: cpl_userfaultfd.cpp
4 : * Project: CPL - Common Portability Library
5 : * Purpose: Use userfaultfd and VSIL to service page faults
6 : * Author: James McClain, <james.mcclain@gmail.com>
7 : *
8 : ******************************************************************************
9 : * Copyright (c) 2018, Dr. James McClain <james.mcclain@gmail.com>
10 : *
11 : * SPDX-License-Identifier: MIT
12 : ****************************************************************************/
13 :
14 : #ifdef ENABLE_UFFD
15 :
16 : #include <cstdlib>
17 : #include <cinttypes>
18 : #include <cstring>
19 : #include <string>
20 :
21 : #include <errno.h>
22 : #include <fcntl.h>
23 : #include <poll.h>
24 : #include <pthread.h>
25 : #include <sched.h>
26 : #include <signal.h>
27 : #include <unistd.h>
28 :
29 : #include <sys/ioctl.h>
30 : #include <sys/mman.h>
31 : #include <sys/stat.h>
32 : #include <sys/syscall.h>
33 : #include <sys/types.h>
34 : #include <sys/utsname.h>
35 : #include <linux/userfaultfd.h>
36 :
37 : #include "cpl_conv.h"
38 : #include "cpl_error.h"
39 : #include "cpl_userfaultfd.h"
40 : #include "cpl_string.h"
41 : #include "cpl_vsi.h"
42 : #include "cpl_multiproc.h"
43 :
44 : #ifndef UFFD_USER_MODE_ONLY
45 : // The UFFD_USER_MODE_ONLY flag got added in kernel 5.11 which is the one
46 : // used by Ubuntu 20.04, but the linux-libc-dev package corresponds to 5.4
47 : #define UFFD_USER_MODE_ONLY 1
48 : #endif
49 :
50 : #define BAD_MMAP (reinterpret_cast<void *>(-1))
51 : #define MAX_MESSAGES (0x100)
52 :
53 : static int64_t get_page_limit();
54 : static void cpl_uffd_fault_handler(void *ptr);
55 : static void signal_handler(int signal);
56 : static void uffd_cleanup(void *ptr);
57 :
58 : struct cpl_uffd_context
59 : {
60 : bool keep_going = false;
61 :
62 : int uffd = -1;
63 : struct uffdio_register uffdio_register = {};
64 : struct uffd_msg uffd_msgs[MAX_MESSAGES];
65 :
66 : std::string filename = std::string("");
67 :
68 : int64_t page_limit = -1;
69 : int64_t pages_used = 0;
70 :
71 : size_t file_size = 0;
72 : size_t page_size = 0;
73 : void *page_ptr = nullptr;
74 : size_t vma_size = 0;
75 : void *vma_ptr = nullptr;
76 : CPLJoinableThread *thread = nullptr;
77 : };
78 :
79 2 : static void uffd_cleanup(void *ptr)
80 : {
81 2 : struct cpl_uffd_context *ctx = static_cast<struct cpl_uffd_context *>(ptr);
82 :
83 2 : if (!ctx)
84 0 : return;
85 :
86 : // Signal shutdown
87 2 : ctx->keep_going = false;
88 2 : if (ctx->thread)
89 : {
90 2 : CPLJoinThread(ctx->thread);
91 2 : ctx->thread = nullptr;
92 : }
93 :
94 2 : if (ctx->uffd != -1)
95 : {
96 2 : ioctl(ctx->uffd, UFFDIO_UNREGISTER, &ctx->uffdio_register);
97 2 : close(ctx->uffd);
98 2 : ctx->uffd = -1;
99 : }
100 2 : if (ctx->page_ptr && ctx->page_size)
101 2 : munmap(ctx->page_ptr, ctx->page_size);
102 2 : if (ctx->vma_ptr && ctx->vma_size)
103 2 : munmap(ctx->vma_ptr, ctx->vma_size);
104 2 : ctx->page_ptr = nullptr;
105 2 : ctx->vma_ptr = nullptr;
106 2 : ctx->page_size = 0;
107 2 : ctx->vma_size = 0;
108 2 : ctx->pages_used = 0;
109 2 : ctx->page_limit = 0;
110 :
111 2 : delete ctx;
112 :
113 2 : return;
114 : }
115 :
116 : #ifdef HAVE_GCC_WARNING_ZERO_AS_NULL_POINTER_CONSTANT
117 : #pragma GCC diagnostic push
118 : #pragma GCC diagnostic ignored "-Wzero-as-null-pointer-constant"
119 : #endif
120 : static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
121 : #ifdef HAVE_GCC_WARNING_ZERO_AS_NULL_POINTER_CONSTANT
122 : #pragma GCC diagnostic pop
123 : #endif
124 :
125 2 : static int64_t get_page_limit()
126 : {
127 : int64_t retval;
128 2 : const char *variable = CPLGetConfigOption(GDAL_UFFD_LIMIT, nullptr);
129 :
130 2 : if (variable && sscanf(variable, "%" PRId64, &retval) == 1)
131 0 : return retval;
132 : else
133 2 : return -1;
134 : }
135 :
136 2 : static void cpl_uffd_fault_handler(void *ptr)
137 : {
138 2 : struct cpl_uffd_context *ctx = static_cast<struct cpl_uffd_context *>(ptr);
139 : struct uffdio_copy uffdio_copy;
140 : struct pollfd pollfd;
141 :
142 : // Setup pollfd structure
143 2 : pollfd.fd = ctx->uffd;
144 2 : pollfd.events = POLLIN;
145 :
146 : // Open asset for reading
147 2 : VSILFILE *file = VSIFOpenL(ctx->filename.c_str(), "rb");
148 :
149 2 : if (!file)
150 0 : return;
151 :
152 : // Loop until told to stop
153 5 : while (ctx->keep_going)
154 : {
155 : // Poll for event
156 3 : if (poll(&pollfd, 1, 16) == -1)
157 0 : break; // 60Hz when no demand
158 3 : if ((pollfd.revents & POLLERR) || (pollfd.revents & POLLNVAL))
159 : break;
160 3 : if (!(pollfd.revents & POLLIN))
161 1 : continue;
162 :
163 : // Read page fault events
164 : ssize_t bytes_read = static_cast<ssize_t>(
165 2 : read(ctx->uffd, ctx->uffd_msgs, MAX_MESSAGES * sizeof(uffd_msg)));
166 2 : if (bytes_read < 1)
167 : {
168 0 : if (errno == EWOULDBLOCK)
169 0 : continue;
170 : else
171 0 : break;
172 : }
173 :
174 : // If too many pages are in use, evict all pages (evict them from
175 : // RAM and swap, not just to swap). It is impossible to control
176 : // which/when threads access the VMA, so access to the VMA has to
177 : // forbidden while the activity is in progress.
178 : //
179 : // That is done by (1) installing special handlers for SIGSEGV and
180 : // SIGBUS, (2) mprotecting the VMA so that any threads accessing
181 : // it receive either SIGSEGV or SIGBUS (which one is apparently a
182 : // function of the C library, at least on one non-Linux GNU
183 : // system[1]), (3) unregistering the VMA from userfaultfd,
184 : // remapping the VMA to evict the pages, registering the VMA
185 : // again, (4) making the VMA accessible again, and finally (5)
186 : // restoring the previous signal-handling behavior.
187 : //
188 : // [1] https://lists.debian.org/debian-bsd/2011/05/msg00032.html
189 2 : if (ctx->page_limit > 0)
190 : {
191 0 : pthread_mutex_lock(&mutex);
192 0 : if (ctx->pages_used > ctx->page_limit)
193 : {
194 : struct sigaction segv;
195 : struct sigaction old_segv;
196 : struct sigaction bus;
197 : struct sigaction old_bus;
198 :
199 0 : memset(&segv, 0, sizeof(segv));
200 0 : memset(&old_segv, 0, sizeof(old_segv));
201 0 : memset(&bus, 0, sizeof(bus));
202 0 : memset(&old_bus, 0, sizeof(old_bus));
203 :
204 : // Step 1 from the block comment above
205 0 : segv.sa_handler = signal_handler;
206 0 : bus.sa_handler = signal_handler;
207 0 : if (sigaction(SIGSEGV, &segv, &old_segv) == -1)
208 : {
209 0 : CPLError(
210 : CE_Failure, CPLE_AppDefined,
211 : "cpl_uffd_fault_handler: sigaction(SIGSEGV) failed");
212 0 : pthread_mutex_unlock(&mutex);
213 0 : break;
214 : }
215 0 : if (sigaction(SIGBUS, &bus, &old_bus) == -1)
216 : {
217 0 : CPLError(
218 : CE_Failure, CPLE_AppDefined,
219 : "cpl_uffd_fault_handler: sigaction(SIGBUS) failed");
220 0 : pthread_mutex_unlock(&mutex);
221 0 : break;
222 : }
223 :
224 : // WARNING: LACK OF THREAD-SAFETY.
225 : //
226 : // For example, if a user program (or another part of the
227 : // library) installs a SIGSEGV or SIGBUS handler from another
228 : // thread after this one has installed its handlers but before
229 : // this one uninstalls its handlers, the intervening handler
230 : // will be eliminated. There are other examples, as well, but
231 : // there can only be a problems with other threads because the
232 : // faulting thread is blocked here.
233 : //
234 : // This implies that one should not use cpl_virtualmem.h API
235 : // while other threads are actively generating faults that use
236 : // this mechanism.
237 : //
238 : // Having multiple active threads that use this mechanism but
239 : // with no changes to signal-handling in other threads is NOT a
240 : // problem.
241 :
242 : // Step 2
243 0 : if (mprotect(ctx->vma_ptr, ctx->vma_size, PROT_NONE) == -1)
244 : {
245 0 : CPLError(CE_Failure, CPLE_AppDefined,
246 : "cpl_uffd_fault_handler: mprotect() failed");
247 0 : pthread_mutex_unlock(&mutex);
248 0 : break;
249 : }
250 :
251 : // Step 3
252 0 : if (ioctl(ctx->uffd, UFFDIO_UNREGISTER, &ctx->uffdio_register))
253 : {
254 0 : CPLError(CE_Failure, CPLE_AppDefined,
255 : "cpl_uffd_fault_handler: ioctl(UFFDIO_UNREGISTER) "
256 : "failed");
257 0 : pthread_mutex_unlock(&mutex);
258 0 : break;
259 : }
260 0 : ctx->vma_ptr =
261 0 : mmap(ctx->vma_ptr, ctx->vma_size, PROT_NONE,
262 : MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0);
263 0 : if (ctx->vma_ptr == BAD_MMAP)
264 : {
265 0 : CPLError(CE_Failure, CPLE_AppDefined,
266 : "cpl_uffd_fault_handler: mmap() failed");
267 0 : ctx->vma_ptr = nullptr;
268 0 : pthread_mutex_unlock(&mutex);
269 0 : break;
270 : }
271 0 : ctx->pages_used = 0;
272 0 : if (ioctl(ctx->uffd, UFFDIO_REGISTER, &ctx->uffdio_register))
273 : {
274 0 : CPLError(CE_Failure, CPLE_AppDefined,
275 : "cpl_uffd_fault_handler: ioctl(UFFDIO_REGISTER) "
276 : "failed");
277 0 : pthread_mutex_unlock(&mutex);
278 0 : break;
279 : }
280 :
281 : // Step 4. Problem: A thread might attempt to read here (before
282 : // the mprotect) and receive a SIGSEGV or SIGBUS.
283 0 : if (mprotect(ctx->vma_ptr, ctx->vma_size, PROT_READ) == -1)
284 : {
285 0 : CPLError(CE_Failure, CPLE_AppDefined,
286 : "cpl_uffd_fault_handler: mprotect() failed");
287 0 : pthread_mutex_unlock(&mutex);
288 0 : break;
289 : }
290 :
291 : // Step 5. Solution: Cannot unregister special handlers before
292 : // any such threads have been handled by them, so sleep for
293 : // 1/100th of a second.
294 : // Coverity complains about sleeping under a mutex
295 : // coverity[sleep]
296 0 : usleep(10000);
297 0 : if (sigaction(SIGSEGV, &old_segv, nullptr) == -1)
298 : {
299 0 : CPLError(
300 : CE_Failure, CPLE_AppDefined,
301 : "cpl_uffd_fault_handler: sigaction(SIGSEGV) failed");
302 0 : pthread_mutex_unlock(&mutex);
303 0 : break;
304 : }
305 0 : if (sigaction(SIGBUS, &old_bus, nullptr) == -1)
306 : {
307 0 : CPLError(
308 : CE_Failure, CPLE_AppDefined,
309 : "cpl_uffd_fault_handler: sigaction(SIGBUS) failed");
310 0 : pthread_mutex_unlock(&mutex);
311 0 : break;
312 : }
313 : }
314 0 : pthread_mutex_unlock(&mutex);
315 : }
316 :
317 : // Handle page fault events
318 4 : for (int i = 0; i < static_cast<int>(bytes_read / sizeof(uffd_msg));
319 : ++i)
320 : {
321 2 : const uintptr_t fault_addr =
322 2 : ctx->uffd_msgs[i].arg.pagefault.address & ~(ctx->page_size - 1);
323 2 : const uintptr_t offset =
324 2 : fault_addr - reinterpret_cast<uintptr_t>(ctx->vma_ptr);
325 2 : size_t bytes_needed = static_cast<size_t>(ctx->file_size - offset);
326 2 : if (bytes_needed > ctx->page_size)
327 0 : bytes_needed = ctx->page_size;
328 :
329 : // Copy data into page
330 4 : if (VSIFSeekL(file, offset, SEEK_SET) != 0 ||
331 2 : VSIFReadL(ctx->page_ptr, bytes_needed, 1, file) != 1)
332 : {
333 0 : CPLError(CE_Failure, CPLE_FileIO,
334 : "Cannot get %d bytes at offset " CPL_FRMT_GUIB " of "
335 : "file %s",
336 : static_cast<int>(bytes_needed),
337 : static_cast<GUIntBig>(offset), ctx->filename.c_str());
338 0 : memset(ctx->page_ptr, 0, bytes_needed);
339 : }
340 2 : ctx->pages_used++;
341 :
342 : // Use the page to fulfill the page fault
343 2 : uffdio_copy.src = reinterpret_cast<uintptr_t>(ctx->page_ptr);
344 2 : uffdio_copy.dst = fault_addr;
345 2 : uffdio_copy.len = static_cast<uintptr_t>(ctx->page_size);
346 2 : uffdio_copy.mode = 0;
347 2 : uffdio_copy.copy = 0;
348 2 : if (ioctl(ctx->uffd, UFFDIO_COPY, &uffdio_copy) == -1)
349 : {
350 0 : CPLError(CE_Failure, CPLE_AppDefined,
351 : "ioctl(UFFDIO_COPY) failed");
352 0 : break;
353 : }
354 : }
355 : } // end of while loop
356 :
357 : // Return resources
358 2 : VSIFCloseL(file);
359 : }
360 :
361 0 : static void signal_handler(int signal)
362 : {
363 0 : if (signal == SIGSEGV || signal == SIGBUS)
364 0 : sched_yield();
365 0 : return;
366 : }
367 :
368 15 : bool CPLIsUserFaultMappingSupported()
369 : {
370 : // Check the Linux kernel version. Linux 4.3 or newer is needed for
371 : // userfaultfd.
372 15 : int major = 0, minor = 0;
373 : struct utsname utsname;
374 :
375 15 : if (uname(&utsname))
376 0 : return false;
377 15 : sscanf(utsname.release, "%d.%d", &major, &minor);
378 15 : if (major < 4)
379 0 : return false;
380 15 : if (major == 4 && minor < 3)
381 0 : return false;
382 :
383 : static int nEnableUserFaultFD = -1;
384 15 : if (nEnableUserFaultFD < 0)
385 : {
386 11 : nEnableUserFaultFD =
387 11 : CPLTestBool(CPLGetConfigOption("CPL_ENABLE_USERFAULTFD", "YES"));
388 : }
389 15 : if (!nEnableUserFaultFD)
390 0 : return false;
391 :
392 : // Since kernel 5.2, raw userfaultfd is disabled since if the fault
393 : // originates from the kernel, that could lead to easier exploitation of
394 : // kernel bugs. Since kernel 5.11, UFFD_USER_MODE_ONLY can be used to
395 : // restrict the mechanism to faults occurring only from user space, which is
396 : // likely to be our use case.
397 15 : int uffd = static_cast<int>(syscall(
398 15 : __NR_userfaultfd, O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY));
399 15 : if (uffd == -1 && errno == EINVAL)
400 0 : uffd =
401 0 : static_cast<int>(syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK));
402 15 : if (uffd == -1)
403 : {
404 0 : const int l_errno = errno;
405 0 : if (l_errno == EPERM)
406 : {
407 : // Since kernel 5.2
408 0 : CPLDebug(
409 : "GDAL",
410 : "CPLIsUserFaultMappingSupported(): syscall(__NR_userfaultfd) "
411 : "failed: "
412 : "insufficient permission. add CAP_SYS_PTRACE capability, or "
413 : "set /proc/sys/vm/unprivileged_userfaultfd to 1");
414 : }
415 : else
416 : {
417 0 : CPLDebug(
418 : "GDAL",
419 : "CPLIsUserFaultMappingSupported(): syscall(__NR_userfaultfd) "
420 : "failed: "
421 : "error = %d",
422 : l_errno);
423 : }
424 0 : nEnableUserFaultFD = false;
425 0 : return false;
426 : }
427 15 : close(uffd);
428 15 : nEnableUserFaultFD = true;
429 15 : return true;
430 : }
431 :
432 : /*
433 : * Returns nullptr on failure, a valid pointer on success.
434 : */
435 2 : cpl_uffd_context *CPLCreateUserFaultMapping(const char *pszFilename,
436 : void **ppVma, uint64_t *pnVmaSize)
437 : {
438 : VSIStatBufL statbuf;
439 2 : struct cpl_uffd_context *ctx = nullptr;
440 :
441 2 : if (!CPLIsUserFaultMappingSupported())
442 : {
443 0 : CPLError(
444 : CE_Failure, CPLE_NotSupported,
445 : "CPLCreateUserFaultMapping(): Linux kernel 4.3 or newer needed");
446 0 : return nullptr;
447 : }
448 :
449 : // Get the size of the asset
450 2 : if (VSIStatL(pszFilename, &statbuf))
451 0 : return nullptr;
452 :
453 : // Setup the `cpl_uffd_context` struct
454 2 : ctx = new cpl_uffd_context();
455 2 : ctx->keep_going = true;
456 2 : ctx->filename = std::string(pszFilename);
457 2 : ctx->page_limit = get_page_limit();
458 2 : ctx->pages_used = 0;
459 2 : ctx->file_size = static_cast<size_t>(statbuf.st_size);
460 2 : ctx->page_size = static_cast<size_t>(sysconf(_SC_PAGESIZE));
461 2 : ctx->vma_size = static_cast<size_t>(
462 2 : ((static_cast<vsi_l_offset>(statbuf.st_size) / ctx->page_size) + 1) *
463 2 : ctx->page_size);
464 2 : if (ctx->vma_size < static_cast<vsi_l_offset>(statbuf.st_size))
465 : { // Check for overflow
466 0 : uffd_cleanup(ctx);
467 0 : CPLError(
468 : CE_Failure, CPLE_AppDefined,
469 : "CPLCreateUserFaultMapping(): File too large for architecture");
470 0 : return nullptr;
471 : }
472 :
473 : // If the mmap failed, free resources and return
474 2 : ctx->vma_ptr = mmap(nullptr, ctx->vma_size, PROT_READ,
475 : MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
476 2 : if (ctx->vma_ptr == BAD_MMAP)
477 : {
478 0 : ctx->vma_ptr = nullptr;
479 0 : uffd_cleanup(ctx);
480 0 : CPLError(CE_Failure, CPLE_AppDefined,
481 : "CPLCreateUserFaultMapping(): mmap() failed");
482 0 : return nullptr;
483 : }
484 :
485 : // Attempt to acquire a scratch page to use to fulfill requests.
486 2 : ctx->page_ptr =
487 2 : mmap(nullptr, static_cast<size_t>(ctx->page_size),
488 : PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
489 2 : if (ctx->page_ptr == BAD_MMAP)
490 : {
491 0 : ctx->page_ptr = nullptr;
492 0 : uffd_cleanup(ctx);
493 0 : CPLError(CE_Failure, CPLE_AppDefined,
494 : "CPLCreateUserFaultMapping(): mmap() failed");
495 0 : return nullptr;
496 : }
497 :
498 : // Get userfaultfd
499 :
500 : // Since kernel 5.2, raw userfaultfd is disabled since if the fault
501 : // originates from the kernel, that could lead to easier exploitation of
502 : // kernel bugs. Since kernel 5.11, UFFD_USER_MODE_ONLY can be used to
503 : // restrict the mechanism to faults occurring only from user space, which is
504 : // likely to be our use case.
505 2 : ctx->uffd = static_cast<int>(syscall(
506 : __NR_userfaultfd, O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY));
507 2 : if (ctx->uffd == -1 && errno == EINVAL)
508 0 : ctx->uffd =
509 0 : static_cast<int>(syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK));
510 2 : if (ctx->uffd == -1)
511 : {
512 0 : const int l_errno = errno;
513 0 : ctx->uffd = -1;
514 0 : uffd_cleanup(ctx);
515 0 : if (l_errno == EPERM)
516 : {
517 : // Since kernel 5.2
518 0 : CPLError(
519 : CE_Failure, CPLE_AppDefined,
520 : "CPLCreateUserFaultMapping(): syscall(__NR_userfaultfd) "
521 : "failed: "
522 : "insufficient permission. add CAP_SYS_PTRACE capability, or "
523 : "set /proc/sys/vm/unprivileged_userfaultfd to 1");
524 : }
525 : else
526 : {
527 0 : CPLError(CE_Failure, CPLE_AppDefined,
528 : "CPLCreateUserFaultMapping(): syscall(__NR_userfaultfd) "
529 : "failed: "
530 : "error = %d",
531 : l_errno);
532 : }
533 0 : return nullptr;
534 : }
535 :
536 : // Query API
537 : {
538 2 : struct uffdio_api uffdio_api = {};
539 :
540 2 : uffdio_api.api = UFFD_API;
541 2 : uffdio_api.features = 0;
542 :
543 2 : if (ioctl(ctx->uffd, UFFDIO_API, &uffdio_api) == -1)
544 : {
545 0 : uffd_cleanup(ctx);
546 0 : CPLError(CE_Failure, CPLE_AppDefined,
547 : "CPLCreateUserFaultMapping(): ioctl(UFFDIO_API) failed");
548 0 : return nullptr;
549 : }
550 : }
551 :
552 : // Register memory range
553 2 : ctx->uffdio_register.range.start =
554 2 : reinterpret_cast<uintptr_t>(ctx->vma_ptr);
555 2 : ctx->uffdio_register.range.len = ctx->vma_size;
556 2 : ctx->uffdio_register.mode = UFFDIO_REGISTER_MODE_MISSING;
557 :
558 2 : if (ioctl(ctx->uffd, UFFDIO_REGISTER, &ctx->uffdio_register) == -1)
559 : {
560 0 : uffd_cleanup(ctx);
561 0 : CPLError(CE_Failure, CPLE_AppDefined,
562 : "CPLCreateUserFaultMapping(): ioctl(UFFDIO_REGISTER) failed");
563 0 : return nullptr;
564 : }
565 :
566 : // Start handler thread
567 2 : ctx->thread = CPLCreateJoinableThread(cpl_uffd_fault_handler, ctx);
568 2 : if (ctx->thread == nullptr)
569 : {
570 0 : CPLError(
571 : CE_Failure, CPLE_AppDefined,
572 : "CPLCreateUserFaultMapping(): CPLCreateJoinableThread() failed");
573 0 : uffd_cleanup(ctx);
574 0 : return nullptr;
575 : }
576 :
577 2 : *ppVma = ctx->vma_ptr;
578 2 : *pnVmaSize = ctx->vma_size;
579 2 : return ctx;
580 : }
581 :
582 644 : void CPLDeleteUserFaultMapping(cpl_uffd_context *ctx)
583 : {
584 644 : if (ctx)
585 : {
586 2 : uffd_cleanup(ctx);
587 : }
588 644 : }
589 :
590 : #endif // ENABLE_UFFD
|
