vo_wlshm: use memfd_create() instead of shm_open()

This syscall avoids the need to guess an unused filename in /dev/shm and
allows seals to be placed on it.  We immediately return if no fd got
returned, as there isn’t anything we can do otherwise.

Seals especially allow the compositor to drop the SIGBUS protections,
since the kernel promises the fd won’t ever shrink.

This removes support for any platform but Linux from this vo.

video/out/vo.c

@@ -77,7 +77,7 @@ const struct vo_driver *const video_out_drivers[] =
 #if HAVE_DIRECT3D
     &video_out_direct3d,
 #endif
-#if HAVE_WAYLAND
+#if HAVE_WAYLAND && HAVE_MEMFD_CREATE
     &video_out_wlshm,
 #endif
 #if HAVE_XV

video/out/vo_wlshm.c

@@ -73,30 +73,18 @@ static void buffer_destroy(void *p)
     munmap(buf->mpi.planes[0], buf->size);
 }
 
-/* modeled after randname and mkostemps from musl */
-static int alloc_shm(size_t size)
+static int allocate_memfd(size_t size)
 {
-    struct timespec ts;
-    unsigned long r;
-    int i, fd, retries = 100;
-    char template[] = "/mpv-XXXXXX";
+    int fd = memfd_create("mpv", MFD_CLOEXEC | MFD_ALLOW_SEALING);
+    if (fd < 0)
+        return -1;
 
-    do {
-        clock_gettime(CLOCK_REALTIME, &ts);
-        r = ts.tv_nsec * 65537 ^ ((uintptr_t)&ts / 16 + (uintptr_t)template);
-        for (i = 5; i < sizeof(template) - 1; i++, r >>= 5)
-            template[i] = 'A' + (r & 15) + (r & 16) * 2;
+    fcntl(fd, F_ADD_SEALS, F_SEAL_SHRINK | F_SEAL_SEAL);
 
-        fd = shm_open(template, O_RDWR | O_CREAT | O_EXCL, 0600);
-        if (fd >= 0) {
-            shm_unlink(template);
-            if (posix_fallocate(fd, 0, size) == 0)
-                return fd;
-            close(fd);
-            break;
-        }
-    } while (--retries && errno == EEXIST);
+    if (posix_fallocate(fd, 0, size) == 0)
+        return fd;
 
+    close(fd);
     return -1;
 }
 
@@ -112,7 +100,7 @@ static struct buffer *buffer_create(struct vo *vo, int width, int height)
 
     stride = MP_ALIGN_UP(width * 4, 16);
     size = height * stride;
-    fd = alloc_shm(size);
+    fd = allocate_memfd(size);
     if (fd < 0)
         goto error0;
     data = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);

wscript

@@ -322,6 +322,12 @@ iconv support use --disable-iconv.",
         'deps': 'os-linux',
         'func': check_statement('sys/vfs.h',
                                 'struct statfs fs; fstatfs(0, &fs); fs.f_namelen')
+    }, {
+        'name': 'memfd_create',
+        'desc': "Linux's memfd_create()",
+        'deps': 'os-linux',
+        'func': check_statement('sys/mman.h',
+                                'memfd_create("mpv", MFD_CLOEXEC | MFD_ALLOW_SEALING)')
     }, {
         'name': '--libsmbclient',
         'desc': 'Samba support (makes mpv GPLv3)',

wscript_build.py

@@ -488,7 +488,7 @@ def build(ctx):
         ( "video/out/vo_tct.c" ),
         ( "video/out/vo_vaapi.c",                "vaapi-x11 && gpl" ),
         ( "video/out/vo_vdpau.c",                "vdpau" ),
-        ( "video/out/vo_wlshm.c",                "wayland" ),
+        ( "video/out/vo_wlshm.c",                "wayland && memfd_create" ),
         ( "video/out/vo_x11.c" ,                 "x11" ),
         ( "video/out/vo_xv.c",                   "xv" ),
         ( "video/out/vulkan/context.c",          "vulkan" ),