va: Lock wayland display dmabuf formats against concurrent access

If there are multiple Wayland event listeners in different threads we
get the formats and modifiers pushed concurrently which leads to
segfault from GArray methods. This patch protects the array.

The problem occurs e.g. when using vaapipostproc together with Qt
qmlglsink, QtWayland will get the events as well as VAAPI.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/5280>

3 files changed, 32 insertions, 9 deletions

diff --git a/subprojects/gstreamer-vaapi/gst-libs/gst/vaapi/gstvaapidisplay_wayland.c b/subprojects/gstreamer-vaapi/gst-libs/gst/vaapi/gstvaapidisplay_wayland.c
index b519ef5504..93ce1915f8 100644
--- a/subprojects/gstreamer-vaapi/gst-libs/gst/vaapi/gstvaapidisplay_wayland.c
+++ b/subprojects/gstreamer-vaapi/gst-libs/gst/vaapi/gstvaapidisplay_wayland.c
@@ -132,7 +132,9 @@ dmabuf_modifier (void *data, struct zwp_linux_dmabuf_v1 *zwp_linux_dmabuf,
       format, gst_video_format_to_string (gst_vaapi_video_format_from_drm_format
           (format)), drm_format.modifier);
 
+  g_mutex_lock (&priv->dmabuf_formats_lock);
   g_array_append_val (priv->dmabuf_formats, drm_format);
+  g_mutex_unlock (&priv->dmabuf_formats_lock);
 }
 
 static const struct zwp_linux_dmabuf_v1_listener dmabuf_listener = {
@@ -261,7 +263,9 @@ gst_vaapi_display_wayland_close_display (GstVaapiDisplay * display)
   g_clear_pointer (&priv->compositor, wl_compositor_destroy);
   g_clear_pointer (&priv->registry, wl_registry_destroy);
 
+  g_mutex_lock (&priv->dmabuf_formats_lock);
   g_array_unref (priv->dmabuf_formats);
+  g_mutex_unlock (&priv->dmabuf_formats_lock);
 
   if (priv->wl_display) {
     if (!priv->use_foreign_display)
@@ -341,6 +345,7 @@ gst_vaapi_display_wayland_init (GstVaapiDisplayWayland * display)
   display->priv = priv;
   priv->event_fd = -1;
   priv->dmabuf_formats = g_array_new (FALSE, FALSE, sizeof (GstDRMFormat));
+  g_mutex_init (&priv->dmabuf_formats_lock);
 }
 
 static void
diff --git a/subprojects/gstreamer-vaapi/gst-libs/gst/vaapi/gstvaapidisplay_wayland_priv.h b/subprojects/gstreamer-vaapi/gst-libs/gst/vaapi/gstvaapidisplay_wayland_priv.h
index a4e9992ed4..ff6a997e5d 100644
--- a/subprojects/gstreamer-vaapi/gst-libs/gst/vaapi/gstvaapidisplay_wayland_priv.h
+++ b/subprojects/gstreamer-vaapi/gst-libs/gst/vaapi/gstvaapidisplay_wayland_priv.h
@@ -77,6 +77,7 @@ struct _GstVaapiDisplayWaylandPrivate
   struct zwp_linux_dmabuf_v1 *dmabuf;
   struct wl_registry *registry;
   GArray *dmabuf_formats;
+  GMutex dmabuf_formats_lock;
   guint width;
   guint height;
   guint phys_width;
diff --git a/subprojects/gstreamer-vaapi/gst-libs/gst/vaapi/gstvaapiwindow_wayland.c b/subprojects/gstreamer-vaapi/gst-libs/gst/vaapi/gstvaapiwindow_wayland.c
index 4f2c68c767..6071245284 100644
--- a/subprojects/gstreamer-vaapi/gst-libs/gst/vaapi/gstvaapiwindow_wayland.c
+++ b/subprojects/gstreamer-vaapi/gst-libs/gst/vaapi/gstvaapiwindow_wayland.c
@@ -616,26 +616,36 @@ static GstVaapiDmabufStatus
 dmabuf_format_supported (GstVaapiDisplayWaylandPrivate * const priv_display,
     guint format, guint64 modifier)
 {
-  GArray *formats = priv_display->dmabuf_formats;
+  GArray *formats;
   gboolean linear = FALSE;
+  gboolean dontcare = FALSE;
   gint i;
 
+  g_mutex_lock (&priv_display->dmabuf_formats_lock);
+  formats = priv_display->dmabuf_formats;
   for (i = 0; i < formats->len; i++) {
     GstDRMFormat fmt = g_array_index (formats, GstDRMFormat, i);
     if (fmt.format == format && (fmt.modifier == modifier ||
-            (fmt.modifier == DRM_FORMAT_MOD_INVALID && modifier == 0)))
-      return GST_VAAPI_DMABUF_SUCCESS;
+            (fmt.modifier == DRM_FORMAT_MOD_INVALID && modifier == 0))) {
+      dontcare = TRUE;
+      break;
+    }
     if (fmt.format == format && (fmt.modifier == 0 ||
             fmt.modifier == DRM_FORMAT_MOD_INVALID))
       linear = TRUE;
   }
+  g_mutex_unlock (&priv_display->dmabuf_formats_lock);
+
+  if (dontcare)
+    return GST_VAAPI_DMABUF_SUCCESS;
+
   if (linear)
     return GST_VAAPI_DMABUF_BAD_MODIFIER;
   else
     return GST_VAAPI_DMABUF_BAD_FORMAT;
 }
 
-GstVideoFormat
+static GstVideoFormat
 check_format (GstVaapiDisplay * const display, gint index,
     GstVideoFormat expect)
 {
@@ -662,22 +672,25 @@ check_format (GstVaapiDisplay * const display, gint index,
   return format;
 }
 
-GstVideoFormat
+static GstVideoFormat
 choose_next_format (GstVaapiDisplay * const display, gint * next_index)
 {
   GstVaapiDisplayWaylandPrivate *const priv_display =
       GST_VAAPI_DISPLAY_WAYLAND_GET_PRIVATE (display);
-  GArray *formats = priv_display->dmabuf_formats;
+  GArray *formats;
   GstVideoFormat format;
   gint i;
 
+  g_mutex_lock (&priv_display->dmabuf_formats_lock);
+  formats = priv_display->dmabuf_formats;
+
   if (*next_index < 0) {
     *next_index = 0;
     /* try GST_VIDEO_FORMAT_RGBA first */
     for (i = 0; i < formats->len; i++) {
       format = check_format (display, i, GST_VIDEO_FORMAT_RGBA);
       if (format != GST_VIDEO_FORMAT_UNKNOWN)
-        return format;
+        goto out;
     }
   }
 
@@ -685,11 +698,15 @@ choose_next_format (GstVaapiDisplay * const display, gint * next_index)
     format = check_format (display, i, GST_VIDEO_FORMAT_UNKNOWN);
     if (format != GST_VIDEO_FORMAT_UNKNOWN) {
       *next_index = i + 1;
-      return format;
+      goto out;
     }
   }
   *next_index = formats->len;
-  return GST_VIDEO_FORMAT_UNKNOWN;
+  format = GST_VIDEO_FORMAT_UNKNOWN;
+
+out:
+  g_mutex_unlock (&priv_display->dmabuf_formats_lock);
+  return format;
 }
 
 static GstVaapiDmabufStatus