validate: Add a mechanism to mark tests as skipped

And use it when a plugin is missing and the user didn't ask for
failure when it happens

And use the TAP[0] synthax to report it

[0]: https://testanything.org

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-devtools/-/merge_requests/187>

11 files changed, 92 insertions, 22 deletions

diff --git a/docs/gst-validate-config.md b/docs/gst-validate-config.md
index c2f5e0784b..a4a9c5ab89 100644
--- a/docs/gst-validate-config.md
+++ b/docs/gst-validate-config.md
@@ -60,6 +60,10 @@ Default: `GST_CLOCK_TIME_NONE` - disabled
 The maximum number of dropped buffers, a `config::too-many-buffers-dropped` issue will be reported
 if that limit is reached.
 
+### `fail-on-missing-plugin`
+
+Default: `false` meaning that tests are marked as skipped when a GStreamer plugin is missing.
+
 ## Variables
 
 You can use variables in the configs the same way you can set them in
diff --git a/validate/gst/validate/gst-validate-override-registry.c b/validate/gst/validate/gst-validate-override-registry.c
index 32073e39ff..e7aa09c8cc 100644
--- a/validate/gst/validate/gst-validate-override-registry.c
+++ b/validate/gst/validate/gst-validate-override-registry.c
@@ -182,7 +182,7 @@ static void
   for (iter = registry->name_overrides.head; iter; iter = g_list_next (iter)) {
     entry = iter->data;
     if (g_regex_match_simple (entry->name, name, 0, 0)) {
-      GST_INFO_OBJECT (registry, "Adding override %s to %s", entry->name, name);
+      GST_INFO ("%p Adding override %s to %s", registry, entry->name, name);
 
       gst_validate_monitor_attach_override (monitor, entry->override);
     }
diff --git a/validate/gst/validate/gst-validate-pipeline-monitor.c b/validate/gst/validate/gst-validate-pipeline-monitor.c
index e0676d242f..1b434cb318 100644
--- a/validate/gst/validate/gst-validate-pipeline-monitor.c
+++ b/validate/gst/validate/gst-validate-pipeline-monitor.c
@@ -28,6 +28,9 @@
 #include "gst-validate-pipeline-monitor.h"
 #include "gst-validate-pad-monitor.h"
 #include "gst-validate-monitor-factory.h"
+#include "gst-validate-report.h"
+#include "gst-validate-utils.h"
+#include "validate.h"
 
 #define PRINT_POSITION_TIMEOUT 250
 
@@ -579,8 +582,13 @@ _bus_handler (GstBus * bus, GstMessage * message,
       gst_message_parse_error_details (message, &details);
 
       if (g_error_matches (err, GST_CORE_ERROR, GST_CORE_ERROR_MISSING_PLUGIN)) {
-        GST_VALIDATE_REPORT (monitor, MISSING_PLUGIN,
-            "Error: %s -- Debug message: %s", err->message, debug);
+        if (!gst_validate_fail_on_missing_plugin ()) {
+          gst_validate_skip_test ("missing plugin: %s -- Debug message: %s\n",
+              err->message, debug);
+        } else {
+          GST_VALIDATE_REPORT (monitor, MISSING_PLUGIN,
+              "Error: %s -- Debug message: %s", err->message, debug);
+        }
       } else if ((g_error_matches (err, GST_STREAM_ERROR,
                   GST_STREAM_ERROR_FAILED) && details
               && gst_structure_get_int (details, "flow-return", &error_flow)
diff --git a/validate/gst/validate/gst-validate-report.c b/validate/gst/validate/gst-validate-report.c
index 558c640078..cd30dc13f9 100644
--- a/validate/gst/validate/gst-validate-report.c
+++ b/validate/gst/validate/gst-validate-report.c
@@ -26,6 +26,7 @@
 #endif
 
 
+#include <stdlib.h>             /* exit */
 #include <stdio.h>              /* fprintf */
 #include <glib/gstdio.h>
 #include <errno.h>
@@ -1277,6 +1278,38 @@ gst_validate_print_position (GstClockTime position, GstClockTime duration,
   g_free (extra_info);
 }
 
+void
+gst_validate_skip_test (const gchar * format, ...)
+{
+  JsonBuilder *jbuilder;
+  va_list va_args;
+  gchar *tmp;
+
+  va_start (va_args, format);
+  tmp = gst_info_strdup_vprintf (format, va_args);
+  va_end (va_args);
+
+  if (!server_ostream) {
+    gchar *f = g_strconcat ("ok 1 # SKIP ", tmp, NULL);
+
+    g_free (tmp);
+    gst_validate_printf (NULL, "%s", f);
+    return;
+  }
+
+  jbuilder = json_builder_new ();
+  json_builder_begin_object (jbuilder);
+  json_builder_set_member_name (jbuilder, "type");
+  json_builder_add_string_value (jbuilder, "skip-test");
+  json_builder_set_member_name (jbuilder, "details");
+  json_builder_add_string_value (jbuilder, tmp);
+  json_builder_end_object (jbuilder);
+  g_free (tmp);
+
+  gst_validate_send (json_builder_get_root (jbuilder));
+  g_object_unref (jbuilder);
+}
+
 static void
 print_issue (gpointer key, GstValidateIssue * issue, gpointer user_data)
 {
diff --git a/validate/gst/validate/gst-validate-report.h b/validate/gst/validate/gst-validate-report.h
index 6029f1a8c2..1643b1b911 100644
--- a/validate/gst/validate/gst-validate-report.h
+++ b/validate/gst/validate/gst-validate-report.h
@@ -318,6 +318,8 @@ GST_VALIDATE_API
 void gst_validate_error_structure (gpointer action, const gchar* format, ...) G_GNUC_PRINTF (2, 3);
 GST_VALIDATE_API
 void gst_validate_abort (const gchar * format, ...) G_GNUC_PRINTF (1, 2);
+GST_VALIDATE_API
+void gst_validate_skip_test (const gchar* format, ...);
 G_END_DECLS
 
 #endif /* __GST_VALIDATE_REPORT_H__ */
diff --git a/validate/gst/validate/gst-validate-utils.c b/validate/gst/validate/gst-validate-utils.c
index 0700f555c4..7d5dc57016 100644
--- a/validate/gst/validate/gst-validate-utils.c
+++ b/validate/gst/validate/gst-validate-utils.c
@@ -40,6 +40,7 @@
 
 #include "gst-validate-utils.h"
 #include "gst-validate-internal.h"
+#include "validate.h"
 #include <gst/gst.h>
 
 #define PARSER_BOOLEAN_EQUALITY_THRESHOLD (1e-10)
@@ -1323,3 +1324,20 @@ gst_validate_set_test_file_globals (GstStructure * meta, const gchar * testfile,
       "videosink", G_TYPE_STRING, videosink,
       "audiosink", G_TYPE_STRING, audiosink, NULL);
 }
+
+gboolean
+gst_validate_fail_on_missing_plugin (void)
+{
+
+  GList *config;
+
+  for (config = gst_validate_plugin_get_config (NULL); config;
+      config = config->next) {
+    gboolean fail_on_missing_plugin;
+
+    if (gst_structure_get_boolean (config->data,
+            "fail-on-missing-plugin", &fail_on_missing_plugin))
+      return fail_on_missing_plugin;
+  }
+  return FALSE;
+}
diff --git a/validate/gst/validate/gst-validate-utils.h b/validate/gst/validate/gst-validate-utils.h
index e6d89fd353..f1f158aa6b 100644
--- a/validate/gst/validate/gst-validate-utils.h
+++ b/validate/gst/validate/gst-validate-utils.h
@@ -79,5 +79,7 @@ GST_VALIDATE_API
 void gst_validate_structure_resolve_variables (GstStructure *structure, GstStructure *local_variables);
 void gst_validate_structure_set_variables_from_struct_file(GstStructure* vars, const gchar* struct_file);
 void gst_validate_set_globals(GstStructure* structure);
+GST_VALIDATE_API
+gboolean gst_validate_fail_on_missing_plugin(void);
 
 #endif
diff --git a/validate/launcher/baseclasses.py b/validate/launcher/baseclasses.py
index c356a5aef2..1bc50c5b17 100644
--- a/validate/launcher/baseclasses.py
+++ b/validate/launcher/baseclasses.py
@@ -187,7 +187,7 @@ class Test(Loggable):
 
     def add_env_variable(self, variable, value=None):
         """
-        Only usefull so that the gst-validate-launcher can print the exact
+        Only useful so that the gst-validate-launcher can print the exact
         right command line to reproduce the tests
         """
         if value is None:
@@ -356,7 +356,7 @@ class Test(Loggable):
         self.message = message
         self.error_str = error
 
-        if result not in [Result.PASSED, Result.NOT_RUN]:
+        if result not in [Result.PASSED, Result.NOT_RUN, Result.SKIPPED]:
             self.add_known_issue_information()
 
     def check_results(self):
@@ -753,6 +753,8 @@ class GstValidateListener(socketserver.BaseRequestHandler, Loggable):
                 test.actions_infos[-1]['execution-duration'] = obj['execution-duration']
             elif obj_type == 'report':
                 test.add_report(obj)
+            elif obj_type == 'skip-test':
+                test.set_result(Result.SKIPPED)
 
 
 class GstValidateTest(Test):
@@ -1004,15 +1006,9 @@ class GstValidateTest(Test):
         return result, msg
 
     def check_results(self):
-        if self.result in [Result.FAILED, self.result is Result.PASSED]:
+        if self.result in [Result.FAILED, Result.PASSED, Result.SKIPPED]:
             return
 
-        for report in self.reports:
-            if report.get('issue-id') == 'runtime::missing-plugin':
-                self.set_result(Result.SKIPPED, "%s\n%s" % (report['summary'],
-                                                            report['details']))
-                return
-
         self.debug("%s returncode: %s", self, self.process.returncode)
 
         expected_issues = copy.deepcopy(self.expected_issues)
@@ -1090,9 +1086,9 @@ class GstValidateTest(Test):
                 msg += " (Expected errors not found: %s) " % mandatory_failures
                 result = Result.FAILED
         elif self.expected_issues:
-            msg += ' %s(Expected errors occured: %s)%s' % (Colors.OKBLUE,
-                                                           self.expected_issues,
-                                                           Colors.ENDC)
+            msg += ' %s(Expected errors occurred: %s)%s' % (Colors.OKBLUE,
+                                                            self.expected_issues,
+                                                            Colors.ENDC)
             result = Result.KNOWN_ERROR
 
         self.set_result(result, msg.strip())
@@ -1321,7 +1317,7 @@ class GstValidateEncodingTestInterface(object):
                                     'summary': 'Expected stream caps during transcoding do not match expectations',
                                     'level': 'critical',
                                     'detected-on': 'pipeline',
-                                    'details': "Field: %s  (from %s) not in caps of the outputed file %s" % (
+                                    'details': "Field: %s  (from %s) not in caps of the outputted file %s" % (
                                         wanted_caps, c, ccaps)
                                 }
                             )
@@ -1374,7 +1370,7 @@ class TestsManager(Loggable):
                     if regex.findall(test.classname):
                         if failure_def.get('allow_flakiness'):
                             test.allow_flakiness = True
-                            self.debug("%s allow flakyness" % (test.classname))
+                            self.debug("%s allow flakiness" % (test.classname))
                         else:
                             for issue in failure_def['issues']:
                                 issue['bug'] = bugid
@@ -1395,7 +1391,7 @@ class TestsManager(Loggable):
                 if regex.findall(test.classname):
                     if failure_def.get('allow_flakiness'):
                         test.allow_flakiness = True
-                        self.debug("%s allow flakyness" % (test.classname))
+                        self.debug("%s allow flakiness" % (test.classname))
                     else:
                         for issue in failure_def['issues']:
                             issue['bug'] = bugid
@@ -2155,7 +2151,7 @@ class Scenario(object):
         return False
 
     def compatible_with_live_content(self):
-        # if a live content is required it's implicitely compatible with
+        # if a live content is required it's implicitly compatible with
         # live content
         if self.needs_live_content():
             return True
@@ -2342,7 +2338,7 @@ class GstValidateBaseTestManager(TestsManager):
         @scenarios A list or a unic scenario name(s) to be run on the tests.
                     They are just the default scenarios, and then depending on
                     the TestsGenerator to be used you can have more fine grained
-                    control on what to be run on each serie of tests.
+                    control on what to be run on each series of tests.
         """
         if isinstance(scenarios, list):
             self._scenarios.extend(scenarios)
@@ -2367,7 +2363,7 @@ class GstValidateBaseTestManager(TestsManager):
                            formats for transcoding test.
                            They are just the default encoding formats, and then depending on
                            the TestsGenerator to be used you can have more fine grained
-                           control on what to be run on each serie of tests.
+                           control on what to be run on each series of tests.
         """
         if isinstance(encoding_formats, list):
             self._encoding_formats.extend(encoding_formats)
@@ -2563,7 +2559,7 @@ class GstValidateMediaDescriptor(MediaDescriptor):
         for ext in [self.MEDIA_INFO_EXT, self.PUSH_MEDIA_INFO_EXT, self.STREAM_INFO_EXT,
                 self.SKIPPED_MEDIA_INFO_EXT, ]:
             if self._xml_path.endswith(ext):
-               return self._xml_path[:len(self._xml_path) - (len(ext) + 1)]
+                return self._xml_path[:len(self._xml_path) - (len(ext) + 1)]
 
         assert "Not reached" is None
 
diff --git a/validate/launcher/reporters.py b/validate/launcher/reporters.py
index 0bec1ae499..3f100bd695 100644
--- a/validate/launcher/reporters.py
+++ b/validate/launcher/reporters.py
@@ -110,6 +110,8 @@ class Reporter(Loggable):
                    Colors.OKBLUE)
         printc("%sPassed: %d" %
                (lenstat * " ", self.stats["passed"]), Colors.OKGREEN)
+        printc("%sSkipped: %d" %
+               (lenstat * " ", self.stats["skipped"]), Colors.WARNING)
         printc("%sFailed: %d" %
                (lenstat * " ", self.stats["failures"]), Colors.FAIL)
         printc("%sKnown error: %d" %
diff --git a/validate/tools/gst-validate-rtsp-server.c b/validate/tools/gst-validate-rtsp-server.c
index 00abbd5d55..017b2ad1b1 100644
--- a/validate/tools/gst-validate-rtsp-server.c
+++ b/validate/tools/gst-validate-rtsp-server.c
@@ -20,6 +20,7 @@
 /* Cc'd from the test-uri example */
 #include <gst/gst.h>
 
+#include <gst/validate/validate.h>
 #include <gst/rtsp-server/rtsp-server.h>
 #include <gst/rtsp-server/rtsp-media-factory-uri.h>
 
diff --git a/validate/tools/gst-validate.c b/validate/tools/gst-validate.c
index 77682d75ff..f91035360d 100644
--- a/validate/tools/gst-validate.c
+++ b/validate/tools/gst-validate.c
@@ -503,6 +503,10 @@ main (int argc, gchar ** argv)
     g_free (argvn);
     exit (1);
   } else if (err) {
+    if (g_error_matches (err, GST_PARSE_ERROR, GST_PARSE_ERROR_NO_SUCH_ELEMENT)) {
+      if (!gst_validate_fail_on_missing_plugin ())
+        gst_validate_skip_test ("missing plugin: %s", err->message);
+    }
     g_printerr ("Erroneous pipeline: %s\n",
         err->message ? err->message : "unknown reason");
     g_clear_error (&err);