[opentelemetry][callback] Support loops error handling (#3599)

* [opentelemetry][callback] simplify condition

* [opentelemetry][callback] initial support for loops

* [opentelemetry][callback] support for loops

* [opentelemetry][callback] enrich loop message with action and item

* [opentelemetry][callback] add UTs for get_error_message_from_results

* [opentelemetry][callback] add UTs for enrich_error_message_from_results

* [opentelemetry][callback] add changelog fragment

* Update plugins/callback/opentelemetry.py

Co-authored-by: Ajpantuso <ajpantuso@gmail.com>

Co-authored-by: Ajpantuso <ajpantuso@gmail.com>

changelogs/fragments/3599-callback_opentelemetry-enriched_errors_in_loops.yml

@@ -0,0 +1,2 @@
+minor_changes:
+  - opentelemetry callback plugin - enriched the stacktrace information for loops with the ``message``, ``exception`` and ``stderr`` fields from the failed item in the tasks in addition to the name of the task and failed item (https://github.com/ansible-collections/community.general/pull/3599).

plugins/callback/opentelemetry.py

@@ -245,22 +245,27 @@ class OpenTelemetrySource(object):
         name = '[%s] %s: %s' % (host_data.name, task_data.play, task_data.name)
 
         message = 'success'
-        status = Status(status_code=StatusCode.OK)
+        res = {}
-        if host_data.status == 'included':
         rc = 0
+        status = Status(status_code=StatusCode.OK)
+        if host_data.status != 'included':
+            # Support loops
+            if 'results' in host_data.result._result:
+                if host_data.status == 'failed':
+                    message = self.get_error_message_from_results(host_data.result._result['results'], task_data.action)
+                    enriched_error_message = self.enrich_error_message_from_results(host_data.result._result['results'], task_data.action)
             else:
                 res = host_data.result._result
                 rc = res.get('rc', 0)
-            if host_data.status == 'failed':
                 message = self.get_error_message(res)
+                enriched_error_message = self.enrich_error_message(res)
+
+            if host_data.status == 'failed':
                 status = Status(status_code=StatusCode.ERROR, description=message)
                 # Record an exception with the task message
-                span.record_exception(BaseException(self.enrich_error_message(res)))
+                span.record_exception(BaseException(enriched_error_message))
             elif host_data.status == 'skipped':
-                if 'skip_reason' in res:
+                message = res['skip_reason'] if 'skip_reason' in res else 'skipped'
-                    message = res['skip_reason']
-                else:
-                    message = 'skipped'
                 status = Status(status_code=StatusCode.UNSET)
 
         span.set_status(status)
@@ -340,6 +345,12 @@ class OpenTelemetrySource(object):
             return OpenTelemetrySource._last_line(result['exception'])
         return result.get('msg', 'failed')
 
+    @staticmethod
+    def get_error_message_from_results(results, action):
+        for result in results:
+            if result.get('failed', False):
+                return ('{0}({1}) - {2}').format(action, result.get('item', 'none'), OpenTelemetrySource.get_error_message(result))
+
     @staticmethod
     def _last_line(text):
         lines = text.strip().split('\n')
@@ -352,6 +363,14 @@ class OpenTelemetrySource(object):
         stderr = result.get('stderr')
         return ('message: "{0}"\nexception: "{1}"\nstderr: "{2}"').format(message, exception, stderr)
 
+    @staticmethod
+    def enrich_error_message_from_results(results, action):
+        message = ""
+        for result in results:
+            if result.get('failed', False):
+                message = ('{0}({1}) - {2}\n{3}').format(action, result.get('item', 'none'), OpenTelemetrySource.enrich_error_message(result), message)
+        return message
+
 
 class CallbackModule(CallbackBase):
     """

tests/unit/plugins/callback/test_opentelemetry.py

@@ -103,6 +103,20 @@ class TestOpentelemetry(unittest.TestCase):
             result = self.opentelemetry.get_error_message(generate_test_data(tc[0], tc[1], tc[2]))
             self.assertEqual(result, tc[3])
 
+    def test_get_error_message_from_results(self):
+        test_cases = (
+            ('my-exception', 'my-msg', None, False, None),
+            (None, 'my-msg', None, False, None),
+            (None, None, None, False, None),
+            ('my-exception', 'my-msg', None, True, 'shell(none) - my-exception'),
+            (None, 'my-msg', None, True, 'shell(none) - my-msg'),
+            (None, None, None, True, 'shell(none) - failed'),
+        )
+
+        for tc in test_cases:
+            result = self.opentelemetry.get_error_message_from_results([generate_test_data(tc[0], tc[1], tc[2], tc[3])], 'shell')
+            self.assertEqual(result, tc[4])
+
     def test_enrich_error_message(self):
         test_cases = (
             ('my-exception', 'my-msg', 'my-stderr', 'message: "my-msg"\nexception: "my-exception"\nstderr: "my-stderr"'),
@@ -116,6 +130,24 @@ class TestOpentelemetry(unittest.TestCase):
             result = self.opentelemetry.enrich_error_message(generate_test_data(tc[0], tc[1], tc[2]))
             self.assertEqual(result, tc[3])
 
+    def test_enrich_error_message_from_results(self):
+        test_cases = (
+            ('my-exception', 'my-msg', 'my-stderr', False, ''),
+            ('my-exception', None, 'my-stderr', False, ''),
+            (None, 'my-msg', 'my-stderr', False, ''),
+            ('my-exception', 'my-msg', None, False, ''),
+            ('my-exception', 'my-msg', '\nline1\nline2', False, ''),
+            ('my-exception', 'my-msg', 'my-stderr', True, 'shell(none) - message: "my-msg"\nexception: "my-exception"\nstderr: "my-stderr"\n'),
+            ('my-exception', None, 'my-stderr', True, 'shell(none) - message: "failed"\nexception: "my-exception"\nstderr: "my-stderr"\n'),
+            (None, 'my-msg', 'my-stderr', True, 'shell(none) - message: "my-msg"\nexception: "None"\nstderr: "my-stderr"\n'),
+            ('my-exception', 'my-msg', None, True, 'shell(none) - message: "my-msg"\nexception: "my-exception"\nstderr: "None"\n'),
+            ('my-exception', 'my-msg', '\nline1\nline2', True, 'shell(none) - message: "my-msg"\nexception: "my-exception"\nstderr: "\nline1\nline2"\n')
+        )
+
+        for tc in test_cases:
+            result = self.opentelemetry.enrich_error_message_from_results([generate_test_data(tc[0], tc[1], tc[2], tc[3])], 'shell')
+            self.assertEqual(result, tc[4])
+
     def test_url_from_args(self):
         test_cases = (
             ({}, ""),
@@ -148,7 +180,7 @@ class TestOpentelemetry(unittest.TestCase):
                 self.assertEqual(result, tc[1])
 
 
-def generate_test_data(exception=None, msg=None, stderr=None):
+def generate_test_data(exception=None, msg=None, stderr=None, failed=False):
     res_data = OrderedDict()
     if exception:
         res_data['exception'] = exception
@@ -156,4 +188,5 @@ def generate_test_data(exception=None, msg=None, stderr=None):
         res_data['msg'] = msg
     if stderr:
         res_data['stderr'] = stderr
+    res_data['failed'] = failed
     return res_data