add cw tests for lambda Invocations and Errors metrics
diff --git a/tests/aws/services/cloudwatch/test_cloudwatch_metrics.py b/tests/aws/services/cloudwatch/test_cloudwatch_metrics.py
new file mode 100644
index 0000000000000..11d315a4c30ba
--- /dev/null
+++ b/tests/aws/services/cloudwatch/test_cloudwatch_metrics.py
@@ -0,0 +1,128 @@
+from datetime import datetime, timedelta
+from typing import TYPE_CHECKING
+
+import pytest
+
+from localstack.testing.pytest import markers
+from localstack.testing.pytest.snapshot import is_aws
+from localstack.utils.strings import short_uid
+
+if TYPE_CHECKING:
+    from mypy_boto3_cloudwatch import CloudWatchClient
+from localstack.utils.sync import retry
+
+TEST_SUCCESSFUL_LAMBDA = """
+def handler(event, context):
+    return {"success": "ok"}
+"""
+
+TEST_FAILING_LAMBDA = """
+def handler(event, context):
+    raise Exception('fail on purpose')
+"""
+
+
+class TestCloudWatchLambdaMetrics:
+    """
+    Tests for metrics that are reported automatically by Lambda
+    see also https://docs.aws.amazon.com/lambda/latest/dg/monitoring-metrics.html
+    """
+
+    @markers.aws.validated
+    def test_lambda_invoke_successful(self, aws_client, create_lambda_function, snapshot):
+        """
+        successful invocation of lambda should report "Invocations" metric
+        """
+        fn_name = f"fn-cw-{short_uid()}"
+        create_lambda_function(
+            func_name=fn_name,
+            handler_file=TEST_SUCCESSFUL_LAMBDA,
+            runtime="python3.9",
+        )
+        result = aws_client.lambda_.invoke(FunctionName=fn_name)
+        assert result["StatusCode"] == 200
+        snapshot.match("invoke", result)
+
+        # wait for metrics
+        result = retry(
+            lambda: self._wait_for_lambda_metric(
+                aws_client.cloudwatch,
+                fn_name=fn_name,
+                metric_name="Invocations",
+                expected_return=[1.0],
+            ),
+            retries=200 if is_aws() else 20,
+            sleep=10 if is_aws() else 1,
+        )
+        snapshot.match("get-metric-data", result)
+
+    @pytest.mark.skipif(not is_aws(), reason="'Errors' metrics not reported by LS")
+    @markers.aws.validated
+    def test_lambda_invoke_error(self, aws_client, create_lambda_function, snapshot):
+        """
+        Unsuccessful Invocation -> resulting in error, should report
+        "Errors" and "Invocations" metrics
+        """
+        fn_name = f"fn-cw-{short_uid()}"
+        create_lambda_function(
+            func_name=fn_name,
+            handler_file=TEST_FAILING_LAMBDA,
+            runtime="python3.9",
+        )
+        result = aws_client.lambda_.invoke(FunctionName=fn_name)
+        snapshot.match("invoke", result)
+
+        # wait for metrics
+        invocation_res = retry(
+            lambda: self._wait_for_lambda_metric(
+                aws_client.cloudwatch,
+                fn_name=fn_name,
+                metric_name="Invocations",
+                expected_return=[1.0],
+            ),
+            retries=200 if is_aws() else 20,
+            sleep=10 if is_aws() else 1,
+        )
+        snapshot.match("get-metric-data-invocations", invocation_res)
+
+        # wait for "Errors"
+        error_res = retry(
+            lambda: self._wait_for_lambda_metric(
+                aws_client.cloudwatch,
+                fn_name=fn_name,
+                metric_name="Errors",
+                expected_return=[1.0],
+            ),
+            retries=200 if is_aws() else 20,
+            sleep=10 if is_aws() else 1,
+        )
+        snapshot.match("get-metric-data-errors", error_res)
+
+    def _wait_for_lambda_metric(
+        self,
+        cloudwatch_client: "CloudWatchClient",
+        fn_name: str,
+        metric_name: str,
+        expected_return: list[float],
+    ):
+        namespace = "AWS/Lambda"
+        dimension = [{"Name": "FunctionName", "Value": fn_name}]
+        metric_query = {
+            "Id": "m1",
+            "MetricStat": {
+                "Metric": {
+                    "Namespace": namespace,
+                    "MetricName": metric_name,
+                    "Dimensions": dimension,
+                },
+                "Period": 3600,
+                "Stat": "Sum",
+            },
+        }
+        res = cloudwatch_client.get_metric_data(
+            MetricDataQueries=[metric_query],
+            StartTime=datetime.utcnow() - timedelta(hours=1),
+            EndTime=datetime.utcnow(),
+        )
+        assert res["MetricDataResults"][0]["Values"] == expected_return
+        return res
diff --git a/tests/aws/services/cloudwatch/test_cloudwatch_metrics.snapshot.json b/tests/aws/services/cloudwatch/test_cloudwatch_metrics.snapshot.json
new file mode 100644
index 0000000000000..905cc3a0453eb
--- /dev/null
+++ b/tests/aws/services/cloudwatch/test_cloudwatch_metrics.snapshot.json
@@ -0,0 +1,94 @@
+{
+  "tests/aws/services/cloudwatch/test_cloudwatch_metrics.py::TestCloudWatchLambdaMetrics::test_lambda_invoke_successful": {
+    "recorded-date": "15-11-2023, 19:46:04",
+    "recorded-content": {
+      "invoke": {
+        "ExecutedVersion": "$LATEST",
+        "Payload": {
+          "success": "ok"
+        },
+        "StatusCode": 200,
+        "ResponseMetadata": {
+          "HTTPHeaders": {},
+          "HTTPStatusCode": 200
+        }
+      },
+      "get-metric-data": {
+        "Messages": [],
+        "MetricDataResults": [
+          {
+            "Id": "m1",
+            "Label": "Invocations",
+            "StatusCode": "Complete",
+            "Timestamps": "timestamp",
+            "Values": [
+              1.0
+            ]
+          }
+        ],
+        "ResponseMetadata": {
+          "HTTPHeaders": {},
+          "HTTPStatusCode": 200
+        }
+      }
+    }
+  },
+  "tests/aws/services/cloudwatch/test_cloudwatch_metrics.py::TestCloudWatchLambdaMetrics::test_lambda_invoke_error": {
+    "recorded-date": "15-11-2023, 19:49:06",
+    "recorded-content": {
+      "invoke": {
+        "ExecutedVersion": "$LATEST",
+        "FunctionError": "Unhandled",
+        "Payload": {
+          "errorMessage": "fail on purpose",
+          "errorType": "Exception",
+          "requestId": "<uuid:1>",
+          "stackTrace": [
+            "  File \"/var/task/handler.py\", line 3, in handler\n    raise Exception('fail on purpose')\n"
+          ]
+        },
+        "StatusCode": 200,
+        "ResponseMetadata": {
+          "HTTPHeaders": {},
+          "HTTPStatusCode": 200
+        }
+      },
+      "get-metric-data-invocations": {
+        "Messages": [],
+        "MetricDataResults": [
+          {
+            "Id": "m1",
+            "Label": "Invocations",
+            "StatusCode": "Complete",
+            "Timestamps": "timestamp",
+            "Values": [
+              1.0
+            ]
+          }
+        ],
+        "ResponseMetadata": {
+          "HTTPHeaders": {},
+          "HTTPStatusCode": 200
+        }
+      },
+      "get-metric-data-errors": {
+        "Messages": [],
+        "MetricDataResults": [
+          {
+            "Id": "m1",
+            "Label": "Errors",
+            "StatusCode": "Complete",
+            "Timestamps": "timestamp",
+            "Values": [
+              1.0
+            ]
+          }
+        ],
+        "ResponseMetadata": {
+          "HTTPHeaders": {},
+          "HTTPStatusCode": 200
+        }
+      }
+    }
+  }
+}

<!-- Please refer to the contribution guidelines before raising a PR: https://github.com/localstack/localstack/blob/master/CONTRIBUTING.md -->

<!-- Why am I raising this PR? Add context such as related issues, PRs, or documentation. -->
## Motivation
While testing the pro-samples we found that lambda is not reporting the `Errors` metrics anymore. 
The metric should be reported when a lambda invocation fails.

We didn't have integration tests for lambda metrics yet, so I added those now.
/cc @joe4dev 

<!-- What notable changes does this PR make? -->
## Changes
* added aws-validated tests
* skipping `test_lambda_invoke_error` for LS as it is currently not working -> this should be fixed 🙂 

<!-- The following sections are optional, but can be useful! 

## Testing

Description of how to test the changes

## TODO

What's left to do:

- [ ] ...
- [ ] ...

-->

