Bug: fix(opensearch): handle index not present errors in search api



Currently in the self deployment mode we rely on fluentd to automatically create the index,

in this case when we spin up the local setup the msearch API errors out since the index is not present,
with the following json response

```json
{
  "took": 17,
  "responses": [
    {
      "took": 15,
      "timed_out": false,
      "_shards": {
        "total": 1,
        "successful": 1,
        "skipped": 0,
        "failed": 0
      },
      "hits": {
        "total": {
          "value": 0,
          "relation": "eq"
        },
        "max_score": null,
        "hits": []
      },
      "status": 200
    },
    {
      "took": 12,
      "timed_out": false,
      "_shards": {
        "total": 1,
        "successful": 1,
        "skipped": 0,
        "failed": 0
      },
      "hits": {
        "total": {
          "value": 0,
          "relation": "eq"
        },
        "max_score": null,
        "hits": []
      },
      "status": 200
    },
    {
      "error": {
        "root_cause": [
          {
            "type": "index_not_found_exception",
            "reason": "no such index [hyperswitch-refund-events]",
            "index": "hyperswitch-refund-events",
            "resource.id": "hyperswitch-refund-events",
            "resource.type": "index_or_alias",
            "index_uuid": "_na_"
          }
        ],
        "type": "index_not_found_exception",
        "reason": "no such index [hyperswitch-refund-events]",
        "index": "hyperswitch-refund-events",
        "resource.id": "hyperswitch-refund-events",
        "resource.type": "index_or_alias",
        "index_uuid": "_na_"
      },
      "status": 404
    },
    {
      "error": {
        "root_cause": [
          {
            "type": "index_not_found_exception",
            "reason": "no such index [hyperswitch-dispute-events]",
            "index": "hyperswitch-dispute-events",
            "resource.id": "hyperswitch-dispute-events",
            "resource.type": "index_or_alias",
            "index_uuid": "_na_"
          }
        ],
        "type": "index_not_found_exception",
        "reason": "no such index [hyperswitch-dispute-events]",
        "index": "hyperswitch-dispute-events",
        "resource.id": "hyperswitch-dispute-events",
        "resource.type": "index_or_alias",
        "index_uuid": "_na_"
      },
      "status": 404
    }
  ]
}
```


In this case it would be better if we treat missing index as 0 hits and log the corresponding errors.

diff --git a/crates/analytics/docs/README.md b/crates/analytics/docs/README.md
index b822edd810e..da9a3c79f1f 100644
--- a/crates/analytics/docs/README.md
+++ b/crates/analytics/docs/README.md
@@ -101,4 +101,18 @@ Here's an example of how to do this:
 [default.features]
 audit_trail=true
 system_metrics=true
-```
\ No newline at end of file
+global_search=true
+```
+
+## Viewing the data on OpenSearch Dashboard
+
+To view the data on the OpenSearch dashboard perform the following steps:
+
+- Go to the OpenSearch Dashboard home and click on `Stack Management` under the Management tab
+- Select `Index Patterns`
+- Click on `Create index pattern`
+- Define an index pattern with the same name that matches your indices and click on `Next Step`
+- Select a time field that will be used for time-based queries
+- Save the index pattern
+
+Now, head on to the `Discover` tab, to select the newly created index pattern and query the data
\ No newline at end of file
diff --git a/crates/analytics/src/opensearch.rs b/crates/analytics/src/opensearch.rs
index 7b19ba0ed06..e8f87aaef2e 100644
--- a/crates/analytics/src/opensearch.rs
+++ b/crates/analytics/src/opensearch.rs
@@ -76,6 +76,8 @@ pub enum OpenSearchError {
     ResponseError,
     #[error("Opensearch query building error")]
     QueryBuildingError,
+    #[error("Opensearch deserialisation error")]
+    DeserialisationError,
 }
 
 impl ErrorSwitch<OpenSearchError> for QueryBuildingError {
@@ -111,6 +113,12 @@ impl ErrorSwitch<ApiErrorResponse> for OpenSearchError {
                 "Query building error",
                 None,
             )),
+            Self::DeserialisationError => ApiErrorResponse::InternalServerError(ApiError::new(
+                "IR",
+                0,
+                "Deserialisation error",
+                None,
+            )),
         }
     }
 }
diff --git a/crates/analytics/src/search.rs b/crates/analytics/src/search.rs
index dc802ff6948..8810dc1e3a1 100644
--- a/crates/analytics/src/search.rs
+++ b/crates/analytics/src/search.rs
@@ -4,7 +4,7 @@ use api_models::analytics::search::{
 };
 use common_utils::errors::{CustomResult, ReportSwitchExt};
 use error_stack::ResultExt;
-use serde_json::Value;
+use router_env::tracing;
 use strum::IntoEnumIterator;
 
 use crate::opensearch::{
@@ -22,27 +22,59 @@ pub async fn msearch_results(
         .add_filter_clause("merchant_id".to_string(), merchant_id.to_string())
         .switch()?;
 
-    let response_body = client
+    let response_text: OpenMsearchOutput = client
         .execute(query_builder)
         .await
         .change_context(OpenSearchError::ConnectionError)?
-        .json::<OpenMsearchOutput<Value>>()
+        .text()
         .await
-        .change_context(OpenSearchError::ResponseError)?;
+        .change_context(OpenSearchError::ResponseError)
+        .and_then(|body: String| {
+            serde_json::from_str::<OpenMsearchOutput>(&body)
+                .change_context(OpenSearchError::DeserialisationError)
+                .attach_printable(body.clone())
+        })?;
+
+    let response_body: OpenMsearchOutput = response_text;
 
     Ok(response_body
         .responses
         .into_iter()
         .zip(SearchIndex::iter())
-        .map(|(index_hit, index)| GetSearchResponse {
-            count: index_hit.hits.total.value,
-            index,
-            hits: index_hit
-                .hits
-                .hits
-                .into_iter()
-                .map(|hit| hit._source)
-                .collect(),
+        .map(|(index_hit, index)| match index_hit {
+            OpensearchOutput::Success(success) => {
+                if success.status == 200 {
+                    GetSearchResponse {
+                        count: success.hits.total.value,
+                        index,
+                        hits: success
+                            .hits
+                            .hits
+                            .into_iter()
+                            .map(|hit| hit.source)
+                            .collect(),
+                    }
+                } else {
+                    tracing::error!("Unexpected status code: {}", success.status,);
+                    GetSearchResponse {
+                        count: 0,
+                        index,
+                        hits: Vec::new(),
+                    }
+                }
+            }
+            OpensearchOutput::Error(error) => {
+                tracing::error!(
+                    index = ?index,
+                    error_response = ?error,
+                    "Search error"
+                );
+                GetSearchResponse {
+                    count: 0,
+                    index,
+                    hits: Vec::new(),
+                }
+            }
         })
         .collect())
 }
@@ -65,22 +97,54 @@ pub async fn search_results(
         .set_offset_n_count(search_req.offset, search_req.count)
         .switch()?;
 
-    let response_body = client
+    let response_text: OpensearchOutput = client
         .execute(query_builder)
         .await
         .change_context(OpenSearchError::ConnectionError)?
-        .json::<OpensearchOutput<Value>>()
+        .text()
         .await
-        .change_context(OpenSearchError::ResponseError)?;
+        .change_context(OpenSearchError::ResponseError)
+        .and_then(|body: String| {
+            serde_json::from_str::<OpensearchOutput>(&body)
+                .change_context(OpenSearchError::DeserialisationError)
+                .attach_printable(body.clone())
+        })?;
+
+    let response_body: OpensearchOutput = response_text;
 
-    Ok(GetSearchResponse {
-        count: response_body.hits.total.value,
-        index: req.index,
-        hits: response_body
-            .hits
-            .hits
-            .into_iter()
-            .map(|hit| hit._source)
-            .collect(),
-    })
+    match response_body {
+        OpensearchOutput::Success(success) => {
+            if success.status == 200 {
+                Ok(GetSearchResponse {
+                    count: success.hits.total.value,
+                    index: req.index,
+                    hits: success
+                        .hits
+                        .hits
+                        .into_iter()
+                        .map(|hit| hit.source)
+                        .collect(),
+                })
+            } else {
+                tracing::error!("Unexpected status code: {}", success.status);
+                Ok(GetSearchResponse {
+                    count: 0,
+                    index: req.index,
+                    hits: Vec::new(),
+                })
+            }
+        }
+        OpensearchOutput::Error(error) => {
+            tracing::error!(
+                index = ?req.index,
+                error_response = ?error,
+                "Search error"
+            );
+            Ok(GetSearchResponse {
+                count: 0,
+                index: req.index,
+                hits: Vec::new(),
+            })
+        }
+    }
 }
diff --git a/crates/api_models/src/analytics/search.rs b/crates/api_models/src/analytics/search.rs
index 6f6a3f22812..c29bb0e9f71 100644
--- a/crates/api_models/src/analytics/search.rs
+++ b/crates/api_models/src/analytics/search.rs
@@ -48,19 +48,40 @@ pub struct GetSearchResponse {
 }
 
 #[derive(Debug, serde::Deserialize)]
-pub struct OpenMsearchOutput<T> {
-    pub responses: Vec<OpensearchOutput<T>>,
+pub struct OpenMsearchOutput {
+    pub responses: Vec<OpensearchOutput>,
 }
 
 #[derive(Debug, serde::Deserialize)]
-pub struct OpensearchOutput<T> {
-    pub hits: OpensearchResults<T>,
+#[serde(untagged)]
+pub enum OpensearchOutput {
+    Success(OpensearchSuccess),
+    Error(OpensearchError),
 }
 
 #[derive(Debug, serde::Deserialize)]
-pub struct OpensearchResults<T> {
+pub struct OpensearchError {
+    pub error: OpensearchErrorDetails,
+    pub status: u16,
+}
+
+#[derive(Debug, serde::Deserialize)]
+pub struct OpensearchErrorDetails {
+    #[serde(rename = "type")]
+    pub error_type: String,
+    pub reason: String,
+}
+
+#[derive(Debug, serde::Deserialize)]
+pub struct OpensearchSuccess {
+    pub status: u16,
+    pub hits: OpensearchHits,
+}
+
+#[derive(Debug, serde::Deserialize)]
+pub struct OpensearchHits {
     pub total: OpensearchResultsTotal,
-    pub hits: Vec<OpensearchHits<T>>,
+    pub hits: Vec<OpensearchHit>,
 }
 
 #[derive(Debug, serde::Deserialize)]
@@ -69,6 +90,7 @@ pub struct OpensearchResultsTotal {
 }
 
 #[derive(Debug, serde::Deserialize)]
-pub struct OpensearchHits<T> {
-    pub _source: T,
+pub struct OpensearchHit {
+    #[serde(rename = "_source")]
+    pub source: Value,
 }

## Type of Change
<!-- Put an `x` in the boxes that apply -->

- [x] Bugfix
- [ ] New feature
- [ ] Enhancement
- [ ] Refactoring
- [ ] Dependency updates
- [ ] Documentation
- [ ] CI/CD

## Description
<!-- Describe your changes in detail -->
- Handled errors when index is not present, by logging the corresponding error message and treating the hits as 0.
- Changed the structure of `OpensearchOutput` and corresponding structs which are used to handle the error.

### Additional Changes

- [ ] This PR modifies the API contract
- [ ] This PR modifies the database schema
- [ ] This PR modifies application configuration/environment variables

<!--
Provide links to the files with corresponding changes.

Following are the paths where you can find config files:
1. `config`
2. `crates/router/src/configs`
3. `loadtest/config`
-->


## Motivation and Context
<!--
Why is this change required? What problem does it solve?
If it fixes an open issue, please link to the issue here.

If you don't have an issue, we'd recommend starting with one first so the PR
can focus on the implementation (unless it is an obvious bug or documentation fix
that will have little conversation).
-->
The msearch API errors out since the index is not present, so handled the error.

## How did you test it?
<!--
Did you write an integration/unit/API test to verify the code changes?
Or did you test this change manually (provide relevant screenshots)?
-->

- Set up the global search locally
- Test the global search with any keyword
- Go through the log, and `5xx errors` should not be present anymore, with the errors logged, and handling of the index not found errors.
![image](https://github.com/juspay/hyperswitch/assets/83278309/5c7334b3-a9a8-4db8-9d31-447364c2ff0c)


## Checklist
<!-- Put an `x` in the boxes that apply -->

- [x] I formatted the code `cargo +nightly fmt --all`
- [x] I addressed lints thrown by `cargo clippy`
- [x] I reviewed the submitted code
- [ ] I added unit tests for my changes where possible
