Bug: Deep health check failing because of OpenSearch



When you don't have OpenSearch deployed for Global Search feature on dashboard , the `/health/ready` ( deep health check ) API which checks health of all the underlying queryable resources fails because if OpenSearch is not present the health check fails which should not be the case.
diff --git a/config/config.example.toml b/config/config.example.toml
index 5f13eb1388d..b422892afc0 100644
--- a/config/config.example.toml
+++ b/config/config.example.toml
@@ -887,6 +887,7 @@ region = "kms_region" # The AWS region used by the KMS SDK for decrypting data.
 
 [opensearch]
 host = "https://localhost:9200"
+enabled = false
 
 [opensearch.auth]
 auth = "basic"
diff --git a/config/deployments/env_specific.toml b/config/deployments/env_specific.toml
index b2ee7f39741..5621a4940e8 100644
--- a/config/deployments/env_specific.toml
+++ b/config/deployments/env_specific.toml
@@ -242,6 +242,7 @@ region = "report_download_config_region"                     # Region of the buc
 
 [opensearch]
 host = "https://localhost:9200"
+enabled = false
 
 [opensearch.auth]
 auth = "basic"
diff --git a/config/development.toml b/config/development.toml
index 89e35448af2..9bf2051b1d7 100644
--- a/config/development.toml
+++ b/config/development.toml
@@ -968,6 +968,7 @@ keys = "accept-language,user-agent,x-profile-id"
 
 [opensearch]
 host = "https://localhost:9200"
+enabled = false
 
 [opensearch.auth]
 auth = "basic"
diff --git a/config/docker_compose.toml b/config/docker_compose.toml
index cdf82f60936..59b36d73fe2 100644
--- a/config/docker_compose.toml
+++ b/config/docker_compose.toml
@@ -763,6 +763,7 @@ keys = "accept-language,user-agent,x-profile-id"
 
 [opensearch]
 host = "https://opensearch:9200"
+enabled = false
 
 [opensearch.auth]
 auth = "basic"
diff --git a/crates/analytics/src/opensearch.rs b/crates/analytics/src/opensearch.rs
index 726e38e5e16..20be0116d99 100644
--- a/crates/analytics/src/opensearch.rs
+++ b/crates/analytics/src/opensearch.rs
@@ -71,6 +71,8 @@ pub struct OpenSearchConfig {
     host: String,
     auth: OpenSearchAuth,
     indexes: OpenSearchIndexes,
+    #[serde(default)]
+    enabled: bool,
 }
 
 impl Default for OpenSearchConfig {
@@ -91,12 +93,15 @@ impl Default for OpenSearchConfig {
                 sessionizer_refunds: "sessionizer-refund-events".to_string(),
                 sessionizer_disputes: "sessionizer-dispute-events".to_string(),
             },
+            enabled: false,
         }
     }
 }
 
 #[derive(Debug, thiserror::Error)]
 pub enum OpenSearchError {
+    #[error("Opensearch is not enabled")]
+    NotEnabled,
     #[error("Opensearch connection error")]
     ConnectionError,
     #[error("Opensearch NON-200 response content: '{0}'")]
@@ -176,6 +181,12 @@ impl ErrorSwitch<ApiErrorResponse> for OpenSearchError {
                 "Access Forbidden error",
                 None,
             )),
+            Self::NotEnabled => ApiErrorResponse::InternalServerError(ApiError::new(
+                "IR",
+                8,
+                "Opensearch is not enabled",
+                None,
+            )),
         }
     }
 }
@@ -408,15 +419,24 @@ impl OpenSearchAuth {
 }
 
 impl OpenSearchConfig {
-    pub async fn get_opensearch_client(&self) -> StorageResult<OpenSearchClient> {
-        Ok(OpenSearchClient::create(self)
-            .await
-            .map_err(|_| StorageError::InitializationError)?)
+    pub async fn get_opensearch_client(&self) -> StorageResult<Option<OpenSearchClient>> {
+        if !self.enabled {
+            return Ok(None);
+        }
+        Ok(Some(
+            OpenSearchClient::create(self)
+                .await
+                .change_context(StorageError::InitializationError)?,
+        ))
     }
 
     pub fn validate(&self) -> Result<(), ApplicationError> {
         use common_utils::{ext_traits::ConfigExt, fp_utils::when};
 
+        if !self.enabled {
+            return Ok(());
+        }
+
         when(self.host.is_default_or_empty(), || {
             Err(ApplicationError::InvalidConfigurationValueError(
                 "Opensearch host must not be empty".into(),
@@ -430,6 +450,7 @@ impl OpenSearchConfig {
         Ok(())
     }
 }
+
 #[derive(Debug, serde::Deserialize, PartialEq)]
 #[serde(rename_all = "lowercase")]
 pub enum OpenSearchHealthStatus {
diff --git a/crates/router/src/analytics.rs b/crates/router/src/analytics.rs
index cc242f86a5b..ca6b870f889 100644
--- a/crates/router/src/analytics.rs
+++ b/crates/router/src/analytics.rs
@@ -2245,7 +2245,10 @@ pub mod routes {
                     .collect();
 
                 analytics::search::msearch_results(
-                    &state.opensearch_client,
+                    state
+                        .opensearch_client
+                        .as_ref()
+                        .ok_or_else(|| error_stack::report!(OpenSearchError::NotEnabled))?,
                     req,
                     search_params,
                     SEARCH_INDEXES.to_vec(),
@@ -2392,9 +2395,16 @@ pub mod routes {
                             })
                     })
                     .collect();
-                analytics::search::search_results(&state.opensearch_client, req, search_params)
-                    .await
-                    .map(ApplicationResponse::Json)
+                analytics::search::search_results(
+                    state
+                        .opensearch_client
+                        .as_ref()
+                        .ok_or_else(|| error_stack::report!(OpenSearchError::NotEnabled))?,
+                    req,
+                    search_params,
+                )
+                .await
+                .map(ApplicationResponse::Json)
             },
             &auth::JWTAuth {
                 permission: Permission::ProfileAnalyticsRead,
diff --git a/crates/router/src/core/health_check.rs b/crates/router/src/core/health_check.rs
index 2e5c2956013..c4892c94d6f 100644
--- a/crates/router/src/core/health_check.rs
+++ b/crates/router/src/core/health_check.rs
@@ -138,12 +138,15 @@ impl HealthCheckInterface for app::SessionState {
     async fn health_check_opensearch(
         &self,
     ) -> CustomResult<HealthState, errors::HealthCheckDBError> {
-        self.opensearch_client
-            .deep_health_check()
-            .await
-            .change_context(errors::HealthCheckDBError::OpensearchError)?;
-
-        Ok(HealthState::Running)
+        if let Some(client) = self.opensearch_client.as_ref() {
+            client
+                .deep_health_check()
+                .await
+                .change_context(errors::HealthCheckDBError::OpensearchError)?;
+            Ok(HealthState::Running)
+        } else {
+            Ok(HealthState::NotApplicable)
+        }
     }
 
     async fn health_check_outgoing(
diff --git a/crates/router/src/routes/app.rs b/crates/router/src/routes/app.rs
index f6386a9e2c5..4ecd093f250 100644
--- a/crates/router/src/routes/app.rs
+++ b/crates/router/src/routes/app.rs
@@ -118,7 +118,7 @@ pub struct SessionState {
     pub base_url: String,
     pub tenant: Tenant,
     #[cfg(feature = "olap")]
-    pub opensearch_client: Arc<OpenSearchClient>,
+    pub opensearch_client: Option<Arc<OpenSearchClient>>,
     pub grpc_client: Arc<GrpcClients>,
     pub theme_storage_client: Arc<dyn FileStorageInterface>,
     pub locale: String,
@@ -223,7 +223,7 @@ pub struct AppState {
     #[cfg(feature = "olap")]
     pub pools: HashMap<id_type::TenantId, AnalyticsProvider>,
     #[cfg(feature = "olap")]
-    pub opensearch_client: Arc<OpenSearchClient>,
+    pub opensearch_client: Option<Arc<OpenSearchClient>>,
     pub request_id: Option<RequestId>,
     pub file_storage_client: Arc<dyn FileStorageInterface>,
     pub encryption_client: Arc<dyn EncryptionManagementInterface>,
@@ -342,12 +342,12 @@ impl AppState {
 
             #[allow(clippy::expect_used)]
             #[cfg(feature = "olap")]
-            let opensearch_client = Arc::new(
-                conf.opensearch
-                    .get_opensearch_client()
-                    .await
-                    .expect("Failed to create opensearch client"),
-            );
+            let opensearch_client = conf
+                .opensearch
+                .get_opensearch_client()
+                .await
+                .expect("Failed to initialize OpenSearch client.")
+                .map(Arc::new);
 
             #[allow(clippy::expect_used)]
             let cache_store = get_cache_store(&conf.clone(), shut_down_signal, testable)
@@ -501,7 +501,7 @@ impl AppState {
             #[cfg(feature = "email")]
             email_client: Arc::clone(&self.email_client),
             #[cfg(feature = "olap")]
-            opensearch_client: Arc::clone(&self.opensearch_client),
+            opensearch_client: self.opensearch_client.clone(),
             grpc_client: Arc::clone(&self.grpc_client),
             theme_storage_client: self.theme_storage_client.clone(),
             locale: locale.unwrap_or(common_utils::consts::DEFAULT_LOCALE.to_string()),

## Type of Change
<!-- Put an `x` in the boxes that apply -->

- [x] Bugfix
- [ ] New feature
- [ ] Enhancement
- [x] Refactoring
- [ ] Dependency updates
- [ ] Documentation
- [ ] CI/CD

## Description
<!-- Describe your changes in detail -->

Making `OpenSearchClient` optional based on the config for querying OpenSearch Server for dashboard based on the Deployment status of the same ( OpenSearch Server ).

### Additional Changes

- [ ] This PR modifies the API contract
- [ ] This PR modifies the database schema
- [x] This PR modifies application configuration/environment variables

<!--
Provide links to the files with corresponding changes.

Following are the paths where you can find config files:
1. `config`
2. `crates/router/src/configs`
3. `loadtest/config`
-->


## Motivation and Context
<!--
Why is this change required? What problem does it solve?
If it fixes an open issue, please link to the issue here.

If you don't have an issue, we'd recommend starting with one first so the PR
can focus on the implementation (unless it is an obvious bug or documentation fix
that will have little conversation).
-->

When you don't have OpenSearch deployed for Global Search feature on dashboard , the `/health/ready` ( deep health check ) API which checks health of all the underlying queryable resources fails because OpenSearch is not present.

To make it optional based on the deployment type needs changes and should be dependent on a config which should be given out on time of router deployment

Currently if you don't have OpenSearch Deployment and have router running
```bash
curl http://localhost:8080/health/ready -v
* Host localhost:8080 was resolved.
* IPv6: ::1
* IPv4: 127.0.0.1
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0*   Trying [::1]:8080...
* Connected to localhost (::1) port 8080
> GET /health/ready HTTP/1.1
> Host: localhost:8080
> User-Agent: curl/8.7.1
> Accept: */*
> 
* Request completely sent off
< HTTP/1.1 500 Internal Server Error
< content-length: 136
< access-control-expose-headers: x-request-id, content-type, via, strict-transport-security
< via: HyperSwitch
< x-request-id: 019634f7-a837-7d50-9a50-bb8e7651bd62
< content-type: application/json
< vary: Origin, Access-Control-Request-Method, Access-Control-Request-Headers
< strict-transport-security: max-age=31536000
< date: Mon, 14 Apr 2025 15:42:47 GMT
< 
{ [136 bytes data]
100   136  100   136    0     0   7613      0 --:--:-- --:--:-- --:--:--  8000
* Connection #0 to host localhost left intact
{
  "error": {
    "type": "api",
    "message": "Opensearch health check failed with error: Error while executing query in Opensearch",
    "code": "HE_00"
  }
}

```

## How did you test it?
<!--
Did you write an integration/unit/API test to verify the code changes?
Or did you test this change manually (provide relevant screenshots)?
-->

```bash
curl http://localhost:8080/health/ready -v | jq .
* Host localhost:8080 was resolved.
* IPv6: ::1
* IPv4: 127.0.0.1
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0*   Trying [::1]:8080...
* Connected to localhost (::1) port 8080
> GET /health/ready HTTP/1.1
> Host: localhost:8080
> User-Agent: curl/8.7.1
> Accept: */*
> 
* Request completely sent off
< HTTP/1.1 200 OK
< content-length: 90
< content-type: application/json
< x-request-id: 01963e0a-e41d-7251-b20a-e9a0a6d1a67a
< vary: Origin, Access-Control-Request-Method, Access-Control-Request-Headers
< access-control-expose-headers: content-type, strict-transport-security, x-request-id, via
< strict-transport-security: max-age=31536000
< via: HyperSwitch
< date: Wed, 16 Apr 2025 10:00:23 GMT
< 
{ [90 bytes data]
100    90  100    90    0     0    236      0 --:--:-- --:--:-- --:--:--   236
* Connection #0 to host localhost left intact
{
  "database": true,
  "redis": true,
  "analytics": true,
  "opensearch": false,
  "outgoing_request": true
}
```


## Checklist
<!-- Put an `x` in the boxes that apply -->

- [ ] I formatted the code `cargo +nightly fmt --all`
- [ ] I addressed lints thrown by `cargo clippy`
- [ ] I reviewed the submitted code
- [ ] I added unit tests for my changes where possible
