Bug: Setup for Opensearch services and global search feature on control center



Add documentation to setup globalsearch & use it in local deployment mode.
diff --git a/config/dashboard.toml b/config/dashboard.toml
index 9bb2b6bf336..c00b167315a 100644
--- a/config/dashboard.toml
+++ b/config/dashboard.toml
@@ -31,7 +31,7 @@ surcharge=false
 dispute_evidence_upload=false
 paypal_automatic_flow=false
 threeds_authenticator=false
-global_search=false
+global_search=true
 dispute_analytics=true
 configure_pmts=false
 branding=false
diff --git a/config/prometheus.yaml b/config/prometheus.yaml
index 574c9dcce7a..85f6335c838 100644
--- a/config/prometheus.yaml
+++ b/config/prometheus.yaml
@@ -35,3 +35,11 @@ scrape_configs:
 
     static_configs:
       - targets: ["otel-collector:8888"]
+
+  - job_name: "vector"
+
+    # metrics_path defaults to '/metrics'
+    # scheme defaults to 'http'.
+
+    static_configs:
+      - targets: ["vector:9598"]
\ No newline at end of file
diff --git a/config/vector.yaml b/config/vector.yaml
new file mode 100644
index 00000000000..d8f5da34666
--- /dev/null
+++ b/config/vector.yaml
@@ -0,0 +1,134 @@
+acknowledgements:
+  enabled: true
+
+api:
+  enabled: true
+  address: 0.0.0.0:8686
+
+sources:
+  kafka_tx_events:
+    type: kafka
+    bootstrap_servers: kafka0:29092
+    group_id: sessionizer
+    topics:
+      - hyperswitch-payment-attempt-events
+      - hyperswitch-payment-intent-events
+      - hyperswitch-refund-events
+      - hyperswitch-dispute-events
+    decoding:
+      codec: json
+
+  app_logs:
+    type: docker_logs
+    include_labels:
+      - "logs=promtail"
+
+  vector_metrics:
+    type: internal_metrics
+
+  node_metrics:
+    type: host_metrics
+
+transforms:
+  plus_1_events:
+    type: filter
+    inputs:
+      - kafka_tx_events
+    condition: ".sign_flag == 1"
+
+  hs_server_logs:
+    type: filter
+    inputs:
+      - app_logs
+    condition: '.labels."com.docker.compose.service" == "hyperswitch-server"'
+
+  parsed_hs_server_logs:
+    type: remap
+    inputs:
+      - app_logs
+    source: |-
+      .message = parse_json!(.message)
+
+  events:
+    type: remap
+    inputs:
+      - plus_1_events
+    source: |-
+      .timestamp = from_unix_timestamp!(.created_at, unit: "seconds")
+
+sinks:
+  opensearch_events:
+    type: elasticsearch
+    inputs:
+      - events
+    endpoints:
+      - "https://opensearch:9200"
+    id_key: message_key
+    api_version: v7
+    tls:
+      verify_certificate: false
+      verify_hostname: false
+    auth:
+      strategy: basic
+      user: admin
+      password: 0penS3arc#
+    encoding:
+      except_fields:
+        - message_key
+        - offset
+        - partition
+        - topic
+    bulk:
+      # Add a date prefixed index for better grouping
+      # index: "vector-{{ .topic }}-%Y-%m-%d"
+      index: "{{ .topic }}"
+
+  opensearch_logs:
+    type: elasticsearch
+    inputs:
+      - parsed_hs_server_logs
+    endpoints:
+      - "https://opensearch:9200"
+    api_version: v7
+    tls:
+      verify_certificate: false
+      verify_hostname: false
+    auth:
+      strategy: basic
+      user: admin
+      password: 0penS3arc#
+    bulk:
+      # Add a date prefixed index for better grouping
+      # index: "vector-{{ .topic }}-%Y-%m-%d"
+      index: "logs-{{ .container_name }}-%Y-%m-%d"
+
+  log_events:
+    type: loki
+    inputs:
+      - kafka_tx_events
+    endpoint: http://loki:3100
+    labels:
+      source: vector
+      topic: "{{ .topic }}"
+      job: kafka
+    encoding:
+      codec: json
+
+  log_app_loki:
+    type: loki
+    inputs:
+      - parsed_hs_server_logs
+    endpoint: http://loki:3100
+    labels:
+      source: vector
+      job: app_logs
+      container: "{{ .container_name }}"
+      stream: "{{ .stream }}"
+    encoding:
+      codec: json
+
+  metrics:
+    type: prometheus_exporter
+    inputs:
+      - vector_metrics
+      - node_metrics
diff --git a/crates/analytics/src/clickhouse.rs b/crates/analytics/src/clickhouse.rs
index 64064c09c88..ef5efddbe99 100644
--- a/crates/analytics/src/clickhouse.rs
+++ b/crates/analytics/src/clickhouse.rs
@@ -263,6 +263,7 @@ impl TryInto<RefundFilterRow> for serde_json::Value {
         ))
     }
 }
+
 impl TryInto<DisputeMetricRow> for serde_json::Value {
     type Error = Report<ParsingError>;
 
diff --git a/docker-compose.yml b/docker-compose.yml
index c993262e744..6211148f39e 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -221,18 +221,6 @@ services:
       - ./config/grafana.ini:/etc/grafana/grafana.ini
       - ./config/grafana-datasource.yaml:/etc/grafana/provisioning/datasources/datasource.yml
 
-  promtail:
-    image: grafana/promtail:latest
-    volumes:
-      - ./logs:/var/log/router
-      - ./config:/etc/promtail
-      - /var/run/docker.sock:/var/run/docker.sock
-    command: -config.file=/etc/promtail/promtail.yaml
-    profiles:
-      - monitoring
-    networks:
-      - router_net
-
   loki:
     image: grafana/loki:latest
     ports:
@@ -361,21 +349,13 @@ services:
         soft: 262144
         hard: 262144
 
-  fluentd:
-    build: ./docker/fluentd
-    volumes:
-      - ./docker/fluentd/conf:/fluentd/etc
-    networks:
-      - router_net
-    profiles:
-      - olap
-
   opensearch:
-    image: public.ecr.aws/opensearchproject/opensearch:1.3.14
+    image: opensearchproject/opensearch:2
     container_name: opensearch
     hostname: opensearch
     environment:
       - "discovery.type=single-node"
+      - OPENSEARCH_INITIAL_ADMIN_PASSWORD=0penS3arc#
     profiles:
       - olap
     ports:
@@ -384,7 +364,7 @@ services:
       - router_net
 
   opensearch-dashboards:
-    image: opensearchproject/opensearch-dashboards:1.3.14
+    image: opensearchproject/opensearch-dashboards:2
     ports:
       - 5601:5601
     profiles:
@@ -393,3 +373,18 @@ services:
       OPENSEARCH_HOSTS: '["https://opensearch:9200"]'
     networks:
       - router_net
+
+  vector:
+    image: timberio/vector:latest-debian
+    ports:
+      - "8686"
+      - "9598"
+    profiles:
+      - olap
+    environment:
+      KAFKA_HOST: 'kafka0:29092'
+    networks:
+      - router_net
+    volumes:
+      - ./config/vector.yaml:/etc/vector/vector.yaml
+      - /var/run/docker.sock:/var/run/docker.sock
\ No newline at end of file

## Type of Change
<!-- Put an `x` in the boxes that apply -->

- [ ] Bugfix
- [x] New feature
- [ ] Enhancement
- [ ] Refactoring
- [ ] Dependency updates
- [ ] Documentation
- [ ] CI/CD

## Description
<!-- Describe your changes in detail -->
- Adding a vector service to handle logs & kafka events

### Additional Changes

- [ ] This PR modifies the API contract
- [ ] This PR modifies the database schema
- [ ] This PR modifies application configuration/environment variables

<!--
Provide links to the files with corresponding changes.

Following are the paths where you can find config files:
1. `config`
2. `crates/router/src/configs`
3. `loadtest/config`
-->


## Motivation and Context
<!--
Why is this change required? What problem does it solve?
If it fixes an open issue, please link to the issue here.

If you don't have an issue, we'd recommend starting with one first so the PR
can focus on the implementation (unless it is an obvious bug or documentation fix
that will have little conversation).
-->
- Add a common service to handle all data transformation and transport

## How did you test it?
<!--
Did you write an integration/unit/API test to verify the code changes?
Or did you test this change manually (provide relevant screenshots)?
-->
- Running it locally to check logs added in loki & opensearch

## Checklist
<!-- Put an `x` in the boxes that apply -->

- [ ] I formatted the code `cargo +nightly fmt --all`
- [ ] I addressed lints thrown by `cargo clippy`
- [ ] I reviewed the submitted code
- [ ] I added unit tests for my changes where possible
