[ci] move rllib small-gpu tests to civ2
diff --git a/.buildkite/pipeline.gpu.yml b/.buildkite/pipeline.gpu.yml
index 48235e1ffe0ce..a2e41d1cfe7c3 100644
--- a/.buildkite/pipeline.gpu.yml
+++ b/.buildkite/pipeline.gpu.yml
@@ -10,25 +10,6 @@
 #    - ./ci/env/env_info.sh
 #    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=gpu python/ray/tune/...
 
-- label: ":tv: :brain: RLlib: GPU Tests"
-  conditions: ["NO_WHEELS_REQUIRED", "RAY_CI_RLLIB_AFFECTED"]
-  parallelism: 2
-  commands:
-    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
-    - RLLIB_TESTING=1 ./ci/env/install-dependencies.sh
-    - pip install -Ur ./python/requirements/ml/dl-gpu-requirements.txt
-    - ./ci/env/env_info.sh
-    # --jobs 1 is necessary as we only have 1 GPU on the machine and running tests in parallel
-    # would cause timeouts as the other scripts would wait for the GPU to become available.
-    - ./ci/run/run_bazel_test_with_sharding.sh --config=ci $(./ci/run/bazel_export_options)
-      --build_tests_only
-      --jobs=1
-      --test_tag_filters=gpu
-      --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1 
-      --test_env=RLLIB_NUM_GPUS=1
-      rllib/...
-
-
 # Serve doesn't have gpu tests now (Oct 6 2023)
 #- label: ":tv: :serverless: Serve Tests"
 #  conditions:
diff --git a/.buildkite/rllib.rayci.yml b/.buildkite/rllib.rayci.yml
index 0540d9d1ea238..13424ed23b0cf 100644
--- a/.buildkite/rllib.rayci.yml
+++ b/.buildkite/rllib.rayci.yml
@@ -77,6 +77,22 @@ steps:
     depends_on: rllibbuild
     job_env: forge
 
+  - label: ":brain: rllib: gpu tests"
+    tags: 
+      - rllib
+      - gpu
+    parallelism: 2
+    instance_type: gpu
+    commands:
+      - bazel run //ci/ray_ci:test_in_docker -- //rllib/... rllib 
+        --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}" --worker-id "$${BUILDKITE_PARALLEL_JOB}"
+        --build-name rllibgpubuild
+        --only-tags gpu
+        --test-env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1 
+        --test-env=RLLIB_NUM_GPUS=1
+    depends_on: rllibgpubuild
+    job_env: forge
+
   - label: ":brain: rllib: rlmodule tests"
     tags: rllib_directly
     instance_type: large
diff --git a/ci/ray_ci/test_tester.py b/ci/ray_ci/test_tester.py
index e39645971c85d..0ff9735e50a34 100644
--- a/ci/ray_ci/test_tester.py
+++ b/ci/ray_ci/test_tester.py
@@ -1,4 +1,5 @@
 import os
+import re
 import sys
 from tempfile import TemporaryDirectory
 from unittest import mock
@@ -11,9 +12,23 @@
     _get_all_test_query,
     _get_test_targets,
     _get_flaky_test_targets,
+    _get_tag_matcher,
 )
 
 
+def test_get_tag_matcher() -> None:
+    assert re.match(
+        # simulate shell character escaping
+        bytes(_get_tag_matcher("tag"), "utf-8").decode("unicode_escape"),
+        "tag",
+    )
+    assert not re.match(
+        # simulate shell character escaping
+        bytes(_get_tag_matcher("tag"), "utf-8").decode("unicode_escape"),
+        "atagb",
+    )
+
+
 def test_get_container() -> None:
     with mock.patch(
         "ci.ray_ci.tester_container.TesterContainer.install_ray",
@@ -71,18 +86,20 @@ def test_get_test_targets() -> None:
 
 def test_get_all_test_query() -> None:
     assert _get_all_test_query(["a", "b"], "core") == (
-        "attr(tags, 'team:core\\\\b', tests(a) union tests(b))"
+        "attr(tags, '\\\\bteam:core\\\\b', tests(a) union tests(b))"
     )
     assert _get_all_test_query(["a"], "core", except_tags="tag") == (
-        "attr(tags, 'team:core\\\\b', tests(a)) except (attr(tags, tag, tests(a)))"
+        "attr(tags, '\\\\bteam:core\\\\b', tests(a)) "
+        "except (attr(tags, '\\\\btag\\\\b', tests(a)))"
     )
     assert _get_all_test_query(["a"], "core", only_tags="tag") == (
-        "attr(tags, 'team:core\\\\b', tests(a)) intersect (attr(tags, tag, tests(a)))"
+        "attr(tags, '\\\\bteam:core\\\\b', tests(a)) "
+        "intersect (attr(tags, '\\\\btag\\\\b', tests(a)))"
     )
     assert _get_all_test_query(["a"], "core", except_tags="tag1", only_tags="tag2") == (
-        "attr(tags, 'team:core\\\\b', tests(a)) "
-        "intersect (attr(tags, tag2, tests(a))) "
-        "except (attr(tags, tag1, tests(a)))"
+        "attr(tags, '\\\\bteam:core\\\\b', tests(a)) "
+        "intersect (attr(tags, '\\\\btag2\\\\b', tests(a))) "
+        "except (attr(tags, '\\\\btag1\\\\b', tests(a)))"
     )
 
 
diff --git a/ci/ray_ci/tester.py b/ci/ray_ci/tester.py
index f2204976e1969..9ff4f4ce04f12 100644
--- a/ci/ray_ci/tester.py
+++ b/ci/ray_ci/tester.py
@@ -160,6 +160,17 @@ def _get_container(
     )
 
 
+def _get_tag_matcher(tag: str) -> str:
+    """
+    Return a regular expression that matches the given bazel tag. This is required for
+    an exact tag match because bazel query uses regex to match tags.
+
+    The word boundary is escaped twice because it is used in a python string and then
+    used again as a string in bazel query.
+    """
+    return f"\\\\b{tag}\\\\b"
+
+
 def _get_all_test_query(
     targets: List[str],
     team: str,
@@ -171,17 +182,23 @@ def _get_all_test_query(
     have the given tags
     """
     test_query = " union ".join([f"tests({target})" for target in targets])
-    query = f"attr(tags, 'team:{team}\\\\b', {test_query})"
+    query = f"attr(tags, '{_get_tag_matcher(f'team:{team}')}', {test_query})"
 
     if only_tags:
         only_query = " union ".join(
-            [f"attr(tags, {t}, {test_query})" for t in only_tags.split(",")]
+            [
+                f"attr(tags, '{_get_tag_matcher(t)}', {test_query})"
+                for t in only_tags.split(",")
+            ]
         )
         query = f"{query} intersect ({only_query})"
 
     if except_tags:
         except_query = " union ".join(
-            [f"attr(tags, {t}, {test_query})" for t in except_tags.split(",")]
+            [
+                f"attr(tags, '{_get_tag_matcher(t)}', {test_query})"
+                for t in except_tags.split(",")
+            ]
         )
         query = f"{query} except ({except_query})"
 

- rllib uses 'gpu' and 'multi-gpu' tag which confuses bazel query tag since it takes the input as a regex. Updated ray_ci to strictly query for an exact match of tag
- we cannot parallelize this job because the machine has only one core

Test:
- CI