[wip][dependencies] pip-compile dependencies as a build step
diff --git a/.buildkite/build.rayci.yml b/.buildkite/build.rayci.yml
index 5ac7b938b5e6d..dba2a01e8b3b7 100644
--- a/.buildkite/build.rayci.yml
+++ b/.buildkite/build.rayci.yml
@@ -40,8 +40,18 @@ steps:
     commands:
       - bazel run //ci/ray_ci:build_in_docker -- docker --python-version py38 
         --platform cu118 --image-type ray-ml
-    depends_on: 
+    depends_on:
       - manylinux
       - forge
       - ray-mlpy38cu118base
     job_env: forge
+
+  - label: ":tapioca: build: pip-compile dependencies"
+    instance_type: small
+    commands:
+      - rm ./python/requirements_compiled.txt
+      - ./ci/ci.sh compile_pip_dependencies
+      - cp -f ./python/requirements_compiled.txt /artifact-mount/
+    depends_on:
+      - forge
+    job_env: forge
diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index 02194c6ba76be..d8f5462c8d700 100644
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -25,5 +25,9 @@ if [ -d "/tmp/artifacts" ]; then
     find /tmp/artifacts -print || true
 fi
 
+echo "Creating var/ artifact directory..."
+docker run --rm -v /tmp/artifacts:/artifact-mount alpine:latest /bin/sh -c 'chown -R 2000 /artifact-mount/' || true
+
+
 RAYCI_CHECKOUT_DIR="$(pwd)"
 export RAYCI_CHECKOUT_DIR
diff --git a/ci/ci.sh b/ci/ci.sh
index 721aa5432870e..cacf5d8939b42 100755
--- a/ci/ci.sh
+++ b/ci/ci.sh
@@ -113,7 +113,7 @@ compile_pip_dependencies() {
   pip install --no-cache-dir numpy torch
 
   pip-compile --resolver=backtracking -q \
-     --pip-args --no-deps --strip-extras --no-annotate --no-header -o \
+     --pip-args --no-deps --strip-extras --no-header -o \
     "${WORKSPACE_DIR}/python/$TARGET" \
     "${WORKSPACE_DIR}/python/requirements.txt" \
     "${WORKSPACE_DIR}/python/requirements/lint-requirements.txt" \

<!-- Thank you for your contribution! Please review https://github.com/ray-project/ray/blob/master/CONTRIBUTING.rst before opening a pull request. -->

<!-- Please add a reviewer to the assignee section when you create a PR. If you don't have the access to it, we will shortly find a reviewer and assign them to your PR. -->

## Why are these changes needed?

It's currently complicated to update dependencies, as it requires a (manual) recompilation of the `requirements_compiled.txt` file.

This PR adds a separate build job that runs pip compile on all input files to generate an updated `requirements_compiled.txt`, that is than exposed as an artifact in the buildkite job.

To update a dependency in the future, the process would be:

1. Open a PR that updates the respective requirements in the input requirements file
2. Wait for the CI job to run through and output the updated compiled requirements file
3. If the compilation was successful, add the contents of the file to the PR

## Related issue number

<!-- For example: "Closes #1234" -->

## Checks

- [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR.
- [ ] I've run `scripts/format.sh` to lint the changes in this PR.
- [ ] I've included any doc changes needed for https://docs.ray.io/en/master/.
    - [ ] I've added any new APIs to the API Reference. For example, if I added a 
           method in Tune, I've added it in `doc/source/tune/api/` under the 
           corresponding `.rst` file.
- [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/
- Testing Strategy
   - [ ] Unit tests
   - [ ] Release tests
   - [ ] This PR is not tested :(
