first commit

README.md

@@ -0,0 +1,10 @@
+# Cloud Server Setup Documentation
+
+This repository contains scripts and documentation for setting up a machine learning cloud server environment with PyTorch, Flash Attention, and Hugging Face tools.
+
+## System Requirements
+
+- A Linux-based cloud server (Ubuntu 20.04+ recommended)
+- At least 16GB RAM (32GB+ recommended for larger models)
+- CUDA-compatible GPU (for PyTorch GPU acceleration)
+- Sufficient disk space (at least 100GB recommended)

script/fetch_data.sh

@@ -0,0 +1,10 @@
+HF_TOKEN="your_token"
+DATADIR="/workspace/seungheon/dataset"
+mkdir -p $DATADIR
+huggingface-cli login --token $HF_TOKEN
+git lfs install
+git clone https://huggingface.co/datasets/seungheondoh/cmd-fma-metadata $DATADIR/fma
+git clone https://huggingface.co/datasets/seungheondoh/cmd-musicnet-metadata $DATADIR/musicnet
+git clone https://huggingface.co/datasets/seungheondoh/cmd-medleydb-metadata $DATADIR/medleydb
+git clone https://huggingface.co/datasets/seungheondoh/cmd-moisesdb-metadata $DATADIR/moisesdb
+git clone https://huggingface.co/datasets/seungheondoh/cmd-mtg_jamendo-metadata $DATADIR/mtg_jamendo

script/virtualenv.sh

@@ -0,0 +1,112 @@
+#!/bin/bash
+# Cloud server setup script for ML environment
+# Installs: Python venv, uv, PyTorch 2.6.0, flash-attn, and Hugging Face CLI
+
+set -e  # Exit immediately if a command exits with a non-zero status
+
+# Update system packages
+echo "Updating system packages..."
+sudo apt update && sudo apt upgrade -y
+# Install system dependencies
+echo "Installing system dependencies..."
+sudo apt install vim ffmpeg libsndfile1 git htop git-lfs -y
+
+# Install Python 3.10
+echo "Installing Python 3.10..."
+sudo apt install python3.10 python3.10-venv -y
+
+# Check if Python 3.10 is installed
+if ! command -v python3.10 &> /dev/null; then
+    echo "Failed to install Python 3.10. Please check your system and try again."
+    exit 1
+fi
+
+# Get Python version
+PYTHON_VERSION=$(python3.10 --version | awk '{print $2}')
+echo "Using Python version: ${PYTHON_VERSION}"
+
+# Check for CUDA
+if command -v nvidia-smi &> /dev/null; then
+    echo "NVIDIA GPU detected. Will install CUDA-enabled PyTorch."
+    CUDA_AVAILABLE=1
+    CUDA_VERSION=$(nvidia-smi --query-gpu=driver_version --format=csv,noheader | head -n 1)
+    echo "CUDA Driver Version: ${CUDA_VERSION}"
+else
+    echo "No NVIDIA GPU detected. Will install CPU-only PyTorch."
+    CUDA_AVAILABLE=0
+fi
+
+# Create a virtual environment if it doesn't exist
+if [ ! -d "venv" ]; then
+    echo "Creating Python virtual environment..."
+    python3.10 -m venv venv
+else
+    echo "Virtual environment already exists."
+fi
+
+# Upgrade pip
+echo "Upgrading pip..."
+pip install --upgrade pip
+
+# Install uv
+echo "Installing uv package manager..."
+curl -LsSf https://astral.sh/uv/install.sh | sh
+
+# Start uv virtualenv
+echo "Starting uv virtualenv..."
+uv venv .venv
+source .venv/bin/activate
+
+# Ensure uv is in PATH
+export PATH="$HOME/.cargo/bin:$PATH"
+# Get CUDA version number if available
+if [ "$CUDA_AVAILABLE" -eq 1 ]; then
+    CUDA_VERSION_NUM=$(nvidia-smi --query-gpu=driver_version --format=csv,noheader | head -n 1 | cut -d'.' -f1)
+    # Map CUDA driver version to PyTorch CUDA version
+    if [ "$CUDA_VERSION_NUM" -ge 535 ]; then
+        PYTORCH_CUDA="cu124"  # CUDA 12.4
+    elif [ "$CUDA_VERSION_NUM" -ge 530 ]; then
+        PYTORCH_CUDA="cu121"  # CUDA 12.1
+    elif [ "$CUDA_VERSION_NUM" -ge 520 ]; then
+        PYTORCH_CUDA="cu118"  # CUDA 11.8
+    elif [ "$CUDA_VERSION_NUM" -ge 510 ]; then
+        PYTORCH_CUDA="cu117"  # CUDA 11.7
+    else
+        echo "Warning: CUDA version may be too old. Defaulting to CPU version."
+        CUDA_AVAILABLE=0
+    fi
+fi
+
+# Install PyTorch 2.6.0
+echo "Installing PyTorch 2.6.0..."
+echo "Installing PyTorch with CUDA support (${PYTORCH_CUDA})..."
+uv pip install torch==2.6.0 --index-url https://download.pytorch.org/whl/${PYTORCH_CUDA}
+
+# Install flash-attention
+echo "Installing flash-attention..."
+if [ "$CUDA_AVAILABLE" -eq 1 ]; then
+    # Install required dependencies for flash-attention
+    uv pip install packaging ninja
+    uv pip install flash-attn --no-build-isolation
+else
+    echo "Warning: flash-attention requires CUDA. Skipping installation."
+fi
+
+# Install Hugging Face CLI and tools
+echo "Installing Hugging Face CLI and tools..."
+uv pip install huggingface_hub
+# Verify installation
+echo "Verifying installation..."
+python -c "import torch; print('PyTorch version:', torch.__version__); print('CUDA available:', torch.cuda.is_available())"
+
+# Try importing flash-attention if CUDA is available
+if [ "$CUDA_AVAILABLE" -eq 1 ]; then
+    python -c "import importlib.util; print('flash-attn available:', importlib.util.find_spec('flash_attn') is not None)"
+fi
+# Check for Hugging Face CLI
+python -c "import huggingface_hub; print('Hugging Face Hub version:', huggingface_hub.__version__)"
+echo "============================================================"
+echo "Setup complete!"
+echo "To activate this environment, run:"
+echo "source venv/bin/activate"
+echo "============================================================"