diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index a3f530d51b..b3ec74d8c1 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -31,8 +31,9 @@ jobs: - name: Install dependencies run: | apt-get update && apt-get install libsndfile1-dev libgl1 -y - python -m pip install -e .[quality,test] - python -m pip install pandas peft + python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" + python -m uv pip install -e [quality,test] + python -m uv pip install pandas peft - name: Environment run: | python utils/print_env.py diff --git a/.github/workflows/nightly_tests.yml b/.github/workflows/nightly_tests.yml index fb0ce92cb6..2019ea1542 100644 --- a/.github/workflows/nightly_tests.yml +++ b/.github/workflows/nightly_tests.yml @@ -60,9 +60,10 @@ jobs: - name: Install dependencies run: | - python -m pip install -e .[quality,test] - python -m pip install -U git+https://github.com/huggingface/transformers - python -m pip install git+https://github.com/huggingface/accelerate + python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" + python -m uv pip install -e [quality,test] + python -m uv pip install -U transformers@git+https://github.com/huggingface/transformers + python -m uv pip install accelerate@git+https://github.com/huggingface/accelerate - name: Environment run: | @@ -73,6 +74,7 @@ jobs: env: HUGGING_FACE_HUB_TOKEN: ${{ secrets.HUGGING_FACE_HUB_TOKEN }} run: | + python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile \ -s -v -k "not Flax and not Onnx" \ --make-reports=tests_${{ matrix.config.report }} \ @@ -83,6 +85,7 @@ jobs: env: HUGGING_FACE_HUB_TOKEN: ${{ secrets.HUGGING_FACE_HUB_TOKEN }} run: | + python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" python -m pytest -n 0 \ -s -v -k "Flax" \ --make-reports=tests_${{ matrix.config.report }} \ @@ -93,6 +96,7 @@ jobs: env: HUGGING_FACE_HUB_TOKEN: ${{ secrets.HUGGING_FACE_HUB_TOKEN }} run: | + python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile \ -s -v -k "Onnx" \ --make-reports=tests_${{ matrix.config.report }} \ @@ -132,10 +136,10 @@ jobs: - name: Install dependencies shell: arch -arch arm64 bash {0} run: | - ${CONDA_RUN} python -m pip install --upgrade pip - ${CONDA_RUN} python -m pip install -e .[quality,test] - ${CONDA_RUN} python -m pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cpu - ${CONDA_RUN} python -m pip install git+https://github.com/huggingface/accelerate + ${CONDA_RUN} python -m pip install --upgrade pip uv + ${CONDA_RUN} python -m uv pip install -e [quality,test] + ${CONDA_RUN} python -m uv pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cpu + ${CONDA_RUN} python -m uv pip install accelerate@git+https://github.com/huggingface/accelerate - name: Environment shell: arch -arch arm64 bash {0} diff --git a/.github/workflows/pr_dependency_test.yml b/.github/workflows/pr_dependency_test.yml index 0839497ed7..f21f09ef87 100644 --- a/.github/workflows/pr_dependency_test.yml +++ b/.github/workflows/pr_dependency_test.yml @@ -25,10 +25,12 @@ jobs: python-version: "3.8" - name: Install dependencies run: | - python -m pip install --upgrade pip - pip install -e . - pip install pytest + python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" + python -m pip install --upgrade pip uv + python -m uv pip install -e . + python -m uv pip install pytest - name: Check for soft dependencies run: | + python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" pytest tests/others/test_dependencies.py \ No newline at end of file diff --git a/.github/workflows/pr_flax_dependency_test.yml b/.github/workflows/pr_flax_dependency_test.yml index e905d94c25..bbad729299 100644 --- a/.github/workflows/pr_flax_dependency_test.yml +++ b/.github/workflows/pr_flax_dependency_test.yml @@ -25,12 +25,14 @@ jobs: python-version: "3.8" - name: Install dependencies run: | - python -m pip install --upgrade pip - pip install -e . - pip install "jax[cpu]>=0.2.16,!=0.3.2" - pip install "flax>=0.4.1" - pip install "jaxlib>=0.1.65" - pip install pytest + python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" + python -m pip install --upgrade pip uv + python -m uv pip install -e . + python -m uv pip install "jax[cpu]>=0.2.16,!=0.3.2" + python -m uv pip install "flax>=0.4.1" + python -m uv pip install "jaxlib>=0.1.65" + python -m uv pip install pytest - name: Check for soft dependencies run: | + python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" pytest tests/others/test_dependencies.py diff --git a/.github/workflows/pr_test_fetcher.yml b/.github/workflows/pr_test_fetcher.yml index b82f8007aa..6a7da2cb73 100644 --- a/.github/workflows/pr_test_fetcher.yml +++ b/.github/workflows/pr_test_fetcher.yml @@ -33,7 +33,8 @@ jobs: - name: Install dependencies run: | apt-get update && apt-get install libsndfile1-dev libgl1 -y - python -m pip install -e .[quality,test] + python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" + python -m uv pip install -e [quality,test] - name: Environment run: | python utils/print_env.py @@ -89,15 +90,18 @@ jobs: - name: Install dependencies run: | apt-get update && apt-get install libsndfile1-dev libgl1 -y - python -m pip install -e .[quality,test] + python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" + python -m pip install -e [quality,test] python -m pip install accelerate - name: Environment run: | + python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" python utils/print_env.py - name: Run all selected tests on CPU run: | + python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" python -m pytest -n 2 --dist=loadfile -v --make-reports=${{ matrix.modules }}_tests_cpu ${{ fromJson(needs.setup_pr_tests.outputs.test_map)[matrix.modules] }} - name: Failure short reports @@ -144,15 +148,18 @@ jobs: - name: Install dependencies run: | apt-get update && apt-get install libsndfile1-dev libgl1 -y - python -m pip install -e .[quality,test] + python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" + python -m pip install -e [quality,test] - name: Environment run: | + python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" python utils/print_env.py - name: Run Hub tests for models, schedulers, and pipelines on a staging env if: ${{ matrix.config.framework == 'hub_tests_pytorch' }} run: | + python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" HUGGINGFACE_CO_STAGING=true python -m pytest \ -m "is_staging_test" \ --make-reports=tests_${{ matrix.config.report }} \ diff --git a/.github/workflows/pr_test_peft_backend.yml b/.github/workflows/pr_test_peft_backend.yml index c93e8bef40..b9fb06d781 100644 --- a/.github/workflows/pr_test_peft_backend.yml +++ b/.github/workflows/pr_test_peft_backend.yml @@ -84,21 +84,24 @@ jobs: - name: Install dependencies run: | apt-get update && apt-get install libsndfile1-dev libgl1 -y - python -m pip install -e .[quality,test] + python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" + python -m uv pip install -e [quality,test] if [ "${{ matrix.lib-versions }}" == "main" ]; then - python -m pip install -U git+https://github.com/huggingface/peft.git - python -m pip install -U git+https://github.com/huggingface/transformers.git - python -m pip install -U git+https://github.com/huggingface/accelerate.git + python -m uv pip install -U peft@git+https://github.com/huggingface/peft.git + python -m uv pip install -U transformers@git+https://github.com/huggingface/transformers.git + python -m uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git else - python -m pip install -U peft transformers accelerate + python -m uv pip install -U peft transformers accelerate fi - name: Environment run: | + python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" python utils/print_env.py - name: Run fast PyTorch LoRA CPU tests with PEFT backend run: | + python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile \ -s -v \ --make-reports=tests_${{ matrix.config.report }} \ diff --git a/.github/workflows/pr_tests.yml b/.github/workflows/pr_tests.yml index d212322034..d196bb7ff4 100644 --- a/.github/workflows/pr_tests.yml +++ b/.github/workflows/pr_tests.yml @@ -111,16 +111,19 @@ jobs: - name: Install dependencies run: | apt-get update && apt-get install libsndfile1-dev libgl1 -y - python -m pip install -e .[quality,test] - python -m pip install accelerate + python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" + python -m uv pip install -e [quality,test] + python -m uv pip install accelerate - name: Environment run: | + python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" python utils/print_env.py - name: Run fast PyTorch Pipeline CPU tests if: ${{ matrix.config.framework == 'pytorch_pipelines' }} run: | + python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" python -m pytest -n 2 --max-worker-restart=0 --dist=loadfile \ -s -v -k "not Flax and not Onnx" \ --make-reports=tests_${{ matrix.config.report }} \ @@ -129,6 +132,7 @@ jobs: - name: Run fast PyTorch Model Scheduler CPU tests if: ${{ matrix.config.framework == 'pytorch_models' }} run: | + python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" python -m pytest -n 2 --max-worker-restart=0 --dist=loadfile \ -s -v -k "not Flax and not Onnx and not Dependency" \ --make-reports=tests_${{ matrix.config.report }} \ @@ -137,6 +141,7 @@ jobs: - name: Run fast Flax TPU tests if: ${{ matrix.config.framework == 'flax' }} run: | + python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" python -m pytest -n 2 --max-worker-restart=0 --dist=loadfile \ -s -v -k "Flax" \ --make-reports=tests_${{ matrix.config.report }} \ @@ -145,7 +150,8 @@ jobs: - name: Run example PyTorch CPU tests if: ${{ matrix.config.framework == 'pytorch_examples' }} run: | - python -m pip install peft + python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" + python -m uv pip install peft python -m pytest -n 2 --max-worker-restart=0 --dist=loadfile \ --make-reports=tests_${{ matrix.config.report }} \ examples @@ -194,15 +200,18 @@ jobs: - name: Install dependencies run: | apt-get update && apt-get install libsndfile1-dev libgl1 -y - python -m pip install -e .[quality,test] + python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" + python -m uv pip install -e [quality,test] - name: Environment run: | + python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" python utils/print_env.py - name: Run Hub tests for models, schedulers, and pipelines on a staging env if: ${{ matrix.config.framework == 'hub_tests_pytorch' }} run: | + python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" HUGGINGFACE_CO_STAGING=true python -m pytest \ -m "is_staging_test" \ --make-reports=tests_${{ matrix.config.report }} \ diff --git a/.github/workflows/pr_torch_dependency_test.yml b/.github/workflows/pr_torch_dependency_test.yml index c8aeb4b2a6..16a7724fe7 100644 --- a/.github/workflows/pr_torch_dependency_test.yml +++ b/.github/workflows/pr_torch_dependency_test.yml @@ -25,10 +25,12 @@ jobs: python-version: "3.8" - name: Install dependencies run: | - python -m pip install --upgrade pip - pip install -e . - pip install torch torchvision torchaudio - pip install pytest + python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" + python -m pip install --upgrade pip uv + python -m uv pip install -e . + python -m uv pip install torch torchvision torchaudio + python -m uv pip install pytest - name: Check for soft dependencies run: | + python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" pytest tests/others/test_dependencies.py diff --git a/.github/workflows/push_tests.yml b/.github/workflows/push_tests.yml index 113b89e6ab..4b6d5bfc37 100644 --- a/.github/workflows/push_tests.yml +++ b/.github/workflows/push_tests.yml @@ -35,8 +35,9 @@ jobs: - name: Install dependencies run: | apt-get update && apt-get install libsndfile1-dev libgl1 -y - python -m pip install -e .[quality,test] - python -m pip install git+https://github.com/huggingface/accelerate.git + python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" + python -m uv pip install -e [quality,test] + python -m uv pip install accelerate@git+https://github.com/huggingface/accelerate.git - name: Environment run: | @@ -79,8 +80,9 @@ jobs: - name: Install dependencies run: | apt-get update && apt-get install libsndfile1-dev libgl1 -y - python -m pip install -e .[quality,test] - python -m pip install git+https://github.com/huggingface/accelerate.git + python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" + python -m uv pip install -e [quality,test] + python -m uv pip install accelerate@git+https://github.com/huggingface/accelerate.git - name: Environment run: | python utils/print_env.py @@ -128,8 +130,9 @@ jobs: - name: Install dependencies run: | apt-get update && apt-get install libsndfile1-dev libgl1 -y - python -m pip install -e .[quality,test] - python -m pip install git+https://github.com/huggingface/accelerate.git + python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" + python -m uv pip install -e [quality,test] + python -m uv pip install accelerate@git+https://github.com/huggingface/accelerate.git - name: Environment run: | @@ -177,9 +180,10 @@ jobs: - name: Install dependencies run: | apt-get update && apt-get install libsndfile1-dev libgl1 -y - python -m pip install -e .[quality,test] - python -m pip install git+https://github.com/huggingface/accelerate.git - python -m pip install git+https://github.com/huggingface/peft.git + python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" + python -m uv pip install -e [quality,test] + python -m uv pip install accelerate@git+https://github.com/huggingface/accelerate.git + python -m uv pip install peft@git+https://github.com/huggingface/peft.git - name: Environment run: | @@ -227,8 +231,9 @@ jobs: - name: Install dependencies run: | apt-get update && apt-get install libsndfile1-dev libgl1 -y - python -m pip install -e .[quality,test] - python -m pip install git+https://github.com/huggingface/accelerate.git + python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" + python -m uv pip install -e [quality,test] + python -m uv pip install accelerate@git+https://github.com/huggingface/accelerate.git - name: Environment run: | @@ -274,8 +279,9 @@ jobs: - name: Install dependencies run: | apt-get update && apt-get install libsndfile1-dev libgl1 -y - python -m pip install -e .[quality,test] - python -m pip install git+https://github.com/huggingface/accelerate.git + python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" + python -m uv pip install -e [quality,test] + python -m uv pip install accelerate@git+https://github.com/huggingface/accelerate.git - name: Environment run: | @@ -323,7 +329,8 @@ jobs: nvidia-smi - name: Install dependencies run: | - python -m pip install -e .[quality,test,training] + python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" + python -m uv pip install -e [quality,test,training] - name: Environment run: | python utils/print_env.py @@ -363,7 +370,8 @@ jobs: nvidia-smi - name: Install dependencies run: | - python -m pip install -e .[quality,test,training] + python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" + python -m uv pip install -e [quality,test,training] - name: Environment run: | python utils/print_env.py @@ -404,16 +412,19 @@ jobs: - name: Install dependencies run: | - python -m pip install -e .[quality,test,training] + python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" + python -m uv pip install -e [quality,test,training] - name: Environment run: | + python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" python utils/print_env.py - name: Run example tests on GPU env: HUGGING_FACE_HUB_TOKEN: ${{ secrets.HUGGING_FACE_HUB_TOKEN }} run: | + python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile -s -v --make-reports=examples_torch_cuda examples/ - name: Failure short reports diff --git a/.github/workflows/push_tests_fast.yml b/.github/workflows/push_tests_fast.yml index 664c973229..6b01577041 100644 --- a/.github/workflows/push_tests_fast.yml +++ b/.github/workflows/push_tests_fast.yml @@ -69,15 +69,18 @@ jobs: - name: Install dependencies run: | apt-get update && apt-get install libsndfile1-dev libgl1 -y - python -m pip install -e .[quality,test] + python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" + python -m uv pip install -e [quality,test] - name: Environment run: | + python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" python utils/print_env.py - name: Run fast PyTorch CPU tests if: ${{ matrix.config.framework == 'pytorch' }} run: | + python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" python -m pytest -n 2 --max-worker-restart=0 --dist=loadfile \ -s -v -k "not Flax and not Onnx" \ --make-reports=tests_${{ matrix.config.report }} \ @@ -86,6 +89,7 @@ jobs: - name: Run fast Flax TPU tests if: ${{ matrix.config.framework == 'flax' }} run: | + python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" python -m pytest -n 2 --max-worker-restart=0 --dist=loadfile \ -s -v -k "Flax" \ --make-reports=tests_${{ matrix.config.report }} \ @@ -94,6 +98,7 @@ jobs: - name: Run fast ONNXRuntime CPU tests if: ${{ matrix.config.framework == 'onnxruntime' }} run: | + python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" python -m pytest -n 2 --max-worker-restart=0 --dist=loadfile \ -s -v -k "Onnx" \ --make-reports=tests_${{ matrix.config.report }} \ @@ -102,7 +107,8 @@ jobs: - name: Run example PyTorch CPU tests if: ${{ matrix.config.framework == 'pytorch_examples' }} run: | - python -m pip install peft + python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" + python -m uv pip install peft python -m pytest -n 2 --max-worker-restart=0 --dist=loadfile \ --make-reports=tests_${{ matrix.config.report }} \ examples diff --git a/.github/workflows/push_tests_mps.yml b/.github/workflows/push_tests_mps.yml index 76c9a0f966..3a14f85634 100644 --- a/.github/workflows/push_tests_mps.yml +++ b/.github/workflows/push_tests_mps.yml @@ -44,11 +44,11 @@ jobs: - name: Install dependencies shell: arch -arch arm64 bash {0} run: | - ${CONDA_RUN} python -m pip install --upgrade pip - ${CONDA_RUN} python -m pip install -e .[quality,test] - ${CONDA_RUN} python -m pip install torch torchvision torchaudio - ${CONDA_RUN} python -m pip install git+https://github.com/huggingface/accelerate.git - ${CONDA_RUN} python -m pip install transformers --upgrade + ${CONDA_RUN} python -m pip install --upgrade pip uv + ${CONDA_RUN} python -m uv pip install -e [quality,test] + ${CONDA_RUN} python -m uv pip install torch torchvision torchaudio + ${CONDA_RUN} python -m uv pip install accelerate@git+https://github.com/huggingface/accelerate.git + ${CONDA_RUN} python -m uv pip install transformers --upgrade - name: Environment shell: arch -arch arm64 bash {0}