From 26ba0bab76d238b559090a1cd0a9159c35a8f886 Mon Sep 17 00:00:00 2001 From: okazaki10 Date: Sat, 24 May 2025 22:53:11 +0700 Subject: [PATCH 1/6] add blackwell support --- one_click.py | 40 +++++++++++++++---- requirements/full/requirements_blackwell.txt | 42 ++++++++++++++++++++ 2 files changed, 74 insertions(+), 8 deletions(-) create mode 100644 requirements/full/requirements_blackwell.txt diff --git a/one_click.py b/one_click.py index 482a6aa9..1eb90d8b 100644 --- a/one_click.py +++ b/one_click.py @@ -17,6 +17,7 @@ import sys # Define the required versions TORCH_VERSION = "2.6.0" +TORCH_VERSION_BLACKWELL = "2.7.0" TORCHVISION_VERSION = "0.21.0" TORCHAUDIO_VERSION = "2.6.0" PYTHON_VERSION = "3.11" @@ -207,7 +208,7 @@ def get_user_choice(question, options_dict): return choice -def update_pytorch_and_python(): +def update_pytorch_and_python(selected_gpu=""): print_big_message("Checking for PyTorch updates.") # Update the Python version. Left here for future reference in case this becomes necessary. @@ -219,8 +220,13 @@ def update_pytorch_and_python(): torver = torch_version() base_cmd = f"python -m pip install --upgrade torch=={TORCH_VERSION} torchvision=={TORCHVISION_VERSION} torchaudio=={TORCHAUDIO_VERSION}" - if "+cu" in torver: - install_cmd = f"{base_cmd} --index-url https://download.pytorch.org/whl/cu124" + if selected_gpu == "NVIDIA_BLACKWELL": + base_cmd = f"python -m pip install --upgrade torch=={TORCH_VERSION_BLACKWELL} torchvision torchaudio" + + if "+cu" in torver and selected_gpu == "NVIDIA_BLACKWELL": + install_cmd = f"{base_cmd} --index-url https://download.pytorch.org/whl/cu128" + elif "+cu" in torver: + install_cmd = f"{base_cmd} --index-url https://download.pytorch.org/whl/cu124" elif "+rocm" in torver: install_cmd = f"{base_cmd} --index-url https://download.pytorch.org/whl/rocm6.2.4" elif "+cpu" in torver: @@ -276,6 +282,7 @@ def install_webui(): 'B': 'AMD - Linux/macOS only, requires ROCm 6.2.4', 'C': 'Apple M Series', 'D': 'Intel Arc (beta)', + 'E': 'NVIDIA - CUDA 12.8 - RTX 50XX BLACKWELL', 'N': 'CPU mode' }, ) @@ -286,6 +293,7 @@ def install_webui(): "B": "AMD", "C": "APPLE", "D": "INTEL", + "E": "NVIDIA_BLACKWELL", "N": "NONE" } @@ -302,6 +310,9 @@ def install_webui(): # Handle CUDA version display elif any((is_windows(), is_linux())) and selected_gpu == "NVIDIA": print("CUDA: 12.4") + + elif any((is_windows(), is_linux())) and selected_gpu == "NVIDIA_BLACKWELL": + print("CUDA: 12.8") # No PyTorch for AMD on Windows (?) elif is_windows() and selected_gpu == "AMD": @@ -311,8 +322,13 @@ def install_webui(): # Find the Pytorch installation command install_pytorch = f"python -m pip install torch=={TORCH_VERSION} torchvision=={TORCHVISION_VERSION} torchaudio=={TORCHAUDIO_VERSION} " + if selected_gpu == "NVIDIA_BLACKWELL": + install_pytorch = f"python -m pip install torch=={TORCH_VERSION_BLACKWELL} torchvision torchaudio " + if selected_gpu == "NVIDIA": install_pytorch += "--index-url https://download.pytorch.org/whl/cu124" + elif selected_gpu == "NVIDIA_BLACKWELL": + install_pytorch += "--index-url https://download.pytorch.org/whl/cu128" elif selected_gpu == "AMD": install_pytorch += "--index-url https://download.pytorch.org/whl/rocm6.2.4" elif selected_gpu in ["APPLE", "NONE"]: @@ -335,10 +351,10 @@ def install_webui(): run_cmd("conda install -y libuv", environment=True) # Install the webui requirements - update_requirements(initial_installation=True, pull=False) + update_requirements(initial_installation=True, pull=False, selected_gpu=selected_gpu) -def update_requirements(initial_installation=False, pull=True): +def update_requirements(initial_installation=False, pull=True, selected_gpu=""): # Create .git directory if missing if not os.path.exists(os.path.join(script_dir, ".git")): run_cmd( @@ -358,6 +374,8 @@ def update_requirements(initial_installation=False, pull=True): file_name = f"requirements_cpu_only{'_noavx2' if not cpu_has_avx2() else ''}.txt" elif is_macos(): file_name = f"requirements_apple_{'intel' if is_x86_64() else 'silicon'}.txt" + elif selected_gpu == "NVIDIA_BLACKWELL": + file_name = f"requirements_blackwell.txt" else: file_name = f"requirements{'_noavx2' if not cpu_has_avx2() else ''}.txt" @@ -431,7 +449,7 @@ def update_requirements(initial_installation=False, pull=True): # Update PyTorch if not initial_installation: - update_pytorch_and_python() + update_pytorch_and_python(selected_gpu=selected_gpu) torver = torch_version() clean_outdated_pytorch_cuda_dependencies() @@ -455,8 +473,11 @@ def update_requirements(initial_installation=False, pull=True): run_cmd(f"python -m pip uninstall -y {package_name}", environment=True) print(f"Uninstalled {package_name}") - # Install/update the project requirements - run_cmd("python -m pip install -r temp_requirements.txt --upgrade", assert_success=True, environment=True) + if selected_gpu == "NVIDIA_BLACKWELL": + # Install/update the project requirements + run_cmd(f"python -m pip install -r {requirements_file} --upgrade", assert_success=True, environment=True) + else: + run_cmd("python -m pip install -r temp_requirements.txt --upgrade", assert_success=True, environment=True) # Clean up os.remove('temp_requirements.txt') @@ -492,6 +513,7 @@ if __name__ == "__main__": 'A': 'Update the web UI', 'B': 'Install/update extensions requirements', 'C': 'Revert local changes to repository files with \"git reset --hard\"', + 'D': 'Upgrade to NVIDIA BLACKWELL', 'N': 'Nothing (exit)' }, ) @@ -515,6 +537,8 @@ if __name__ == "__main__": update_requirements(pull=False) elif choice == 'C': run_cmd("git reset --hard", assert_success=True, environment=True) + elif choice == 'D': + update_requirements(selected_gpu='NVIDIA_BLACKWELL') elif choice == 'N': sys.exit() else: diff --git a/requirements/full/requirements_blackwell.txt b/requirements/full/requirements_blackwell.txt new file mode 100644 index 00000000..8efaf44c --- /dev/null +++ b/requirements/full/requirements_blackwell.txt @@ -0,0 +1,42 @@ +accelerate==1.5.* +bitsandbytes==0.45.* +colorama +datasets +einops +fastapi==0.112.4 +gradio==4.37.* +jinja2==3.1.6 +markdown +numpy +pandas +peft==0.15.* +Pillow>=9.5.0 +psutil +pydantic==2.8.2 +pyyaml +requests +rich +safetensors==0.5.* +scipy +sentencepiece +tensorboard +transformers +tqdm +wandb +triton-windows + +# API +flask_cloudflared==0.0.14 +sse-starlette==1.6.5 +tiktoken + +# CUDA wheels +https://github.com/abetlen/llama-cpp-python/releases/download/v0.3.4-cu124/llama_cpp_python-0.3.4-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" +https://github.com/abetlen/llama-cpp-python/releases/download/v0.3.4-cu124/llama_cpp_python-0.3.4-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" +https://github.com/turboderp-org/exllamav3/releases/download/v0.0.2/exllamav3-0.0.2+cu128.torch2.7.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" +https://github.com/turboderp-org/exllamav3/releases/download/v0.0.2/exllamav3-0.0.2+cu128.torch2.7.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" +https://github.com/turboderp-org/exllamav2/releases/download/v0.3.0/exllamav2-0.3.0+cu128.torch2.7.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" +https://github.com/turboderp-org/exllamav2/releases/download/v0.3.0/exllamav2-0.3.0+cu128.torch2.7.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" +https://github.com/turboderp-org/exllamav2/releases/download/v0.3.0/exllamav2-0.3.0-py3-none-any.whl; platform_system == "Linux" and platform_machine != "x86_64" +https://github.com/kingbri1/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu128torch2.7.0cxx11abiFALSE-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" +https://github.com/kingbri1/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu128torch2.7.0cxx11abiFALSE-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" From a819b8c9b71fac7694de808e10e50937e0f119ad Mon Sep 17 00:00:00 2001 From: okazaki10 Date: Tue, 27 May 2025 18:08:22 +0700 Subject: [PATCH 2/6] add linux support --- requirements/full/requirements_blackwell.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements/full/requirements_blackwell.txt b/requirements/full/requirements_blackwell.txt index 8efaf44c..91c8d4bb 100644 --- a/requirements/full/requirements_blackwell.txt +++ b/requirements/full/requirements_blackwell.txt @@ -23,7 +23,8 @@ tensorboard transformers tqdm wandb -triton-windows +triton-windows; platform_system == "Windows" +triton; platform_system == "Linux" # API flask_cloudflared==0.0.14 From 4610de324cae174d9622b5cbfe09e4984bd62fcf Mon Sep 17 00:00:00 2001 From: okazaki10 Date: Wed, 28 May 2025 21:53:41 +0700 Subject: [PATCH 3/6] add web search --- requirements/full/requirements_blackwell.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/requirements/full/requirements_blackwell.txt b/requirements/full/requirements_blackwell.txt index 91c8d4bb..621ba083 100644 --- a/requirements/full/requirements_blackwell.txt +++ b/requirements/full/requirements_blackwell.txt @@ -25,6 +25,8 @@ tqdm wandb triton-windows; platform_system == "Windows" triton; platform_system == "Linux" +beautifulsoup4==4.13.4 +duckduckgo_search==8.0.2 # API flask_cloudflared==0.0.14 From d5b8ce3b01735538c83bb22ecab138cf17066cf9 Mon Sep 17 00:00:00 2001 From: okazaki10 Date: Wed, 28 May 2025 22:07:50 +0700 Subject: [PATCH 4/6] add pypdf --- requirements/full/requirements_blackwell.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/requirements/full/requirements_blackwell.txt b/requirements/full/requirements_blackwell.txt index 621ba083..a434a678 100644 --- a/requirements/full/requirements_blackwell.txt +++ b/requirements/full/requirements_blackwell.txt @@ -1,7 +1,9 @@ accelerate==1.5.* +beautifulsoup4==4.13.4 bitsandbytes==0.45.* colorama datasets +duckduckgo_search==8.0.2 einops fastapi==0.112.4 gradio==4.37.* @@ -13,6 +15,7 @@ peft==0.15.* Pillow>=9.5.0 psutil pydantic==2.8.2 +PyPDF2==3.0.1 pyyaml requests rich @@ -25,8 +28,6 @@ tqdm wandb triton-windows; platform_system == "Windows" triton; platform_system == "Linux" -beautifulsoup4==4.13.4 -duckduckgo_search==8.0.2 # API flask_cloudflared==0.0.14 From c151308ef5cfbaed7e8b26f54bac5312dbf8f756 Mon Sep 17 00:00:00 2001 From: okazaki10 Date: Thu, 29 May 2025 20:19:45 +0700 Subject: [PATCH 5/6] add cu128 llama cpp for windows --- requirements/full/requirements_blackwell.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/full/requirements_blackwell.txt b/requirements/full/requirements_blackwell.txt index a434a678..d16c0045 100644 --- a/requirements/full/requirements_blackwell.txt +++ b/requirements/full/requirements_blackwell.txt @@ -35,7 +35,7 @@ sse-starlette==1.6.5 tiktoken # CUDA wheels -https://github.com/abetlen/llama-cpp-python/releases/download/v0.3.4-cu124/llama_cpp_python-0.3.4-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" +https://github.com/boneylizard/llama-cpp-python-cu128-gemma3/releases/download/0.3.8%2Bcu128.gemma3/llama_cpp_python-0.3.8+cu128.gemma3-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" https://github.com/abetlen/llama-cpp-python/releases/download/v0.3.4-cu124/llama_cpp_python-0.3.4-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" https://github.com/turboderp-org/exllamav3/releases/download/v0.0.2/exllamav3-0.0.2+cu128.torch2.7.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" https://github.com/turboderp-org/exllamav3/releases/download/v0.0.2/exllamav3-0.0.2+cu128.torch2.7.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" From 34faa8975e55361c2b4d965985ef698955677082 Mon Sep 17 00:00:00 2001 From: okazaki10 Date: Thu, 29 May 2025 20:38:22 +0700 Subject: [PATCH 6/6] add llama cpp binaries --- requirements/full/requirements_blackwell.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements/full/requirements_blackwell.txt b/requirements/full/requirements_blackwell.txt index d16c0045..4a83e158 100644 --- a/requirements/full/requirements_blackwell.txt +++ b/requirements/full/requirements_blackwell.txt @@ -35,8 +35,8 @@ sse-starlette==1.6.5 tiktoken # CUDA wheels -https://github.com/boneylizard/llama-cpp-python-cu128-gemma3/releases/download/0.3.8%2Bcu128.gemma3/llama_cpp_python-0.3.8+cu128.gemma3-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" -https://github.com/abetlen/llama-cpp-python/releases/download/v0.3.4-cu124/llama_cpp_python-0.3.4-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.15.0/llama_cpp_binaries-0.15.0+cu124-py3-none-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.15.0/llama_cpp_binaries-0.15.0+cu124-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" https://github.com/turboderp-org/exllamav3/releases/download/v0.0.2/exllamav3-0.0.2+cu128.torch2.7.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" https://github.com/turboderp-org/exllamav3/releases/download/v0.0.2/exllamav3-0.0.2+cu128.torch2.7.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" https://github.com/turboderp-org/exllamav2/releases/download/v0.3.0/exllamav2-0.3.0+cu128.torch2.7.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"