" + data.html + "
";
+ }
+
+ const queryScope = target_element;
+
// Track open blocks
const openBlocks = new Set();
- document.querySelectorAll(".thinking-block").forEach(block => {
+ queryScope.querySelectorAll(".thinking-block").forEach(block => {
const blockId = block.getAttribute("data-block-id");
// If block exists and is open, add to open set
if (blockId && block.hasAttribute("open")) {
@@ -242,7 +255,7 @@ function handleMorphdomUpdate(text) {
// Store scroll positions for any open blocks
const scrollPositions = {};
- document.querySelectorAll(".thinking-block[open]").forEach(block => {
+ queryScope.querySelectorAll(".thinking-block[open]").forEach(block => {
const content = block.querySelector(".thinking-content");
const blockId = block.getAttribute("data-block-id");
if (content && blockId) {
@@ -255,8 +268,8 @@ function handleMorphdomUpdate(text) {
});
morphdom(
- document.getElementById("chat").parentNode,
- "" + text + "
",
+ target_element,
+ target_html,
{
onBeforeElUpdated: function(fromEl, toEl) {
// Preserve code highlighting
@@ -307,7 +320,7 @@ function handleMorphdomUpdate(text) {
);
// Add toggle listeners for new blocks
- document.querySelectorAll(".thinking-block").forEach(block => {
+ queryScope.querySelectorAll(".thinking-block").forEach(block => {
if (!block._hasToggleListener) {
block.addEventListener("toggle", function(e) {
if (this.open) {
diff --git a/modules/chat.py b/modules/chat.py
index 1222d2bb..f1ea16f1 100644
--- a/modules/chat.py
+++ b/modules/chat.py
@@ -825,7 +825,9 @@ def generate_chat_reply_wrapper(text, state, regenerate=False, _continue=False):
last_save_time = time.monotonic()
save_interval = 8
for i, history in enumerate(generate_chat_reply(text, state, regenerate, _continue, loading_message=True, for_ui=True)):
- yield chat_html_wrapper(history, state['name1'], state['name2'], state['mode'], state['chat_style'], state['character_menu']), history
+ yield chat_html_wrapper(history, state['name1'], state['name2'], state['mode'], state['chat_style'], state['character_menu'], last_message_only=(i > 0)), history
+ if i == 0:
+ time.sleep(0.125) # We need this to make sure the first update goes through
current_time = time.monotonic()
# Save on first iteration or if save_interval seconds have passed
diff --git a/modules/html_generator.py b/modules/html_generator.py
index 03b5d485..f90e3b04 100644
--- a/modules/html_generator.py
+++ b/modules/html_generator.py
@@ -462,64 +462,69 @@ def actions_html(history, i, role, info_message=""):
f'{version_nav_html}')
-def generate_instruct_html(history):
- output = f'"
+
return output
-def generate_cai_chat_html(history, name1, name2, style, character, reset_cache=False):
- output = f'
'
+def generate_cai_chat_html(history, name1, name2, style, character, reset_cache=False, last_message_only=False):
+ if not last_message_only:
+ output = f'"
+ # Determine range
+ start_idx = len(history['visible']) - 1 if last_message_only else 0
+ end_idx = len(history['visible'])
+
+ for i in range(start_idx, end_idx):
+ row_visible = history['visible'][i]
+ row_internal = history['internal'][i]
+
+ # Convert content
+ if last_message_only:
+ converted_visible = [None, convert_to_markdown_wrapped(row_visible[1], message_id=i, use_cache=i != len(history['visible']) - 1)]
+ else:
+ converted_visible = [convert_to_markdown_wrapped(entry, message_id=i, use_cache=i != len(history['visible']) - 1) for entry in row_visible]
+
+ # Generate messages
+ if not last_message_only and converted_visible[0]:
+ output += create_message("user", converted_visible[0], row_internal[0])
+
+ output += create_message("assistant", converted_visible[1], row_internal[1])
+
+ if not last_message_only:
+ output += "
'
+ else:
+ output = ""
# We use ?character and ?time.time() to force the browser to reset caches
img_bot = (
@@ -527,110 +532,117 @@ def generate_cai_chat_html(history, name1, name2, style, character, reset_cache=
if Path("user_data/cache/pfp_character_thumb.png").exists() else ''
)
- img_me = (
- f' if reset_cache else )
'
- if Path("user_data/cache/pfp_me.png").exists() else ''
- )
+ def create_message(role, content, raw_content):
+ """Inner function for CAI-style messages."""
+ circle_class = "circle-you" if role == "user" else "circle-bot"
+ name = name1 if role == "user" else name2
- for i in range(len(history['visible'])):
- row_visible = history['visible'][i]
- row_internal = history['internal'][i]
- converted_visible = [convert_to_markdown_wrapped(entry, message_id=i, use_cache=i != len(history['visible']) - 1) for entry in row_visible]
+ # Get role-specific data
+ timestamp = format_message_timestamp(history, role, i, tooltip_include_timestamp=False)
+ attachments = format_message_attachments(history, role, i)
- # Get timestamps
- user_timestamp = format_message_timestamp(history, "user", i, tooltip_include_timestamp=False)
- assistant_timestamp = format_message_timestamp(history, "assistant", i, tooltip_include_timestamp=False)
+ # Get appropriate image
+ if role == "user":
+ img = (f''
+ if Path("user_data/cache/pfp_me.png").exists() else '')
+ else:
+ img = img_bot
- # Get attachments
- user_attachments = format_message_attachments(history, "user", i)
- assistant_attachments = format_message_attachments(history, "assistant", i)
-
- if converted_visible[0]: # Don't display empty user messages
- output += (
- f''
- )
-
- output += (
+ return (
f''
)
- output += "
'
- if Path("user_data/cache/pfp_me.png").exists() else ''
- )
+ def create_message(role, content, raw_content):
+ """Inner function for CAI-style messages."""
+ circle_class = "circle-you" if role == "user" else "circle-bot"
+ name = name1 if role == "user" else name2
- for i in range(len(history['visible'])):
- row_visible = history['visible'][i]
- row_internal = history['internal'][i]
- converted_visible = [convert_to_markdown_wrapped(entry, message_id=i, use_cache=i != len(history['visible']) - 1) for entry in row_visible]
+ # Get role-specific data
+ timestamp = format_message_timestamp(history, role, i, tooltip_include_timestamp=False)
+ attachments = format_message_attachments(history, role, i)
- # Get timestamps
- user_timestamp = format_message_timestamp(history, "user", i, tooltip_include_timestamp=False)
- assistant_timestamp = format_message_timestamp(history, "assistant", i, tooltip_include_timestamp=False)
+ # Get appropriate image
+ if role == "user":
+ img = (f''
+ if Path("user_data/cache/pfp_me.png").exists() else '')
+ else:
+ img = img_bot
- # Get attachments
- user_attachments = format_message_attachments(history, "user", i)
- assistant_attachments = format_message_attachments(history, "assistant", i)
-
- if converted_visible[0]: # Don't display empty user messages
- output += (
- f''
- )
-
- output += (
+ return (
f''
)
- output += ""
+
return output
@@ -644,15 +656,15 @@ def time_greeting():
return "Good evening!"
-def chat_html_wrapper(history, name1, name2, mode, style, character, reset_cache=False):
+def chat_html_wrapper(history, name1, name2, mode, style, character, reset_cache=False, last_message_only=False):
if len(history['visible']) == 0:
greeting = f"
Date: Mon, 2 Jun 2025 09:57:55 -0300
Subject: [PATCH 50/59] Simplify the one-click installer (#7039)
---
one_click.py | 253 +++++++++++++++++++++++++--------------------------
1 file changed, 123 insertions(+), 130 deletions(-)
diff --git a/one_click.py b/one_click.py
index 482a6aa9..cccb0dc9 100644
--- a/one_click.py
+++ b/one_click.py
@@ -70,12 +70,8 @@ def is_installed():
def cpu_has_avx2():
try:
import cpuinfo
-
info = cpuinfo.get_cpu_info()
- if 'avx2' in info['flags']:
- return True
- else:
- return False
+ return 'avx2' in info['flags']
except:
return True
@@ -83,30 +79,112 @@ def cpu_has_avx2():
def cpu_has_amx():
try:
import cpuinfo
-
info = cpuinfo.get_cpu_info()
- if 'amx' in info['flags']:
- return True
- else:
- return False
+ return 'amx' in info['flags']
except:
return True
-def torch_version():
- site_packages_path = None
- for sitedir in site.getsitepackages():
- if "site-packages" in sitedir and conda_env_path in sitedir:
- site_packages_path = sitedir
- break
+def load_state():
+ """Load installer state from JSON file"""
+ if os.path.exists(state_file):
+ try:
+ with open(state_file, 'r') as f:
+ return json.load(f)
+ except:
+ return {}
+ return {}
- if site_packages_path:
- torch_version_file = open(os.path.join(site_packages_path, 'torch', 'version.py')).read().splitlines()
- torver = [line for line in torch_version_file if line.startswith('__version__')][0].split('__version__ = ')[1].strip("'")
+
+def save_state(state):
+ """Save installer state to JSON file"""
+ with open(state_file, 'w') as f:
+ json.dump(state, f)
+
+
+def get_gpu_choice():
+ """Get GPU choice from state file or ask user"""
+ state = load_state()
+ gpu_choice = state.get('gpu_choice')
+
+ if not gpu_choice:
+ if "GPU_CHOICE" in os.environ:
+ choice = os.environ["GPU_CHOICE"].upper()
+ print_big_message(f"Selected GPU choice \"{choice}\" based on the GPU_CHOICE environment variable.")
+ else:
+ choice = get_user_choice(
+ "What is your GPU?",
+ {
+ 'A': 'NVIDIA - CUDA 12.4',
+ 'B': 'AMD - Linux/macOS only, requires ROCm 6.2.4',
+ 'C': 'Apple M Series',
+ 'D': 'Intel Arc (beta)',
+ 'N': 'CPU mode'
+ },
+ )
+
+ # Convert choice to GPU name
+ gpu_choice = {"A": "NVIDIA", "B": "AMD", "C": "APPLE", "D": "INTEL", "N": "NONE"}[choice]
+
+ # Save choice to state
+ state['gpu_choice'] = gpu_choice
+ save_state(state)
+
+ return gpu_choice
+
+
+def get_pytorch_install_command(gpu_choice):
+ """Get PyTorch installation command based on GPU choice"""
+ base_cmd = f"python -m pip install torch=={TORCH_VERSION} torchvision=={TORCHVISION_VERSION} torchaudio=={TORCHAUDIO_VERSION} "
+
+ if gpu_choice == "NVIDIA":
+ return base_cmd + "--index-url https://download.pytorch.org/whl/cu124"
+ elif gpu_choice == "AMD":
+ return base_cmd + "--index-url https://download.pytorch.org/whl/rocm6.2.4"
+ elif gpu_choice in ["APPLE", "NONE"]:
+ return base_cmd + "--index-url https://download.pytorch.org/whl/cpu"
+ elif gpu_choice == "INTEL":
+ if is_linux():
+ return "python -m pip install torch==2.1.0a0 torchvision==0.16.0a0 torchaudio==2.1.0a0 intel-extension-for-pytorch==2.1.10+xpu --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/"
+ else:
+ return "python -m pip install torch==2.1.0a0 torchvision==0.16.0a0 torchaudio==2.1.0a0 intel-extension-for-pytorch==2.1.10 --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/"
else:
- from torch import __version__ as torver
+ return base_cmd
- return torver
+
+def get_pytorch_update_command(gpu_choice):
+ """Get PyTorch update command based on GPU choice"""
+ base_cmd = f"python -m pip install --upgrade torch=={TORCH_VERSION} torchvision=={TORCHVISION_VERSION} torchaudio=={TORCHAUDIO_VERSION}"
+
+ if gpu_choice == "NVIDIA":
+ return f"{base_cmd} --index-url https://download.pytorch.org/whl/cu124"
+ elif gpu_choice == "AMD":
+ return f"{base_cmd} --index-url https://download.pytorch.org/whl/rocm6.2.4"
+ elif gpu_choice in ["APPLE", "NONE"]:
+ return f"{base_cmd} --index-url https://download.pytorch.org/whl/cpu"
+ elif gpu_choice == "INTEL":
+ intel_extension = "intel-extension-for-pytorch==2.1.10+xpu" if is_linux() else "intel-extension-for-pytorch==2.1.10"
+ return f"{base_cmd} {intel_extension} --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/"
+ else:
+ return base_cmd
+
+
+def get_requirements_file(gpu_choice):
+ """Get requirements file path based on GPU choice"""
+ requirements_base = os.path.join("requirements", "full")
+
+ if gpu_choice == "AMD":
+ file_name = f"requirements_amd{'_noavx2' if not cpu_has_avx2() else ''}.txt"
+ elif gpu_choice == "APPLE":
+ file_name = f"requirements_apple_{'intel' if is_x86_64() else 'silicon'}.txt"
+ elif gpu_choice in ["INTEL", "NONE"]:
+ file_name = f"requirements_cpu_only{'_noavx2' if not cpu_has_avx2() else ''}.txt"
+ elif gpu_choice == "NVIDIA":
+ file_name = f"requirements{'_noavx2' if not cpu_has_avx2() else ''}.txt"
+ else:
+ raise ValueError(f"Unknown GPU choice: {gpu_choice}")
+
+ return os.path.join(requirements_base, file_name)
def get_current_commit():
@@ -209,28 +287,8 @@ def get_user_choice(question, options_dict):
def update_pytorch_and_python():
print_big_message("Checking for PyTorch updates.")
-
- # Update the Python version. Left here for future reference in case this becomes necessary.
- # print_big_message("Checking for PyTorch and Python updates.")
- # current_python_version = f"{sys.version_info.major}.{sys.version_info.minor}"
- # if current_python_version != PYTHON_VERSION:
- # run_cmd(f"conda install -y python={PYTHON_VERSION}", assert_success=True, environment=True)
-
- torver = torch_version()
- base_cmd = f"python -m pip install --upgrade torch=={TORCH_VERSION} torchvision=={TORCHVISION_VERSION} torchaudio=={TORCHAUDIO_VERSION}"
-
- if "+cu" in torver:
- install_cmd = f"{base_cmd} --index-url https://download.pytorch.org/whl/cu124"
- elif "+rocm" in torver:
- install_cmd = f"{base_cmd} --index-url https://download.pytorch.org/whl/rocm6.2.4"
- elif "+cpu" in torver:
- install_cmd = f"{base_cmd} --index-url https://download.pytorch.org/whl/cpu"
- elif "+cxx11" in torver:
- intel_extension = "intel-extension-for-pytorch==2.1.10+xpu" if is_linux() else "intel-extension-for-pytorch==2.1.10"
- install_cmd = f"{base_cmd} {intel_extension} --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/"
- else:
- install_cmd = base_cmd
-
+ gpu_choice = get_gpu_choice()
+ install_cmd = get_pytorch_update_command(gpu_choice)
run_cmd(install_cmd, assert_success=True, environment=True)
@@ -256,43 +314,11 @@ def install_webui():
if os.path.isfile(state_file):
os.remove(state_file)
- # Ask the user for the GPU vendor
- if "GPU_CHOICE" in os.environ:
- choice = os.environ["GPU_CHOICE"].upper()
- print_big_message(f"Selected GPU choice \"{choice}\" based on the GPU_CHOICE environment variable.")
-
- # Warn about changed meanings and handle old choices
- if choice == "B":
- print_big_message("Warning: GPU_CHOICE='B' now means 'AMD' in the new version.")
- elif choice == "C":
- print_big_message("Warning: GPU_CHOICE='C' now means 'Apple M Series' in the new version.")
- elif choice == "D":
- print_big_message("Warning: GPU_CHOICE='D' now means 'Intel Arc' in the new version.")
- else:
- choice = get_user_choice(
- "What is your GPU?",
- {
- 'A': 'NVIDIA - CUDA 12.4',
- 'B': 'AMD - Linux/macOS only, requires ROCm 6.2.4',
- 'C': 'Apple M Series',
- 'D': 'Intel Arc (beta)',
- 'N': 'CPU mode'
- },
- )
-
- # Convert choices to GPU names for compatibility
- gpu_choice_to_name = {
- "A": "NVIDIA",
- "B": "AMD",
- "C": "APPLE",
- "D": "INTEL",
- "N": "NONE"
- }
-
- selected_gpu = gpu_choice_to_name[choice]
+ # Get GPU choice and save it to state
+ gpu_choice = get_gpu_choice()
# Write a flag to CMD_FLAGS.txt for CPU mode
- if selected_gpu == "NONE":
+ if gpu_choice == "NONE":
cmd_flags_path = os.path.join(script_dir, "user_data", "CMD_FLAGS.txt")
with open(cmd_flags_path, 'r+') as cmd_flags_file:
if "--cpu" not in cmd_flags_file.read():
@@ -300,34 +326,20 @@ def install_webui():
cmd_flags_file.write("\n--cpu\n")
# Handle CUDA version display
- elif any((is_windows(), is_linux())) and selected_gpu == "NVIDIA":
+ elif any((is_windows(), is_linux())) and gpu_choice == "NVIDIA":
print("CUDA: 12.4")
# No PyTorch for AMD on Windows (?)
- elif is_windows() and selected_gpu == "AMD":
+ elif is_windows() and gpu_choice == "AMD":
print("PyTorch setup on Windows is not implemented yet. Exiting...")
sys.exit(1)
- # Find the Pytorch installation command
- install_pytorch = f"python -m pip install torch=={TORCH_VERSION} torchvision=={TORCHVISION_VERSION} torchaudio=={TORCHAUDIO_VERSION} "
-
- if selected_gpu == "NVIDIA":
- install_pytorch += "--index-url https://download.pytorch.org/whl/cu124"
- elif selected_gpu == "AMD":
- install_pytorch += "--index-url https://download.pytorch.org/whl/rocm6.2.4"
- elif selected_gpu in ["APPLE", "NONE"]:
- install_pytorch += "--index-url https://download.pytorch.org/whl/cpu"
- elif selected_gpu == "INTEL":
- if is_linux():
- install_pytorch = "python -m pip install torch==2.1.0a0 torchvision==0.16.0a0 torchaudio==2.1.0a0 intel-extension-for-pytorch==2.1.10+xpu --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/"
- else:
- install_pytorch = "python -m pip install torch==2.1.0a0 torchvision==0.16.0a0 torchaudio==2.1.0a0 intel-extension-for-pytorch==2.1.10 --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/"
-
# Install Git and then Pytorch
print_big_message("Installing PyTorch.")
+ install_pytorch = get_pytorch_install_command(gpu_choice)
run_cmd(f"conda install -y ninja git && {install_pytorch} && python -m pip install py-cpuinfo==9.0.0", assert_success=True, environment=True)
- if selected_gpu == "INTEL":
+ if gpu_choice == "INTEL":
# Install oneAPI dependencies via conda
print_big_message("Installing Intel oneAPI runtime libraries.")
run_cmd("conda install -y -c https://software.repos.intel.com/python/conda/ -c conda-forge dpcpp-cpp-rt=2024.0 mkl-dpcpp=2024.0", environment=True)
@@ -349,31 +361,15 @@ def update_requirements(initial_installation=False, pull=True):
assert_success=True
)
- torver = torch_version()
- requirements_base = os.path.join("requirements", "full")
-
- if "+rocm" in torver:
- file_name = f"requirements_amd{'_noavx2' if not cpu_has_avx2() else ''}.txt"
- elif "+cpu" in torver or "+cxx11" in torver:
- file_name = f"requirements_cpu_only{'_noavx2' if not cpu_has_avx2() else ''}.txt"
- elif is_macos():
- file_name = f"requirements_apple_{'intel' if is_x86_64() else 'silicon'}.txt"
- else:
- file_name = f"requirements{'_noavx2' if not cpu_has_avx2() else ''}.txt"
-
- requirements_file = os.path.join(requirements_base, file_name)
-
- # Load state from JSON file
current_commit = get_current_commit()
- wheels_changed = False
- if os.path.exists(state_file):
- with open(state_file, 'r') as f:
- last_state = json.load(f)
-
- if 'wheels_changed' in last_state or last_state.get('last_installed_commit') != current_commit:
+ wheels_changed = not os.path.exists(state_file)
+ if not wheels_changed:
+ state = load_state()
+ if 'wheels_changed' in state or state.get('last_installed_commit') != current_commit:
wheels_changed = True
- else:
- wheels_changed = True
+
+ gpu_choice = get_gpu_choice()
+ requirements_file = get_requirements_file(gpu_choice)
if pull:
# Read .whl lines before pulling
@@ -409,19 +405,17 @@ def update_requirements(initial_installation=False, pull=True):
print_big_message(f"File '{file}' was updated during 'git pull'. Please run the script again.")
# Save state before exiting
- current_state = {}
+ state = load_state()
if wheels_changed:
- current_state['wheels_changed'] = True
-
- with open(state_file, 'w') as f:
- json.dump(current_state, f)
-
+ state['wheels_changed'] = True
+ save_state(state)
sys.exit(1)
# Save current state
- current_state = {'last_installed_commit': current_commit}
- with open(state_file, 'w') as f:
- json.dump(current_state, f)
+ state = load_state()
+ state['last_installed_commit'] = current_commit
+ state.pop('wheels_changed', None) # Remove wheels_changed flag
+ save_state(state)
if os.environ.get("INSTALL_EXTENSIONS", "").lower() in ("yes", "y", "true", "1", "t", "on"):
install_extensions_requirements()
@@ -432,11 +426,10 @@ def update_requirements(initial_installation=False, pull=True):
# Update PyTorch
if not initial_installation:
update_pytorch_and_python()
- torver = torch_version()
clean_outdated_pytorch_cuda_dependencies()
print_big_message(f"Installing webui requirements from file: {requirements_file}")
- print(f"TORCH: {torver}\n")
+ print(f"GPU Choice: {gpu_choice}\n")
# Prepare the requirements file
textgen_requirements = open(requirements_file).read().splitlines()
From b30a73016d626e985e248de15fa65e5a531c8bd2 Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Mon, 2 Jun 2025 07:49:22 -0700
Subject: [PATCH 51/59] Remove the "Is typing..." yield by default
---
modules/chat.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/modules/chat.py b/modules/chat.py
index f1ea16f1..3c4c3636 100644
--- a/modules/chat.py
+++ b/modules/chat.py
@@ -656,7 +656,7 @@ def chatbot_wrapper(text, state, regenerate=False, _continue=False, loading_mess
update_message_metadata(output['metadata'], "user", row_idx, timestamp=get_current_timestamp())
# *Is typing...*
- if loading_message:
+ if loading_message and shared.processing_message:
yield {
'visible': output['visible'][:-1] + [[output['visible'][-1][0], shared.processing_message]],
'internal': output['internal'],
@@ -680,7 +680,7 @@ def chatbot_wrapper(text, state, regenerate=False, _continue=False, loading_mess
})
output['metadata'][key]["current_version_index"] = len(output['metadata'][key]["versions"]) - 1
- if loading_message:
+ if loading_message and shared.processing_message:
yield {
'visible': output['visible'][:-1] + [[visible_text, shared.processing_message]],
'internal': output['internal'][:-1] + [[text, '']],
From b38ec0ec385d44d49d3fe7adf2ad77ae62302214 Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Mon, 2 Jun 2025 11:33:17 -0700
Subject: [PATCH 52/59] Update llama.cpp
---
requirements/full/requirements.txt | 4 ++--
requirements/full/requirements_amd.txt | 4 ++--
requirements/full/requirements_amd_noavx2.txt | 4 ++--
requirements/full/requirements_apple_intel.txt | 4 ++--
requirements/full/requirements_apple_silicon.txt | 6 +++---
requirements/full/requirements_cpu_only.txt | 4 ++--
requirements/full/requirements_cpu_only_noavx2.txt | 4 ++--
requirements/full/requirements_noavx2.txt | 4 ++--
requirements/portable/requirements.txt | 4 ++--
requirements/portable/requirements_apple_intel.txt | 4 ++--
requirements/portable/requirements_apple_silicon.txt | 6 +++---
requirements/portable/requirements_cpu_only.txt | 4 ++--
requirements/portable/requirements_cpu_only_noavx2.txt | 4 ++--
requirements/portable/requirements_noavx2.txt | 4 ++--
requirements/portable/requirements_vulkan.txt | 4 ++--
requirements/portable/requirements_vulkan_noavx2.txt | 4 ++--
16 files changed, 34 insertions(+), 34 deletions(-)
diff --git a/requirements/full/requirements.txt b/requirements/full/requirements.txt
index 04d97220..277f8249 100644
--- a/requirements/full/requirements.txt
+++ b/requirements/full/requirements.txt
@@ -34,8 +34,8 @@ sse-starlette==1.6.5
tiktoken
# CUDA wheels
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0+cu124-py3-none-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0+cu124-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+cu124-py3-none-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+cu124-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
https://github.com/oobabooga/exllamav3/releases/download/v0.0.3/exllamav3-0.0.3+cu124.torch2.6.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
https://github.com/oobabooga/exllamav3/releases/download/v0.0.3/exllamav3-0.0.3+cu124.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
https://github.com/turboderp-org/exllamav2/releases/download/v0.3.1/exllamav2-0.3.1+cu124.torch2.6.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
diff --git a/requirements/full/requirements_amd.txt b/requirements/full/requirements_amd.txt
index f807199d..dbf35c34 100644
--- a/requirements/full/requirements_amd.txt
+++ b/requirements/full/requirements_amd.txt
@@ -33,7 +33,7 @@ sse-starlette==1.6.5
tiktoken
# AMD wheels
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0+vulkan-py3-none-win_amd64.whl; platform_system == "Windows"
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0+vulkan-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+vulkan-py3-none-win_amd64.whl; platform_system == "Windows"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+vulkan-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
https://github.com/turboderp-org/exllamav2/releases/download/v0.3.1/exllamav2-0.3.1+rocm6.2.4.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
https://github.com/turboderp-org/exllamav2/releases/download/v0.3.1/exllamav2-0.3.1-py3-none-any.whl; platform_system != "Darwin" and platform_machine != "x86_64"
diff --git a/requirements/full/requirements_amd_noavx2.txt b/requirements/full/requirements_amd_noavx2.txt
index 4fb70eb1..2e5eb6c9 100644
--- a/requirements/full/requirements_amd_noavx2.txt
+++ b/requirements/full/requirements_amd_noavx2.txt
@@ -33,7 +33,7 @@ sse-starlette==1.6.5
tiktoken
# AMD wheels
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0+vulkanavx-py3-none-win_amd64.whl; platform_system == "Windows"
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0+vulkanavx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+vulkanavx-py3-none-win_amd64.whl; platform_system == "Windows"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+vulkanavx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
https://github.com/turboderp-org/exllamav2/releases/download/v0.3.1/exllamav2-0.3.1+rocm6.2.4.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
https://github.com/turboderp-org/exllamav2/releases/download/v0.3.1/exllamav2-0.3.1-py3-none-any.whl; platform_system != "Darwin" and platform_machine != "x86_64"
diff --git a/requirements/full/requirements_apple_intel.txt b/requirements/full/requirements_apple_intel.txt
index a311ab9b..9a19ab29 100644
--- a/requirements/full/requirements_apple_intel.txt
+++ b/requirements/full/requirements_apple_intel.txt
@@ -33,7 +33,7 @@ sse-starlette==1.6.5
tiktoken
# Mac wheels
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0-py3-none-macosx_15_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "24.0.0" and platform_release < "25.0.0" and python_version == "3.11"
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0-py3-none-macosx_14_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" and python_version == "3.11"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0-py3-none-macosx_15_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "24.0.0" and platform_release < "25.0.0" and python_version == "3.11"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0-py3-none-macosx_14_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" and python_version == "3.11"
https://github.com/oobabooga/exllamav3/releases/download/v0.0.3/exllamav3-0.0.3-py3-none-any.whl
https://github.com/turboderp-org/exllamav2/releases/download/v0.3.1/exllamav2-0.3.1-py3-none-any.whl
diff --git a/requirements/full/requirements_apple_silicon.txt b/requirements/full/requirements_apple_silicon.txt
index 30e8409a..973d9bfb 100644
--- a/requirements/full/requirements_apple_silicon.txt
+++ b/requirements/full/requirements_apple_silicon.txt
@@ -33,8 +33,8 @@ sse-starlette==1.6.5
tiktoken
# Mac wheels
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0-py3-none-macosx_15_0_arm64.whl; platform_system == "Darwin" and platform_release >= "24.0.0" and platform_release < "25.0.0" and python_version == "3.11"
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0-py3-none-macosx_14_0_arm64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" and python_version == "3.11"
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0-py3-none-macosx_13_0_arm64.whl; platform_system == "Darwin" and platform_release >= "22.0.0" and platform_release < "23.0.0" and python_version == "3.11"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0-py3-none-macosx_15_0_arm64.whl; platform_system == "Darwin" and platform_release >= "24.0.0" and platform_release < "25.0.0" and python_version == "3.11"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0-py3-none-macosx_14_0_arm64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" and python_version == "3.11"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0-py3-none-macosx_13_0_arm64.whl; platform_system == "Darwin" and platform_release >= "22.0.0" and platform_release < "23.0.0" and python_version == "3.11"
https://github.com/oobabooga/exllamav3/releases/download/v0.0.3/exllamav3-0.0.3-py3-none-any.whl
https://github.com/turboderp-org/exllamav2/releases/download/v0.3.1/exllamav2-0.3.1-py3-none-any.whl
diff --git a/requirements/full/requirements_cpu_only.txt b/requirements/full/requirements_cpu_only.txt
index 70949949..4a48a51f 100644
--- a/requirements/full/requirements_cpu_only.txt
+++ b/requirements/full/requirements_cpu_only.txt
@@ -33,5 +33,5 @@ sse-starlette==1.6.5
tiktoken
# llama.cpp (CPU only, AVX2)
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0+cpuavx2-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0+cpuavx2-py3-none-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+cpuavx2-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+cpuavx2-py3-none-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
diff --git a/requirements/full/requirements_cpu_only_noavx2.txt b/requirements/full/requirements_cpu_only_noavx2.txt
index 318bb93a..76bde864 100644
--- a/requirements/full/requirements_cpu_only_noavx2.txt
+++ b/requirements/full/requirements_cpu_only_noavx2.txt
@@ -33,5 +33,5 @@ sse-starlette==1.6.5
tiktoken
# llama.cpp (CPU only, no AVX2)
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0+cpuavx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0+cpuavx-py3-none-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+cpuavx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+cpuavx-py3-none-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
diff --git a/requirements/full/requirements_noavx2.txt b/requirements/full/requirements_noavx2.txt
index 7c3635cc..6cd0fa65 100644
--- a/requirements/full/requirements_noavx2.txt
+++ b/requirements/full/requirements_noavx2.txt
@@ -34,8 +34,8 @@ sse-starlette==1.6.5
tiktoken
# CUDA wheels
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0+cu124avx-py3-none-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0+cu124avx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+cu124avx-py3-none-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+cu124avx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
https://github.com/oobabooga/exllamav3/releases/download/v0.0.3/exllamav3-0.0.3+cu124.torch2.6.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
https://github.com/oobabooga/exllamav3/releases/download/v0.0.3/exllamav3-0.0.3+cu124.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
https://github.com/turboderp-org/exllamav2/releases/download/v0.3.1/exllamav2-0.3.1+cu124.torch2.6.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
diff --git a/requirements/portable/requirements.txt b/requirements/portable/requirements.txt
index bde310e1..60ce941e 100644
--- a/requirements/portable/requirements.txt
+++ b/requirements/portable/requirements.txt
@@ -19,5 +19,5 @@ sse-starlette==1.6.5
tiktoken
# CUDA wheels
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0+cu124-py3-none-win_amd64.whl; platform_system == "Windows"
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0+cu124-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+cu124-py3-none-win_amd64.whl; platform_system == "Windows"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+cu124-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
diff --git a/requirements/portable/requirements_apple_intel.txt b/requirements/portable/requirements_apple_intel.txt
index 521edc0c..b1649bc9 100644
--- a/requirements/portable/requirements_apple_intel.txt
+++ b/requirements/portable/requirements_apple_intel.txt
@@ -19,5 +19,5 @@ sse-starlette==1.6.5
tiktoken
# Mac wheels
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0-py3-none-macosx_15_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "24.0.0" and platform_release < "25.0.0"
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0-py3-none-macosx_14_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0-py3-none-macosx_15_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "24.0.0" and platform_release < "25.0.0"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0-py3-none-macosx_14_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0"
diff --git a/requirements/portable/requirements_apple_silicon.txt b/requirements/portable/requirements_apple_silicon.txt
index ef7946ff..571eba52 100644
--- a/requirements/portable/requirements_apple_silicon.txt
+++ b/requirements/portable/requirements_apple_silicon.txt
@@ -19,6 +19,6 @@ sse-starlette==1.6.5
tiktoken
# Mac wheels
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0-py3-none-macosx_15_0_arm64.whl; platform_system == "Darwin" and platform_release >= "24.0.0" and platform_release < "25.0.0"
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0-py3-none-macosx_14_0_arm64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0"
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0-py3-none-macosx_13_0_arm64.whl; platform_system == "Darwin" and platform_release >= "22.0.0" and platform_release < "23.0.0"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0-py3-none-macosx_15_0_arm64.whl; platform_system == "Darwin" and platform_release >= "24.0.0" and platform_release < "25.0.0"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0-py3-none-macosx_14_0_arm64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0-py3-none-macosx_13_0_arm64.whl; platform_system == "Darwin" and platform_release >= "22.0.0" and platform_release < "23.0.0"
diff --git a/requirements/portable/requirements_cpu_only.txt b/requirements/portable/requirements_cpu_only.txt
index a3ad743e..88170cf3 100644
--- a/requirements/portable/requirements_cpu_only.txt
+++ b/requirements/portable/requirements_cpu_only.txt
@@ -19,5 +19,5 @@ sse-starlette==1.6.5
tiktoken
# llama.cpp (CPU only, AVX2)
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0+cpuavx2-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0+cpuavx2-py3-none-win_amd64.whl; platform_system == "Windows"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+cpuavx2-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+cpuavx2-py3-none-win_amd64.whl; platform_system == "Windows"
diff --git a/requirements/portable/requirements_cpu_only_noavx2.txt b/requirements/portable/requirements_cpu_only_noavx2.txt
index eec052d3..e96cef49 100644
--- a/requirements/portable/requirements_cpu_only_noavx2.txt
+++ b/requirements/portable/requirements_cpu_only_noavx2.txt
@@ -19,5 +19,5 @@ sse-starlette==1.6.5
tiktoken
# llama.cpp (CPU only, no AVX2)
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0+cpuavx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0+cpuavx-py3-none-win_amd64.whl; platform_system == "Windows"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+cpuavx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+cpuavx-py3-none-win_amd64.whl; platform_system == "Windows"
diff --git a/requirements/portable/requirements_noavx2.txt b/requirements/portable/requirements_noavx2.txt
index c9898a05..78f94aa5 100644
--- a/requirements/portable/requirements_noavx2.txt
+++ b/requirements/portable/requirements_noavx2.txt
@@ -19,5 +19,5 @@ sse-starlette==1.6.5
tiktoken
# CUDA wheels
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0+cu124avx-py3-none-win_amd64.whl; platform_system == "Windows"
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0+cu124avx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+cu124avx-py3-none-win_amd64.whl; platform_system == "Windows"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+cu124avx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
diff --git a/requirements/portable/requirements_vulkan.txt b/requirements/portable/requirements_vulkan.txt
index 0de9c7cb..3e41427d 100644
--- a/requirements/portable/requirements_vulkan.txt
+++ b/requirements/portable/requirements_vulkan.txt
@@ -19,5 +19,5 @@ sse-starlette==1.6.5
tiktoken
# CUDA wheels
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0+vulkan-py3-none-win_amd64.whl; platform_system == "Windows"
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0+vulkan-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+vulkan-py3-none-win_amd64.whl; platform_system == "Windows"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+vulkan-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
diff --git a/requirements/portable/requirements_vulkan_noavx2.txt b/requirements/portable/requirements_vulkan_noavx2.txt
index 2bfb4d51..022ebb61 100644
--- a/requirements/portable/requirements_vulkan_noavx2.txt
+++ b/requirements/portable/requirements_vulkan_noavx2.txt
@@ -19,5 +19,5 @@ sse-starlette==1.6.5
tiktoken
# CUDA wheels
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0+vulkanavx-py3-none-win_amd64.whl; platform_system == "Windows"
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0+vulkanavx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+vulkanavx-py3-none-win_amd64.whl; platform_system == "Windows"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+vulkanavx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
From 93b3752cdf9f43dd391462168e2e14dd2ab75643 Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Wed, 4 Jun 2025 09:40:30 -0700
Subject: [PATCH 53/59] Revert "Remove the "Is typing..." yield by default"
This reverts commit b30a73016d626e985e248de15fa65e5a531c8bd2.
---
modules/chat.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/modules/chat.py b/modules/chat.py
index 3c4c3636..f1ea16f1 100644
--- a/modules/chat.py
+++ b/modules/chat.py
@@ -656,7 +656,7 @@ def chatbot_wrapper(text, state, regenerate=False, _continue=False, loading_mess
update_message_metadata(output['metadata'], "user", row_idx, timestamp=get_current_timestamp())
# *Is typing...*
- if loading_message and shared.processing_message:
+ if loading_message:
yield {
'visible': output['visible'][:-1] + [[output['visible'][-1][0], shared.processing_message]],
'internal': output['internal'],
@@ -680,7 +680,7 @@ def chatbot_wrapper(text, state, regenerate=False, _continue=False, loading_mess
})
output['metadata'][key]["current_version_index"] = len(output['metadata'][key]["versions"]) - 1
- if loading_message and shared.processing_message:
+ if loading_message:
yield {
'visible': output['visible'][:-1] + [[visible_text, shared.processing_message]],
'internal': output['internal'][:-1] + [[text, '']],
From 9bd7359ffab5e434b7cdfdb43ee91cb3ad397c0d Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Wed, 4 Jun 2025 10:47:14 -0700
Subject: [PATCH 54/59] Scroll the textarea into view when editing a message
---
js/global_scope_js.js | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/js/global_scope_js.js b/js/global_scope_js.js
index d5140c93..801f1574 100644
--- a/js/global_scope_js.js
+++ b/js/global_scope_js.js
@@ -95,6 +95,12 @@ function startEditing(messageElement, messageBody, isUserMessage) {
editingInterface.textarea.focus();
editingInterface.textarea.setSelectionRange(rawText.length, rawText.length);
+ // Scroll the textarea into view
+ editingInterface.textarea.scrollIntoView({
+ behavior: "smooth",
+ block: "center"
+ });
+
// Setup event handlers
setupEditingHandlers(editingInterface.textarea, messageElement, originalHTML, messageBody, isUserMessage);
}
From 66a75c899a4b0786cd8744886a189864923287b5 Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Wed, 4 Jun 2025 10:59:43 -0700
Subject: [PATCH 55/59] Improve the scrollbars in code blocks
---
js/main.js | 1 +
1 file changed, 1 insertion(+)
diff --git a/js/main.js b/js/main.js
index 8090937f..2e72d82e 100644
--- a/js/main.js
+++ b/js/main.js
@@ -229,6 +229,7 @@ function doSyntaxHighlighting() {
codeBlocks.forEach((codeBlock) => {
hljs.highlightElement(codeBlock);
codeBlock.setAttribute("data-highlighted", "true");
+ codeBlock.classList.add("pretty_scrollbar");
});
renderMathInElement(messageBody, {
From 3d676cd50f8661ca96a20a452611422acb47177c Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Wed, 4 Jun 2025 11:02:04 -0700
Subject: [PATCH 56/59] Optimize syntax highlighting
---
js/main.js | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/js/main.js b/js/main.js
index 2e72d82e..9a620fa9 100644
--- a/js/main.js
+++ b/js/main.js
@@ -217,7 +217,7 @@ function isElementVisibleOnScreen(element) {
}
function doSyntaxHighlighting() {
- const messageBodies = document.querySelectorAll(".message-body");
+ const messageBodies = document.getElementById("chat").querySelectorAll(".message-body");
if (messageBodies.length > 0) {
observer.disconnect();
From 3829507d0fd66eccc532b5d8d0e3d77c38143d0c Mon Sep 17 00:00:00 2001
From: Hanusz Leszek
Date: Wed, 4 Jun 2025 20:13:36 +0200
Subject: [PATCH 57/59] Stop model during graceful shutdown (#7042)
---
server.py | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/server.py b/server.py
index c22ed1f1..99d2e171 100644
--- a/server.py
+++ b/server.py
@@ -60,6 +60,14 @@ from modules.utils import gradio
def signal_handler(sig, frame):
logger.info("Received Ctrl+C. Shutting down Text generation web UI gracefully.")
+
+ # Explicitly stop LlamaServer to avoid __del__ cleanup issues during shutdown
+ if shared.model and shared.model.__class__.__name__ == 'LlamaServer':
+ try:
+ shared.model.stop()
+ except:
+ pass
+
sys.exit(0)
From 977ec801b7682c3239fe3e6fdfcb8b90c1e802f3 Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Thu, 5 Jun 2025 06:33:45 -0700
Subject: [PATCH 58/59] Improve table colors in instruct mode
---
css/html_instruct_style.css | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/css/html_instruct_style.css b/css/html_instruct_style.css
index 6ad250aa..9831ee8f 100644
--- a/css/html_instruct_style.css
+++ b/css/html_instruct_style.css
@@ -17,6 +17,14 @@
color: #d1d5db !important;
}
+.chat .message-body :is(th, td) {
+ border-color: #40404096 !important;
+}
+
+.dark .chat .message-body :is(th, td) {
+ border-color: #ffffff75 !important;
+}
+
.chat .message-body :is(p, ul, ol) {
margin: 1.25em 0 !important;
}
From d47c8eb956a72ebc7c1f582718758697aef62118 Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Thu, 5 Jun 2025 06:56:24 -0700
Subject: [PATCH 59/59] Remove quotes from LLM-generated websearch query
(closes #7045).
Fix by @Quiet-Joker
---
modules/chat.py | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/modules/chat.py b/modules/chat.py
index f1ea16f1..14f2a4f7 100644
--- a/modules/chat.py
+++ b/modules/chat.py
@@ -604,7 +604,12 @@ def generate_search_query(user_message, state):
query = ""
for reply in generate_reply(formatted_prompt, search_state, stopping_strings=[], is_chat=True):
- query = reply.strip()
+ query = reply
+
+ # Strip and remove surrounding quotes if present
+ query = query.strip()
+ if len(query) >= 2 and query.startswith('"') and query.endswith('"'):
+ query = query[1:-1]
return query
{time_greeting()} How can I help you today?
"
result = f'{greeting}
'
elif mode == 'instruct':
- result = generate_instruct_html(history)
+ result = generate_instruct_html(history, last_message_only=last_message_only)
elif style == 'wpp':
- result = generate_chat_html(history, name1, name2)
+ result = generate_chat_html(history, name1, name2, last_message_only=last_message_only)
else:
- result = generate_cai_chat_html(history, name1, name2, style, character, reset_cache)
+ result = generate_cai_chat_html(history, name1, name2, style, character, reset_cache=reset_cache, last_message_only=last_message_only)
- return {'html': result}
+ return {'html': result, 'last_message_only': last_message_only}
diff --git a/modules/shared.py b/modules/shared.py
index d2305f30..f712f7f8 100644
--- a/modules/shared.py
+++ b/modules/shared.py
@@ -21,7 +21,7 @@ lora_names = []
# Generation variables
stop_everything = False
generation_lock = None
-processing_message = '*Is typing...*'
+processing_message = ''
# UI variables
gradio = {}
@@ -47,7 +47,6 @@ settings = {
'max_new_tokens_max': 4096,
'prompt_lookup_num_tokens': 0,
'max_tokens_second': 0,
- 'max_updates_second': 12,
'auto_max_new_tokens': True,
'ban_eos_token': False,
'add_bos_token': True,
diff --git a/modules/text_generation.py b/modules/text_generation.py
index 1fd6d810..0d499d50 100644
--- a/modules/text_generation.py
+++ b/modules/text_generation.py
@@ -65,41 +65,39 @@ def _generate_reply(question, state, stopping_strings=None, is_chat=False, escap
all_stop_strings += st
shared.stop_everything = False
- last_update = -1
reply = ''
is_stream = state['stream']
if len(all_stop_strings) > 0 and not state['stream']:
state = copy.deepcopy(state)
state['stream'] = True
- min_update_interval = 0
- if state.get('max_updates_second', 0) > 0:
- min_update_interval = 1 / state['max_updates_second']
-
# Generate
+ last_update = -1
+ latency_threshold = 1 / 1000
for reply in generate_func(question, original_question, state, stopping_strings, is_chat=is_chat):
+ cur_time = time.monotonic()
reply, stop_found = apply_stopping_strings(reply, all_stop_strings)
if escape_html:
reply = html.escape(reply)
if is_stream:
- cur_time = time.time()
-
# Limit number of tokens/second to make text readable in real time
if state['max_tokens_second'] > 0:
diff = 1 / state['max_tokens_second'] - (cur_time - last_update)
if diff > 0:
time.sleep(diff)
- last_update = time.time()
+ last_update = time.monotonic()
yield reply
# Limit updates to avoid lag in the Gradio UI
# API updates are not limited
else:
- if cur_time - last_update > min_update_interval:
- last_update = cur_time
+ # If 'generate_func' takes less than 0.001 seconds to yield the next token
+ # (equivalent to more than 1000 tok/s), assume that the UI is lagging behind and skip yielding
+ if (cur_time - last_update) > latency_threshold:
yield reply
+ last_update = time.monotonic()
if stop_found or (state['max_tokens_second'] > 0 and shared.stop_everything):
break
diff --git a/modules/ui.py b/modules/ui.py
index 9f4d67cb..14a09d2b 100644
--- a/modules/ui.py
+++ b/modules/ui.py
@@ -6,6 +6,7 @@ import yaml
import extensions
from modules import shared
+from modules.chat import load_history
with open(Path(__file__).resolve().parent / '../css/NotoSans/stylesheet.css', 'r') as f:
css = f.read()
@@ -194,7 +195,6 @@ def list_interface_input_elements():
'max_new_tokens',
'prompt_lookup_num_tokens',
'max_tokens_second',
- 'max_updates_second',
'do_sample',
'dynamic_temperature',
'temperature_last',
@@ -270,6 +270,10 @@ def gather_interface_values(*args):
if not shared.args.multi_user:
shared.persistent_interface_state = output
+ # Prevent history loss if backend is restarted but UI is not refreshed
+ if output['history'] is None and output['unique_id'] is not None:
+ output['history'] = load_history(output['unique_id'], output['character_menu'], output['mode'])
+
return output
diff --git a/modules/ui_chat.py b/modules/ui_chat.py
index 822b77b8..0d5a2c18 100644
--- a/modules/ui_chat.py
+++ b/modules/ui_chat.py
@@ -18,7 +18,7 @@ def create_ui():
mu = shared.args.multi_user
shared.gradio['Chat input'] = gr.State()
- shared.gradio['history'] = gr.JSON(visible=False)
+ shared.gradio['history'] = gr.State({'internal': [], 'visible': [], 'metadata': {}})
with gr.Tab('Chat', id='Chat', elem_id='chat-tab'):
with gr.Row(elem_id='past-chats-row', elem_classes=['pretty_scrollbar']):
@@ -195,7 +195,7 @@ def create_event_handlers():
shared.reload_inputs = gradio(reload_arr)
# Morph HTML updates instead of updating everything
- shared.gradio['display'].change(None, gradio('display'), None, js="(data) => handleMorphdomUpdate(data.html)")
+ shared.gradio['display'].change(None, gradio('display'), None, js="(data) => handleMorphdomUpdate(data)")
shared.gradio['Generate'].click(
ui.gather_interface_values, gradio(shared.input_elements), gradio('interface_state')).then(
diff --git a/modules/ui_parameters.py b/modules/ui_parameters.py
index 733d0901..84f9fbfc 100644
--- a/modules/ui_parameters.py
+++ b/modules/ui_parameters.py
@@ -71,8 +71,6 @@ def create_ui(default_preset):
shared.gradio['max_new_tokens'] = gr.Slider(minimum=shared.settings['max_new_tokens_min'], maximum=shared.settings['max_new_tokens_max'], value=shared.settings['max_new_tokens'], step=1, label='max_new_tokens', info='⚠️ Setting this too high can cause prompt truncation.')
shared.gradio['prompt_lookup_num_tokens'] = gr.Slider(value=shared.settings['prompt_lookup_num_tokens'], minimum=0, maximum=10, step=1, label='prompt_lookup_num_tokens', info='Activates Prompt Lookup Decoding.')
shared.gradio['max_tokens_second'] = gr.Slider(value=shared.settings['max_tokens_second'], minimum=0, maximum=20, step=1, label='Maximum tokens/second', info='To make text readable in real time.')
- shared.gradio['max_updates_second'] = gr.Slider(value=shared.settings['max_updates_second'], minimum=0, maximum=24, step=1, label='Maximum UI updates/second', info='Set this if you experience lag in the UI during streaming.')
-
with gr.Column():
with gr.Row():
with gr.Column():
diff --git a/user_data/settings-template.yaml b/user_data/settings-template.yaml
index ce0f77e1..db481e84 100644
--- a/user_data/settings-template.yaml
+++ b/user_data/settings-template.yaml
@@ -18,7 +18,6 @@ max_new_tokens_min: 1
max_new_tokens_max: 4096
prompt_lookup_num_tokens: 0
max_tokens_second: 0
-max_updates_second: 12
auto_max_new_tokens: true
ban_eos_token: false
add_bos_token: true
From 7278548cd18a9ba05062eb2db59d7f2965d8a9f6 Mon Sep 17 00:00:00 2001
From: oobabooga