'`, where `img_str` is base-64 jpeg data. Note that you will need to launch `server.py` with the arguments `--api --extensions multimodal`.
-
-### Completions endpoint
-
-Python example:
-
-```Python
-import base64
-import requests
-
-CONTEXT = "You are LLaVA, a large language and vision assistant trained by UW Madison WAIV Lab. You are able to understand the visual content that the user provides, and assist the user with a variety of tasks using natural language. Follow the instructions carefully and explain your answers in detail.### Human: Hi!### Assistant: Hi there! How can I help you today?\n"
-
-with open('extreme_ironing.jpg', 'rb') as f:
- img_str = base64.b64encode(f.read()).decode('utf-8')
- prompt = CONTEXT + f'### Human: What is unusual about this image: \n### Assistant: '
- print(requests.post('http://127.0.0.1:5000/v1/completions', json={'prompt': prompt, 'max_tokens': 200, 'stop': ['\n###']}).json())
-```
-script output:
-```Python
-{'results': [{'text': "The unusual aspect of this image is that a man is standing on top of a yellow minivan while doing his laundry. He has set up a makeshift clothes line using the car's rooftop as an outdoor drying area. This scene is uncommon because people typically do their laundry indoors, in a dedicated space like a laundromat or a room in their home, rather than on top of a moving vehicle. Additionally, hanging clothes on the car could be potentially hazardous or illegal in some jurisdictions due to the risk of damaging the vehicle or causing accidents on the road.\n##"}]}
-```
-
-## For pipeline developers/technical description
-see [DOCS.md](https://github.com/oobabooga/text-generation-webui/blob/main/extensions/multimodal/DOCS.md)
diff --git a/extensions/multimodal/abstract_pipeline.py b/extensions/multimodal/abstract_pipeline.py
deleted file mode 100644
index 9c49935a..00000000
--- a/extensions/multimodal/abstract_pipeline.py
+++ /dev/null
@@ -1,63 +0,0 @@
-from abc import ABC, abstractmethod
-from typing import List, Optional
-
-import torch
-from PIL import Image
-from transformers import is_torch_xpu_available
-
-
-class AbstractMultimodalPipeline(ABC):
- @staticmethod
- @abstractmethod
- def name() -> str:
- 'name of the pipeline, should be same as in --multimodal-pipeline'
- pass
-
- @staticmethod
- @abstractmethod
- def image_start() -> Optional[str]:
- 'return image start string, string representation of image start token, or None if not applicable'
- pass
-
- @staticmethod
- @abstractmethod
- def image_end() -> Optional[str]:
- 'return image end string, string representation of image end token, or None if not applicable'
- pass
-
- @staticmethod
- @abstractmethod
- def placeholder_token_id() -> int:
- 'return placeholder token id'
- pass
-
- @staticmethod
- @abstractmethod
- def num_image_embeds() -> int:
- 'return the number of embeds used by a single image (for example: 256 for LLaVA)'
- pass
-
- @abstractmethod
- def embed_images(self, images: List[Image.Image]) -> torch.Tensor:
- 'forward the images through vision pipeline, and return their embeddings'
- pass
-
- @staticmethod
- @abstractmethod
- def embed_tokens(input_ids: torch.Tensor) -> torch.Tensor:
- 'embed tokens, the exact function varies by LLM, for LLaMA it is `shared.model.model.embed_tokens`'
- pass
-
- @staticmethod
- @abstractmethod
- def placeholder_embeddings() -> torch.Tensor:
- 'get placeholder embeddings if there are multiple images, and `add_all_images_to_prompt` is False'
- pass
-
- def _get_device(self, setting_name: str, params: dict):
- if params[setting_name] is None:
- return torch.device("cuda:0" if torch.cuda.is_available() else "xpu:0" if is_torch_xpu_available() else "cpu")
- return torch.device(params[setting_name])
-
- def _get_dtype(self, setting_name: str, params: dict):
- return torch.float32 if int(params[setting_name]) == 32 else torch.float16
diff --git a/extensions/multimodal/multimodal_embedder.py b/extensions/multimodal/multimodal_embedder.py
deleted file mode 100644
index 626077cb..00000000
--- a/extensions/multimodal/multimodal_embedder.py
+++ /dev/null
@@ -1,178 +0,0 @@
-import base64
-import re
-from dataclasses import dataclass
-from io import BytesIO
-from typing import Any, List, Optional
-
-import torch
-from PIL import Image
-
-from extensions.multimodal.pipeline_loader import load_pipeline
-from modules import shared
-from modules.logging_colors import logger
-from modules.text_generation import encode, get_max_prompt_length
-
-
-@dataclass
-class PromptPart:
- text: str
- image: Optional[Image.Image] = None
- is_image: bool = False
- input_ids: Optional[torch.Tensor] = None
- embedding: Optional[torch.Tensor] = None
-
-
-class MultimodalEmbedder:
- def __init__(self, params: dict):
- pipeline, source = load_pipeline(params)
- self.pipeline = pipeline
- logger.info(f'Multimodal: loaded pipeline {self.pipeline.name()} from pipelines/{source} ({self.pipeline.__class__.__name__})')
-
- def _split_prompt(self, prompt: str, load_images: bool = False) -> List[PromptPart]:
- """Splits a prompt into a list of `PromptParts` to separate image data from text.
- It will also append `image_start` and `image_end` before and after the image, and optionally parse and load the images,
- if `load_images` is `True`.
- """
- parts: List[PromptPart] = []
- curr = 0
- while True:
- match = re.search(r')
', prompt[curr:])
- if match is None:
- # no more image tokens, append the rest of the prompt
- if curr > 0:
- # add image end token after last image
- parts.append(PromptPart(text=self.pipeline.image_end() + prompt[curr:]))
- else:
- parts.append(PromptPart(text=prompt))
- break
- # found an image, append image start token to the text
- if match.start() > 0:
- parts.append(PromptPart(text=prompt[curr:curr + match.start()] + self.pipeline.image_start()))
- else:
- parts.append(PromptPart(text=self.pipeline.image_start()))
- # append the image
- parts.append(PromptPart(
- text=match.group(0),
- image=Image.open(BytesIO(base64.b64decode(match.group(1)))) if load_images else None,
- is_image=True
- ))
- curr += match.end()
- return parts
-
- def _len_in_tokens_prompt_parts(self, parts: List[PromptPart]) -> int:
- """Total length in tokens of all `parts`"""
- tokens = 0
- for part in parts:
- if part.is_image:
- tokens += self.pipeline.num_image_embeds()
- elif part.input_ids is not None:
- tokens += len(part.input_ids)
- else:
- tokens += len(encode(part.text)[0])
- return tokens
-
- def len_in_tokens(self, prompt: str) -> int:
- """Total length in tokens for a given text `prompt`"""
- parts = self._split_prompt(prompt, False)
- return self._len_in_tokens_prompt_parts(parts)
-
- def _encode_single_text(self, part: PromptPart, add_bos_token: bool) -> PromptPart:
- """Encode a single prompt `part` to `input_ids`. Returns a `PromptPart`"""
- if part.is_image:
- placeholders = torch.ones((self.pipeline.num_image_embeds())) * self.pipeline.placeholder_token_id()
- part.input_ids = placeholders.to(shared.model.device, dtype=torch.int64)
- else:
- part.input_ids = encode(part.text, add_bos_token=add_bos_token)[0].to(shared.model.device, dtype=torch.int64)
- return part
-
- @staticmethod
- def _num_images(parts: List[PromptPart]) -> int:
- count = 0
- for part in parts:
- if part.is_image:
- count += 1
- return count
-
- def _encode_text(self, state, parts: List[PromptPart]) -> List[PromptPart]:
- """Encode text to token_ids, also truncate the prompt, if necessary.
-
- The chat/instruct mode should make prompts that fit in get_max_prompt_length, but if max_new_tokens are set
- such that the context + min_rows don't fit, we can get a prompt which is too long.
- We can't truncate image embeddings, as it leads to broken generation, so remove the images instead and warn the user
- """
- encoded: List[PromptPart] = []
- for i, part in enumerate(parts):
- encoded.append(self._encode_single_text(part, i == 0 and state['add_bos_token']))
-
- # truncation:
- max_len = get_max_prompt_length(state)
- removed_images = 0
-
- # 1. remove entire text/image blocks
- while self._len_in_tokens_prompt_parts(encoded[1:]) > max_len:
- if encoded[0].is_image:
- removed_images += 1
- encoded = encoded[1:]
-
- # 2. check if the last prompt part doesn't need to get truncated
- if self._len_in_tokens_prompt_parts(encoded) > max_len:
- if encoded[0].is_image:
- # don't truncate image embeddings, just remove the image, otherwise generation will be broken
- removed_images += 1
- encoded = encoded[1:]
- elif len(encoded) > 1 and encoded[0].text.endswith(self.pipeline.image_start()):
- # see if we can keep image_start token
- len_image_start = len(encode(self.pipeline.image_start(), add_bos_token=state['add_bos_token'])[0])
- if self._len_in_tokens_prompt_parts(encoded[1:]) + len_image_start > max_len:
- # we can't -> remove this text, and the image
- encoded = encoded[2:]
- removed_images += 1
- else:
- # we can -> just truncate the text
- trunc_len = self._len_in_tokens_prompt_parts(encoded) - max_len
- encoded[0].input_ids = encoded[0].input_ids[trunc_len:]
- elif len(encoded) > 0:
- # only one text left, truncate it normally
- trunc_len = self._len_in_tokens_prompt_parts(encoded) - max_len
- encoded[0].input_ids = encoded[0].input_ids[trunc_len:]
-
- # notify user if we truncated an image
- if removed_images > 0:
- logger.warning(f"Multimodal: removed {removed_images} image(s) from prompt. Try decreasing max_new_tokens if generation is broken")
-
- return encoded
-
- def _embed(self, parts: List[PromptPart]) -> List[PromptPart]:
- # batch images
- image_indicies = [i for i, part in enumerate(parts) if part.is_image]
- embedded = self.pipeline.embed_images([parts[i].image for i in image_indicies])
- for i, embeds in zip(image_indicies, embedded):
- parts[i].embedding = embeds
- # embed text
- for (i, part) in enumerate(parts):
- if not part.is_image:
- parts[i].embedding = self.pipeline.embed_tokens(part.input_ids)
- return parts
-
- def _remove_old_images(self, parts: List[PromptPart], params: dict) -> List[PromptPart]:
- if params['add_all_images_to_prompt']:
- return parts
- already_added = False
- for i, part in reversed(list(enumerate(parts))):
- if part.is_image:
- if already_added:
- parts[i].embedding = self.pipeline.placeholder_embeddings()
- else:
- already_added = True
- return parts
-
- def forward(self, prompt: str, state: Any, params: dict):
- prompt_parts = self._split_prompt(prompt, True)
- prompt_parts = self._encode_text(state, prompt_parts)
- prompt_parts = self._embed(prompt_parts)
- prompt_parts = self._remove_old_images(prompt_parts, params)
- embeds = tuple(part.embedding for part in prompt_parts)
- ids = tuple(part.input_ids for part in prompt_parts)
- input_embeds = torch.cat(embeds, dim=0)
- input_ids = torch.cat(ids, dim=0)
- return prompt, input_ids, input_embeds, self._num_images(prompt_parts)
diff --git a/extensions/multimodal/pipeline_loader.py b/extensions/multimodal/pipeline_loader.py
deleted file mode 100644
index 8fcd0a9b..00000000
--- a/extensions/multimodal/pipeline_loader.py
+++ /dev/null
@@ -1,52 +0,0 @@
-import traceback
-from importlib import import_module
-from pathlib import Path
-from typing import Tuple
-
-from extensions.multimodal.abstract_pipeline import AbstractMultimodalPipeline
-from modules import shared
-from modules.logging_colors import logger
-
-
-def _get_available_pipeline_modules():
- pipeline_path = Path(__file__).parent / 'pipelines'
- modules = [p for p in pipeline_path.iterdir() if p.is_dir()]
- return [m.name for m in modules if (m / 'pipelines.py').exists()]
-
-
-def load_pipeline(params: dict) -> Tuple[AbstractMultimodalPipeline, str]:
- pipeline_modules = {}
- available_pipeline_modules = _get_available_pipeline_modules()
- for name in available_pipeline_modules:
- try:
- pipeline_modules[name] = import_module(f'extensions.multimodal.pipelines.{name}.pipelines')
- except:
- logger.warning(f'Failed to get multimodal pipelines from {name}')
- logger.warning(traceback.format_exc())
-
- if shared.args.multimodal_pipeline is not None:
- for k in pipeline_modules:
- if hasattr(pipeline_modules[k], 'get_pipeline'):
- pipeline = getattr(pipeline_modules[k], 'get_pipeline')(shared.args.multimodal_pipeline, params)
- if pipeline is not None:
- return (pipeline, k)
- else:
- model_name = shared.args.model.lower()
- for k in pipeline_modules:
- if hasattr(pipeline_modules[k], 'get_pipeline_from_model_name'):
- pipeline = getattr(pipeline_modules[k], 'get_pipeline_from_model_name')(model_name, params)
- if pipeline is not None:
- return (pipeline, k)
-
- available = []
- for k in pipeline_modules:
- if hasattr(pipeline_modules[k], 'available_pipelines'):
- pipelines = getattr(pipeline_modules[k], 'available_pipelines')
- available += pipelines
-
- if shared.args.multimodal_pipeline is not None:
- log = f'Multimodal - ERROR: Failed to load multimodal pipeline "{shared.args.multimodal_pipeline}", available pipelines are: {available}.'
- else:
- log = f'Multimodal - ERROR: Failed to determine multimodal pipeline for model {shared.args.model}, please select one manually using --multimodal-pipeline [PIPELINE]. Available pipelines are: {available}.'
- logger.critical(f'{log} Please specify a correct pipeline, or disable the extension')
- raise RuntimeError(f'{log} Please specify a correct pipeline, or disable the extension')
diff --git a/extensions/multimodal/pipelines/llava/README.md b/extensions/multimodal/pipelines/llava/README.md
deleted file mode 100644
index aff64faa..00000000
--- a/extensions/multimodal/pipelines/llava/README.md
+++ /dev/null
@@ -1,9 +0,0 @@
-## LLaVA pipeline
-
-This module provides 2 pipelines:
-- `llava-7b` - for use with LLaVA v0 7B model (finetuned LLaMa 7B)
-- `llava-13b` - for use with LLaVA v0 13B model (finetuned LLaMa 13B)
-
-[LLaVA](https://github.com/haotian-liu/LLaVA) uses CLIP `openai/clip-vit-large-patch14` as the vision model, and then a single linear layer. For 13B the projector weights are in `liuhaotian/LLaVA-13b-delta-v0`, and for 7B they are in `liuhaotian/LLaVA-7b-delta-v0`.
-
-The supported parameter combinations for both the vision model, and the projector are: CUDA/32bit, CUDA/16bit, CPU/32bit
diff --git a/extensions/multimodal/pipelines/llava/llava.py b/extensions/multimodal/pipelines/llava/llava.py
deleted file mode 100644
index 09b5aff7..00000000
--- a/extensions/multimodal/pipelines/llava/llava.py
+++ /dev/null
@@ -1,262 +0,0 @@
-import time
-from abc import abstractmethod
-from typing import List, Tuple
-
-import torch
-from huggingface_hub import hf_hub_download
-from PIL import Image
-from transformers import CLIPImageProcessor, CLIPVisionModel
-
-from extensions.multimodal.abstract_pipeline import AbstractMultimodalPipeline
-from modules import shared
-from modules.logging_colors import logger
-from modules.text_generation import encode
-
-
-def expand2square(pil_img: Image.Image, background_color: Tuple[int]) -> Image.Image:
- width, height = pil_img.size
- if width == height:
- return pil_img
- elif width > height:
- result = Image.new(pil_img.mode, (width, width), background_color)
- result.paste(pil_img, (0, (width - height) // 2))
- return result
- else:
- result = Image.new(pil_img.mode, (height, height), background_color)
- result.paste(pil_img, ((height - width) // 2, 0))
- return result
-
-
-class LLaVA_v0_Pipeline(AbstractMultimodalPipeline):
- CLIP_REPO = "openai/clip-vit-large-patch14"
-
- def __init__(self, params: dict) -> None:
- super().__init__()
- self.clip_device = self._get_device("vision_device", params)
- self.clip_dtype = self._get_dtype("vision_bits", params)
- self.projector_device = self._get_device("projector_device", params)
- self.projector_dtype = self._get_dtype("projector_bits", params)
- self.image_processor, self.vision_tower, self.mm_projector = self._load_models()
-
- def _load_models(self):
- start_ts = time.time()
-
- logger.info(f"LLaVA - Loading CLIP from {self.CLIP_REPO} as {self.clip_dtype} on {self.clip_device}...")
- image_processor = CLIPImageProcessor.from_pretrained(self.CLIP_REPO, torch_dtype=self.clip_dtype)
- vision_tower = CLIPVisionModel.from_pretrained(self.CLIP_REPO, torch_dtype=self.clip_dtype).to(self.clip_device)
-
- logger.info(f"LLaVA - Loading projector from {self.llava_projector_repo()} as {self.projector_dtype} on {self.projector_device}...")
- projector_path = hf_hub_download(self.llava_projector_repo(), self.llava_projector_filename())
- mm_projector = self.build_mm_projector()
- projector_data = torch.load(projector_path)
- projector_data = {k[19:]: v for k, v in projector_data.items() if k.startswith('model.mm_projector.')}
- mm_projector.load_state_dict(projector_data)
- mm_projector = mm_projector.to(self.projector_device)
-
- logger.info(f"LLaVA supporting models loaded, took {time.time() - start_ts:.2f} seconds")
- return image_processor, vision_tower, mm_projector
-
- def build_mm_projector(self) -> torch.nn.Module:
- projector_shape = self.llava_projector_shape()
- if len(projector_shape) == 2:
- return torch.nn.Linear(*projector_shape)
- else:
- modules = []
- modules.append(torch.nn.Linear(projector_shape[0], projector_shape[1]))
- for i in range(2, len(projector_shape)):
- modules.append(torch.nn.GELU())
- modules.append(torch.nn.Linear(projector_shape[i-1], projector_shape[i]))
- return torch.nn.Sequential(*modules)
-
- @staticmethod
- def image_start() -> str:
- return "'
-
- if '
', prompt)
-
- if image_match is None:
- return prompt, input_ids, input_embeds
-
- prompt, input_ids, input_embeds, total_embedded = multimodal_embedder.forward(prompt, state, params)
- logger.info(f'Embedded {total_embedded} image(s) in {time.time()-start_ts:.2f}s')
- return (prompt,
- input_ids.unsqueeze(0).to(shared.model.device, dtype=torch.int64),
- input_embeds.unsqueeze(0).to(shared.model.device, dtype=shared.model.dtype))
-
-
-def ui():
- global multimodal_embedder
- multimodal_embedder = MultimodalEmbedder(params)
- with gr.Column():
- picture_select = gr.Image(label='Send a picture', type='pil')
- # The models don't seem to deal well with multiple images
- single_image_checkbox = gr.Checkbox(False, label='Embed all images, not only the last one')
- # Prepare the input hijack
- picture_select.upload(
- lambda picture: input_hijack.update({"state": True, "value": partial(add_chat_picture, picture)}),
- [picture_select],
- None
- )
- picture_select.clear(lambda: input_hijack.update({"state": False, "value": ["", ""]}), None, None)
- single_image_checkbox.change(lambda x: params.update({"add_all_images_to_prompt": x}), single_image_checkbox, None)
- shared.gradio['Generate'].click(lambda: None, None, picture_select)
- shared.gradio['textbox'].submit(lambda: None, None, picture_select)
diff --git a/extensions/openai/logits.py b/extensions/openai/logits.py
index 357e70fa..280612db 100644
--- a/extensions/openai/logits.py
+++ b/extensions/openai/logits.py
@@ -5,7 +5,5 @@ from modules.logits import get_next_logits
def _get_next_logits(body):
# Pre-process the input payload to simulate a real generation
use_samplers = body['use_samplers']
- state = process_parameters(body) if use_samplers else {}
- state['stream'] = True
-
+ state = process_parameters(body)
return get_next_logits(body['prompt'], state, use_samplers, "", top_logits=body['top_logits'], return_dict=True)
diff --git a/modules/cache_utils.py b/modules/cache_utils.py
deleted file mode 100644
index 0d1368a2..00000000
--- a/modules/cache_utils.py
+++ /dev/null
@@ -1,115 +0,0 @@
-import torch
-from numba import njit
-
-from modules import shared
-
-
-def process_llamacpp_cache(model, new_sequence, past_sequence):
- if len(past_sequence) == 0 or len(new_sequence) == 0:
- return past_sequence
-
- i1, i2, j1, j2 = find_longest_common_substring_indices(past_sequence, new_sequence)
- overlap_length = i2 - i1 + 1
-
- # Do StreamingLLM if i1 > 0 (ie the longest common subsequence is not a prefix)
- # and the overlap length is sufficiently long.
- if i1 > 0 and overlap_length > 0.2 * len(new_sequence):
-
- new_sequence = torch.tensor(new_sequence)
- past_sequence = torch.tensor(past_sequence)
-
- prefix_length = find_prefix_length(past_sequence[:i1], new_sequence[:j1])
- sink_length = max(prefix_length, shared.args.attention_sink_size)
- removed_length = i1 - sink_length
-
- if removed_length <= 0:
- return past_sequence.tolist()
-
- matching_prefix = past_sequence[:prefix_length]
- removed_chunk = past_sequence[sink_length:i1]
- overlapping_sequence = new_sequence[j1:j2 + 1]
- added_chunk = new_sequence[j2 + 1:]
-
- # print(past_sequence.tolist())
- # print(new_sequence.tolist())
-
- print()
- print('MATCHING PREFIX=', repr(shared.tokenizer.decode(matching_prefix)))
- print('ADDED CHUNK=', repr(shared.tokenizer.decode(added_chunk)))
- print('REMOVED CHUNK=', repr(shared.tokenizer.decode(removed_chunk)))
- print('REMOVED LENGTH=', removed_length)
- print()
-
- # Remove interval [sink_length, sink_length + removed_length) from the context
- # Update model.n_tokens
- model._ctx.kv_cache_seq_rm(0, sink_length, sink_length + removed_length)
- model._ctx.kv_cache_seq_shift(0, sink_length + removed_length, -1, -removed_length)
-
- new_sequence = new_sequence.tolist()
- model.input_ids[:j2 + 1] = new_sequence[:j2 + 1]
- model.n_tokens = j2 + 1
-
- return new_sequence[:j2 + 1]
- else:
- return past_sequence
-
-
-def find_prefix_length(past_seq, seq_tensor):
- '''
- Given two torch tensors, finds the length of the longest
- common prefix between the two.
- '''
- min_length = min(past_seq.shape[0], seq_tensor.shape[0])
- indices = torch.nonzero(~torch.eq(past_seq[:min_length], seq_tensor[:min_length]))
- if len(indices) > 0:
- prefix_length = indices[0].item()
- else:
- prefix_length = min_length
-
- return prefix_length
-
-
-@njit
-def find_longest_common_substring_indices(list1, list2):
- '''
- Given two lists, solves the Longest Common Substring problem.
-
- It returns the indices where the substring starts and ends in
- s1 and s2.
-
- Example:
-
- ir, jr, ir2, jr2 = find_longest_common_substring_indices(s1, s2)
- print(s1[ir:jr + 1])
- print(s2[ir2:jr2 + 1])
-
- Adapted from
- https://rosettacode.org/wiki/Longest_common_substring#Python
- '''
-
- len_list1, len_list2 = len(list1), len(list2)
- start_index_list1, end_index_list1 = 0, -1
- start_index_list2, end_index_list2 = 0, -1
-
- # for index1 in tqdm(range(0, len_list1), desc="StreamingLLM prompt comparison", leave=False):
- for index1 in range(0, len_list1):
- try:
- index2 = list2.index(list1[index1])
- except:
- continue
-
- while index2 >= 0:
- temp_index1, temp_index2 = index1, index2
- while temp_index1 < len_list1 and temp_index2 < len_list2 and list2[temp_index2] == list1[temp_index1]:
- if temp_index1 - index1 >= end_index_list1 - start_index_list1:
- start_index_list1, end_index_list1 = index1, temp_index1
- start_index_list2, end_index_list2 = index2, temp_index2
-
- temp_index1 += 1
- temp_index2 += 1
- try:
- index2 = list2.index(list1[index1], index2 + 1)
- except:
- break
-
- return start_index_list1, end_index_list1, start_index_list2, end_index_list2
diff --git a/modules/evaluate.py b/modules/evaluate.py
index 35c72689..dd7ef9a7 100644
--- a/modules/evaluate.py
+++ b/modules/evaluate.py
@@ -40,17 +40,13 @@ def calculate_perplexity(models, input_dataset, stride, _max_length):
'''
if shared.args.loader == "llama.cpp":
- logger.error("llamacpp_HF is required for perplexity evaluation with GGUF models. Please reload the model with llamacpp_HF instead of llama.cpp.")
+ logger.error("Perplexity evaluation is not implemented for the llama.cpp loader.")
raise ValueError
if shared.args.loader == "ExLlamav2":
logger.error("ExLlamav2_HF is required for perplexity evaluation with EXL2 models. Please reload the model with ExLlamav2_HF instead of ExLlamav2.")
raise ValueError
- if shared.args.loader == "llamacpp_HF" and not shared.args.logits_all:
- logger.error("--logits_all is required for perplexity evaluation with GGUF models. Please reload the model with that option set/checked.")
- raise ValueError
-
if not shared.args.no_use_fast:
logger.warning("--no_use_fast is not set. If tokenizing the input dataset takes a long time, try reloading the model with that option set/checked.")
diff --git a/modules/exllamav2_hf.py b/modules/exllamav2_hf.py
index 62d1e054..6486e438 100644
--- a/modules/exllamav2_hf.py
+++ b/modules/exllamav2_hf.py
@@ -157,6 +157,9 @@ class Exllamav2HF(PreTrainedModel):
else:
self.past_seq = seq_tensor
+ if torch.cuda.is_available():
+ torch.cuda.synchronize()
+
loss = None
if labels is not None:
# Shift so that tokens < n predict n
diff --git a/modules/exllamav3_hf.py b/modules/exllamav3_hf.py
index 3bf44c9b..0f742fa2 100644
--- a/modules/exllamav3_hf.py
+++ b/modules/exllamav3_hf.py
@@ -153,6 +153,9 @@ class Exllamav3HF(PreTrainedModel):
else:
self.past_seq = seq_tensor
+ if torch.cuda.is_available():
+ torch.cuda.synchronize()
+
loss = None
if labels is not None:
# Shift so that tokens < n predict n
diff --git a/modules/html_generator.py b/modules/html_generator.py
index 6bad0f89..144f2593 100644
--- a/modules/html_generator.py
+++ b/modules/html_generator.py
@@ -391,8 +391,10 @@ def generate_chat_html(history, name1, name2, reset_cache=False):
def chat_html_wrapper(history, name1, name2, mode, style, character, reset_cache=False):
if mode == 'instruct':
- return generate_instruct_html(history)
+ result = generate_instruct_html(history)
elif style == 'wpp':
- return generate_chat_html(history, name1, name2)
+ result = generate_chat_html(history, name1, name2)
else:
- return generate_cai_chat_html(history, name1, name2, style, character, reset_cache)
+ result = generate_cai_chat_html(history, name1, name2, style, character, reset_cache)
+
+ return {'html': result}
diff --git a/modules/llama_cpp_python_hijack.py b/modules/llama_cpp_python_hijack.py
deleted file mode 100644
index 8572cd81..00000000
--- a/modules/llama_cpp_python_hijack.py
+++ /dev/null
@@ -1,165 +0,0 @@
-import importlib
-import platform
-from typing import Sequence
-
-import numpy as np
-from tqdm import tqdm
-
-from modules import shared
-from modules.cache_utils import process_llamacpp_cache
-
-imported_module = None
-not_available_modules = set()
-
-
-def llama_cpp_lib():
- global imported_module, not_available_modules
-
- # Determine the platform
- is_macos = platform.system() == 'Darwin'
-
- # Define the library names based on the platform
- if is_macos:
- lib_names = [
- (None, 'llama_cpp')
- ]
- else:
- lib_names = [
- ('cpu', 'llama_cpp'),
- ('tensorcores', 'llama_cpp_cuda_tensorcores'),
- (None, 'llama_cpp_cuda'),
- (None, 'llama_cpp')
- ]
-
- for arg, lib_name in lib_names:
- if lib_name in not_available_modules:
- continue
-
- should_import = (arg is None or getattr(shared.args, arg))
-
- if should_import:
- if imported_module and imported_module != lib_name:
- # Conflict detected, raise an exception
- raise Exception(f"Cannot import `{lib_name}` because `{imported_module}` is already imported. Switching to a different version of llama-cpp-python currently requires a server restart.")
-
- try:
- return_lib = importlib.import_module(lib_name)
- imported_module = lib_name
- monkey_patch_llama_cpp_python(return_lib)
- return return_lib
- except ImportError:
- not_available_modules.add(lib_name)
- continue
-
- return None
-
-
-def eval_with_progress(self, tokens: Sequence[int]):
- """
- A copy of
-
- https://github.com/abetlen/llama-cpp-python/blob/main/llama_cpp/llama.py
-
- with tqdm to show prompt processing progress.
- """
- self._ctx.kv_cache_seq_rm(-1, self.n_tokens, -1)
-
- if len(tokens) > self.n_batch:
- progress_bar = tqdm(range(0, len(tokens), self.n_batch), desc="Prompt evaluation", leave=False)
- else:
- progress_bar = range(0, len(tokens), self.n_batch)
-
- for i in progress_bar:
- batch = tokens[i : min(len(tokens), i + self.n_batch)]
- n_past = self.n_tokens
- n_tokens = len(batch)
- self._batch.set_batch(
- batch=batch, n_past=n_past, logits_all=self.context_params.logits_all
- )
- self._ctx.decode(self._batch)
- # Save tokens
- self.input_ids[n_past : n_past + n_tokens] = batch
- # Save logits
- if self.context_params.logits_all:
- rows = n_tokens
- cols = self._n_vocab
- logits = np.ctypeslib.as_array(
- self._ctx.get_logits(), shape=(rows * cols,)
- )
- self.scores[n_past : n_past + n_tokens, :].reshape(-1)[::] = logits
- self.last_updated_index = n_past + n_tokens - 1
- else:
- rows = 1
- cols = self._n_vocab
- logits = np.ctypeslib.as_array(
- self._ctx.get_logits(), shape=(rows * cols,)
- )
- last_token_index = min(n_past + n_tokens - 1, self.scores.shape[0] - 1)
- self.scores[last_token_index, :] = logits.reshape(-1)
- self.last_updated_index = last_token_index
- # Update n_tokens
- self.n_tokens += n_tokens
-
-
-def monkey_patch_llama_cpp_python(lib):
- if getattr(lib.Llama, '_is_patched', False):
- # If the patch is already applied, do nothing
- return
-
- def my_generate(self, *args, **kwargs):
- if shared.args.streaming_llm:
- new_sequence = args[0]
- past_sequence = self._input_ids
-
- # Do the cache trimming for StreamingLLM
- process_llamacpp_cache(self, new_sequence, past_sequence)
-
- for output in self.original_generate(*args, **kwargs):
- yield output
-
- lib.Llama.eval = eval_with_progress
- lib.Llama.original_generate = lib.Llama.generate
- lib.Llama.generate = my_generate
-
- # Also patch Jinja2ChatFormatter to handle loop controls
- if hasattr(lib, 'llama_chat_format') and hasattr(lib.llama_chat_format, 'Jinja2ChatFormatter'):
- Formatter = lib.llama_chat_format.Jinja2ChatFormatter
-
- if not getattr(Formatter, '_is_patched', False):
- def patched_init(self, *args, **kwargs):
- # Extract parameters from args or kwargs
- if args:
- self.template = args[0]
- self.eos_token = args[1] if len(args) > 1 else kwargs.get('eos_token')
- self.bos_token = args[2] if len(args) > 2 else kwargs.get('bos_token')
- self.add_generation_prompt = args[3] if len(args) > 3 else kwargs.get('add_generation_prompt', True)
- self.stop_token_ids = args[4] if len(args) > 4 else kwargs.get('stop_token_ids')
- else:
- self.template = kwargs.get('template')
- self.eos_token = kwargs.get('eos_token')
- self.bos_token = kwargs.get('bos_token')
- self.add_generation_prompt = kwargs.get('add_generation_prompt', True)
- self.stop_token_ids = kwargs.get('stop_token_ids')
-
- # Process stop tokens as in the original
- self.stop_token_ids = (
- set(self.stop_token_ids) if self.stop_token_ids is not None else None
- )
-
- # Create environment with loopcontrols extension
- import jinja2
- from jinja2.ext import loopcontrols
-
- self._environment = jinja2.sandbox.ImmutableSandboxedEnvironment(
- loader=jinja2.BaseLoader(),
- trim_blocks=True,
- lstrip_blocks=True,
- extensions=[loopcontrols]
- ).from_string(self.template)
-
- # Replace the original __init__ with our patched version
- Formatter.__init__ = patched_init
- Formatter._is_patched = True
-
- # Set the flag to indicate that the patch has been applied
- lib.Llama._is_patched = True
diff --git a/modules/llama_cpp_server.py b/modules/llama_cpp_server.py
new file mode 100644
index 00000000..db500c9c
--- /dev/null
+++ b/modules/llama_cpp_server.py
@@ -0,0 +1,339 @@
+import json
+import pprint
+import socket
+import subprocess
+import sys
+import threading
+import time
+
+import llama_cpp_binaries
+import requests
+
+from modules import shared
+from modules.logging_colors import logger
+
+llamacpp_valid_cache_types = {"fp16", "q8_0", "q4_0"}
+
+
+class LlamaServer:
+ def __init__(self, model_path, server_path=None):
+ """
+ Initialize and start a server for llama.cpp models.
+ """
+ self.model_path = model_path
+ self.server_path = server_path
+ self.port = self._find_available_port()
+ self.process = None
+ self.vocabulary_size = None
+ self.bos_token = "', elem_id='gr-hover')
@@ -181,7 +181,7 @@ def create_event_handlers():
shared.reload_inputs = gradio(reload_arr)
# Morph HTML updates instead of updating everything
- shared.gradio['display'].change(None, gradio('display'), None, js="(text) => handleMorphdomUpdate(text)")
+ shared.gradio['display'].change(None, gradio('display'), None, js="(data) => handleMorphdomUpdate(data.html)")
shared.gradio['Generate'].click(
ui.gather_interface_values, gradio(shared.input_elements), gradio('interface_state')).then(
diff --git a/modules/ui_model_menu.py b/modules/ui_model_menu.py
index 4fc1de08..8db48621 100644
--- a/modules/ui_model_menu.py
+++ b/modules/ui_model_menu.py
@@ -87,7 +87,7 @@ def create_ui():
shared.gradio['n_gpu_layers'] = gr.Slider(label="n-gpu-layers", minimum=0, maximum=256, value=shared.args.n_gpu_layers, info='Must be greater than 0 for the GPU to be used. ⚠️ Lower this value if you can\'t load the model.')
shared.gradio['threads'] = gr.Slider(label="threads", minimum=0, step=1, maximum=256, value=shared.args.threads)
shared.gradio['threads_batch'] = gr.Slider(label="threads_batch", minimum=0, step=1, maximum=256, value=shared.args.threads_batch)
- shared.gradio['n_batch'] = gr.Slider(label="n_batch", minimum=1, maximum=2048, step=1, value=shared.args.n_batch)
+ shared.gradio['batch_size'] = gr.Slider(label="batch_size", minimum=1, maximum=4096, step=1, value=shared.args.batch_size)
shared.gradio['hqq_backend'] = gr.Dropdown(label="hqq_backend", choices=["PYTORCH", "PYTORCH_COMPILE", "ATEN"], value=shared.args.hqq_backend)
shared.gradio['n_ctx'] = gr.Number(label="n_ctx", precision=0, step=256, value=shared.args.n_ctx, info='Context length. ⚠️ Lower this value if you can\'t load the model. Common values: 2048, 4096, 8192, 16384, 32768.')
shared.gradio['max_seq_len'] = gr.Number(label='max_seq_len', precision=0, step=256, value=shared.args.max_seq_len, info='Context length. ⚠️ Lower this value if you can\'t load the model. Common values: 2048, 4096, 8192, 16384, 32768.')
@@ -99,23 +99,19 @@ def create_ui():
shared.gradio['compress_pos_emb'] = gr.Number(label='compress_pos_emb', value=shared.args.compress_pos_emb, precision=2, info='Positional embeddings compression factor. Should be set to (context length) / (model\'s original context length). Equal to 1/rope_freq_scale.')
shared.gradio['compute_dtype'] = gr.Dropdown(label="compute_dtype", choices=["bfloat16", "float16", "float32"], value=shared.args.compute_dtype, info='Used by load-in-4bit.')
shared.gradio['quant_type'] = gr.Dropdown(label="quant_type", choices=["nf4", "fp4"], value=shared.args.quant_type, info='Used by load-in-4bit.')
- shared.gradio['attention_sink_size'] = gr.Number(label="attention_sink_size", value=shared.args.attention_sink_size, precision=0, info='StreamingLLM: number of sink tokens. Only used if the trimmed prompt doesn\'t share a prefix with the old prompt.')
shared.gradio['num_experts_per_token'] = gr.Number(label="Number of experts per token", value=shared.args.num_experts_per_token, info='Only applies to MoE models like Mixtral.')
with gr.Column():
- shared.gradio['tensorcores'] = gr.Checkbox(label="tensorcores", value=shared.args.tensorcores, info='NVIDIA only: use llama-cpp-python compiled without GGML_CUDA_FORCE_MMQ. This may improve performance on newer cards.')
shared.gradio['load_in_8bit'] = gr.Checkbox(label="load-in-8bit", value=shared.args.load_in_8bit)
shared.gradio['load_in_4bit'] = gr.Checkbox(label="load-in-4bit", value=shared.args.load_in_4bit)
shared.gradio['torch_compile'] = gr.Checkbox(label="torch-compile", value=shared.args.torch_compile, info='Compile the model with torch.compile for improved performance.')
shared.gradio['flash_attn'] = gr.Checkbox(label="flash_attn", value=shared.args.flash_attn, info='Use flash-attention.')
shared.gradio['use_flash_attention_2'] = gr.Checkbox(label="use_flash_attention_2", value=shared.args.use_flash_attention_2, info='Set use_flash_attention_2=True while loading the model.')
- shared.gradio['streaming_llm'] = gr.Checkbox(label="streaming_llm", value=shared.args.streaming_llm, info='(experimental) Activate StreamingLLM to avoid re-evaluating the entire prompt when old messages are removed.')
shared.gradio['auto_devices'] = gr.Checkbox(label="auto-devices", value=shared.args.auto_devices)
shared.gradio['cpu'] = gr.Checkbox(label="cpu", value=shared.args.cpu, info='llama.cpp: Use llama-cpp-python compiled without GPU acceleration. Transformers: use PyTorch in CPU mode.')
shared.gradio['disk'] = gr.Checkbox(label="disk", value=shared.args.disk)
shared.gradio['row_split'] = gr.Checkbox(label="row_split", value=shared.args.row_split, info='Split the model by rows across GPUs. This may improve multi-gpu performance.')
- shared.gradio['no_offload_kqv'] = gr.Checkbox(label="no_offload_kqv", value=shared.args.no_offload_kqv, info='Do not offload the K, Q, V to the GPU. This saves VRAM but reduces the performance.')
- shared.gradio['no_mul_mat_q'] = gr.Checkbox(label="no_mul_mat_q", value=shared.args.no_mul_mat_q, info='Disable the mulmat kernels.')
+ shared.gradio['no_kv_offload'] = gr.Checkbox(label="no_kv_offload", value=shared.args.no_kv_offload, info='Do not offload the K, Q, V to the GPU. This saves VRAM but reduces the performance.')
shared.gradio['no_mmap'] = gr.Checkbox(label="no-mmap", value=shared.args.no_mmap)
shared.gradio['mlock'] = gr.Checkbox(label="mlock", value=shared.args.mlock)
shared.gradio['numa'] = gr.Checkbox(label="numa", value=shared.args.numa, info='NUMA support can help on some systems with non-uniform memory access.')
@@ -129,10 +125,8 @@ def create_ui():
shared.gradio['no_sdpa'] = gr.Checkbox(label="no_sdpa", value=shared.args.no_sdpa)
shared.gradio['cfg_cache'] = gr.Checkbox(label="cfg-cache", value=shared.args.cfg_cache, info='Necessary to use CFG with this loader.')
shared.gradio['cpp_runner'] = gr.Checkbox(label="cpp-runner", value=shared.args.cpp_runner, info='Enable inference with ModelRunnerCpp, which is faster than the default ModelRunner.')
- shared.gradio['logits_all'] = gr.Checkbox(label="logits_all", value=shared.args.logits_all, info='Needs to be set for perplexity evaluation to work with this loader. Otherwise, ignore it, as it makes prompt processing slower.')
shared.gradio['trust_remote_code'] = gr.Checkbox(label="trust-remote-code", value=shared.args.trust_remote_code, info='Set trust_remote_code=True while loading the tokenizer/model. To enable this option, start the web UI with the --trust-remote-code flag.', interactive=shared.args.trust_remote_code)
shared.gradio['no_use_fast'] = gr.Checkbox(label="no_use_fast", value=shared.args.no_use_fast, info='Set use_fast=False while loading the tokenizer.')
- shared.gradio['llamacpp_HF_info'] = gr.Markdown("llamacpp_HF loads llama.cpp as a Transformers model. To use it, you need to place your GGUF in a subfolder of models/ with the necessary tokenizer files.\n\nYou can use the \"llamacpp_HF creator\" menu to do that automatically.")
shared.gradio['exllamav2_info'] = gr.Markdown("ExLlamav2_HF is recommended over ExLlamav2 for better integration with extensions and more consistent sampling behavior across loaders.")
shared.gradio['tensorrt_llm_info'] = gr.Markdown('* TensorRT-LLM has to be installed manually in a separate Python 3.10 environment at the moment. For a guide, consult the description of [this PR](https://github.com/oobabooga/text-generation-webui/pull/5715). \n\n* `max_seq_len` is only used when `cpp-runner` is checked.\n\n* `cpp_runner` does not support streaming at the moment.')
@@ -147,15 +141,6 @@ def create_ui():
shared.gradio['download_model_button'] = gr.Button("Download", variant='primary', interactive=not mu)
shared.gradio['get_file_list'] = gr.Button("Get file list", interactive=not mu)
- with gr.Tab("llamacpp_HF creator"):
- with gr.Row():
- shared.gradio['gguf_menu'] = gr.Dropdown(choices=utils.get_available_ggufs(), value=lambda: shared.model_name, label='Choose your GGUF', elem_classes='slim-dropdown', interactive=not mu)
- ui.create_refresh_button(shared.gradio['gguf_menu'], lambda: None, lambda: {'choices': utils.get_available_ggufs()}, 'refresh-button', interactive=not mu)
-
- shared.gradio['unquantized_url'] = gr.Textbox(label="Enter the URL for the original (unquantized) model", info="Example: https://huggingface.co/lmsys/vicuna-13b-v1.5", max_lines=1)
- shared.gradio['create_llamacpp_hf_button'] = gr.Button("Submit", variant="primary", interactive=not mu)
- gr.Markdown("This will move your gguf file into a subfolder of `models` along with the necessary tokenizer files.")
-
with gr.Tab("Customize instruction template"):
with gr.Row():
shared.gradio['customized_template'] = gr.Dropdown(choices=utils.get_available_instruction_templates(), value='None', label='Select the desired instruction template', elem_classes='slim-dropdown')
@@ -195,7 +180,6 @@ def create_event_handlers():
shared.gradio['download_model_button'].click(download_model_wrapper, gradio('custom_model_menu', 'download_specific_file'), gradio('model_status'), show_progress=True)
shared.gradio['get_file_list'].click(partial(download_model_wrapper, return_links=True), gradio('custom_model_menu', 'download_specific_file'), gradio('model_status'), show_progress=True)
shared.gradio['autoload_model'].change(lambda x: gr.update(visible=not x), gradio('autoload_model'), gradio('load_model'))
- shared.gradio['create_llamacpp_hf_button'].click(create_llamacpp_hf, gradio('gguf_menu', 'unquantized_url'), gradio('model_status'), show_progress=True)
shared.gradio['customized_template_submit'].click(save_instruction_template, gradio('model_menu', 'customized_template'), gradio('model_status'), show_progress=True)
@@ -286,34 +270,11 @@ def download_model_wrapper(repo_id, specific_file, progress=gr.Progress(), retur
yield traceback.format_exc().replace('\n', '\n\n')
-def create_llamacpp_hf(gguf_name, unquantized_url, progress=gr.Progress()):
- try:
- downloader = importlib.import_module("download-model").ModelDownloader()
-
- progress(0.0)
- model, branch = downloader.sanitize_model_and_branch_names(unquantized_url, None)
-
- yield ("Getting the tokenizer files links from Hugging Face")
- links, sha256, is_lora, is_llamacpp = downloader.get_download_links_from_huggingface(model, branch, text_only=True)
- output_folder = Path(shared.args.model_dir) / (re.sub(r'(?i)\.gguf$', '', gguf_name) + "-HF")
-
- yield (f"Downloading tokenizer to `{output_folder}/`")
- downloader.download_model_files(model, branch, links, sha256, output_folder, progress_bar=progress, threads=4, is_llamacpp=False)
-
- # Move the GGUF
- (Path(shared.args.model_dir) / gguf_name).rename(output_folder / gguf_name)
-
- yield (f"Model saved to `{output_folder}/`.\n\nYou can now load it using llamacpp_HF.")
- except:
- progress(1.0)
- yield traceback.format_exc().replace('\n', '\n\n')
-
-
def update_truncation_length(current_length, state):
if 'loader' in state:
if state['loader'].lower().startswith('exllama'):
return state['max_seq_len']
- elif state['loader'] in ['llama.cpp', 'llamacpp_HF']:
+ elif state['loader'] == 'llama.cpp':
return state['n_ctx']
return current_length
diff --git a/modules/ui_session.py b/modules/ui_session.py
index dfb95b83..66386d12 100644
--- a/modules/ui_session.py
+++ b/modules/ui_session.py
@@ -69,10 +69,8 @@ def set_interface_arguments(extensions, bool_active):
def get_boolean_arguments(active=False):
- exclude = shared.deprecated_args
-
cmd_list = vars(shared.args)
- bool_list = sorted([k for k in cmd_list if type(cmd_list[k]) is bool and k not in exclude + ui.list_model_elements()])
+ bool_list = sorted([k for k in cmd_list if type(cmd_list[k]) is bool and k not in ui.list_model_elements()])
bool_active = [k for k in bool_list if vars(shared.args)[k]]
if active:
diff --git a/modules/utils.py b/modules/utils.py
index f4333031..f6be7541 100644
--- a/modules/utils.py
+++ b/modules/utils.py
@@ -73,21 +73,61 @@ def natural_keys(text):
def get_available_models():
- model_list = []
- for item in list(Path(f'{shared.args.model_dir}/').glob('*')):
- if not item.name.endswith(('.txt', '-np', '.pt', '.json', '.yaml', '.py')) and 'llama-tokenizer' not in item.name:
- model_list.append(item.name)
+ # Get all GGUF files
+ gguf_files = get_available_ggufs()
- return ['None'] + sorted(model_list, key=natural_keys)
+ model_dir = Path(shared.args.model_dir)
+
+ # Find top-level directories containing GGUF files
+ dirs_with_gguf = set()
+ for gguf_path in gguf_files:
+ path = Path(gguf_path)
+ if path.parts: # If in a subdirectory
+ dirs_with_gguf.add(path.parts[0]) # Add top-level directory
+
+ # Find directories with safetensors files directly under them
+ dirs_with_safetensors = set()
+ for item in os.listdir(model_dir):
+ item_path = model_dir / item
+ if item_path.is_dir():
+ # Check if there are safetensors files directly under this directory
+ if any(file.lower().endswith(('.safetensors', '.pt')) for file in os.listdir(item_path) if (item_path / file).is_file()):
+ dirs_with_safetensors.add(item)
+
+ # Find valid model directories
+ model_dirs = []
+
+ for item in os.listdir(model_dir):
+ item_path = model_dir / item
+
+ # Skip if not a directory
+ if not item_path.is_dir():
+ continue
+
+ # Include directory if it either:
+ # 1. Doesn't contain GGUF files, OR
+ # 2. Contains both GGUF and safetensors files
+ if item not in dirs_with_gguf or item in dirs_with_safetensors:
+ model_dirs.append(item)
+
+ model_dirs = sorted(model_dirs, key=natural_keys)
+
+ # Combine all models
+ return ['None'] + gguf_files + model_dirs
def get_available_ggufs():
model_list = []
- for item in Path(f'{shared.args.model_dir}/').glob('*'):
- if item.is_file() and item.name.lower().endswith(".gguf"):
- model_list.append(item.name)
+ model_dir = Path(shared.args.model_dir)
- return ['None'] + sorted(model_list, key=natural_keys)
+ for dirpath, _, files in os.walk(model_dir, followlinks=True):
+ for file in files:
+ if file.lower().endswith(".gguf"):
+ model_path = Path(dirpath) / file
+ rel_path = model_path.relative_to(model_dir)
+ model_list.append(str(rel_path))
+
+ return sorted(model_list, key=natural_keys)
def get_available_presets():
diff --git a/one_click.py b/one_click.py
index 9f46a2df..99bdc41e 100644
--- a/one_click.py
+++ b/one_click.py
@@ -26,6 +26,7 @@ LIBSTDCXX_VERSION_LINUX = "12.1.0"
# Environment
script_dir = os.getcwd()
conda_env_path = os.path.join(script_dir, "installer_files", "env")
+state_file = '.installer_state.json'
# Command-line flags
cmd_flags_path = os.path.join(script_dir, "CMD_FLAGS.txt")
@@ -238,6 +239,9 @@ def get_user_choice(question, options_dict):
def install_webui():
+ if os.path.isfile(state_file):
+ os.remove(state_file)
+
# Ask the user for the GPU vendor
if "GPU_CHOICE" in os.environ:
choice = os.environ["GPU_CHOICE"].upper()
@@ -372,7 +376,6 @@ def update_requirements(initial_installation=False, pull=True):
requirements_file = "requirements" + ("_noavx2" if not cpu_has_avx2() else "") + ".txt"
# Load state from JSON file
- state_file = '.installer_state.json'
current_commit = get_current_commit()
wheels_changed = False
if os.path.exists(state_file):
diff --git a/requirements.txt b/requirements.txt
index b9b4ea7a..607efda0 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -31,21 +31,11 @@ flask_cloudflared==0.0.14
sse-starlette==1.6.5
tiktoken
-# llama-cpp-python (CPU only, AVX2)
-https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.3.8+cpuavx2-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
-https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.3.8+cpuavx2-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
-
-# llama-cpp-python (CUDA, with GGML_CUDA_FORCE_MMQ)
-https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.3.8+cu124-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
-https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.3.8+cu124-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
-
-# llama-cpp-python (CUDA, without GGML_CUDA_FORCE_MMQ)
-https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda_tensorcores-0.3.8+cu124-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
-https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda_tensorcores-0.3.8+cu124-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
-
# CUDA wheels
-https://github.com/oobabooga/exllamav3/releases/download/v0.0.1/exllamav3-0.0.1+cu124.torch2.6.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
-https://github.com/oobabooga/exllamav3/releases/download/v0.0.1/exllamav3-0.0.1+cu124.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/textgen-webui/llama_cpp_binaries-0.2.0+cu124-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/textgen-webui/llama_cpp_binaries-0.2.0+cu124-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
+https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a3/exllamav3-0.0.1a3+cu124.torch2.6.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
+https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a3/exllamav3-0.0.1a3+cu124.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
https://github.com/oobabooga/exllamav2/releases/download/v0.2.8/exllamav2-0.2.8+cu124.torch2.6.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
https://github.com/oobabooga/exllamav2/releases/download/v0.2.8/exllamav2-0.2.8+cu124.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
https://github.com/oobabooga/exllamav2/releases/download/v0.2.8/exllamav2-0.2.8-py3-none-any.whl; platform_system == "Linux" and platform_machine != "x86_64"
diff --git a/requirements_amd.txt b/requirements_amd.txt
index 3d24891f..b242d4ad 100644
--- a/requirements_amd.txt
+++ b/requirements_amd.txt
@@ -30,11 +30,7 @@ flask_cloudflared==0.0.14
sse-starlette==1.6.5
tiktoken
-# llama-cpp-python (CPU only, AVX2)
-https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.3.8+cpuavx2-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
-https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.3.8+cpuavx2-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
-
# AMD wheels
-https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/rocm/llama_cpp_python_cuda-0.3.8+rocm6.1.2-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/rocm/llama_cpp_binaries-0.2.0+rocm6.1.2-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
https://github.com/oobabooga/exllamav2/releases/download/v0.2.8/exllamav2-0.2.8+rocm6.1.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
https://github.com/oobabooga/exllamav2/releases/download/v0.2.8/exllamav2-0.2.8-py3-none-any.whl; platform_system != "Darwin" and platform_machine != "x86_64"
diff --git a/requirements_amd_noavx2.txt b/requirements_amd_noavx2.txt
index 057b631d..b6105209 100644
--- a/requirements_amd_noavx2.txt
+++ b/requirements_amd_noavx2.txt
@@ -30,10 +30,7 @@ flask_cloudflared==0.0.14
sse-starlette==1.6.5
tiktoken
-# llama-cpp-python (CPU only, no AVX2)
-https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.3.8+cpuavx-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
-https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.3.8+cpuavx-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
-
# AMD wheels
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/rocm/llama_cpp_binaries-0.2.0+rocm6.1.2avx-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
https://github.com/oobabooga/exllamav2/releases/download/v0.2.8/exllamav2-0.2.8+rocm6.1.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
https://github.com/oobabooga/exllamav2/releases/download/v0.2.8/exllamav2-0.2.8-py3-none-any.whl; platform_system != "Darwin" and platform_machine != "x86_64"
diff --git a/requirements_apple_intel.txt b/requirements_apple_intel.txt
index eba21ec2..ce730f63 100644
--- a/requirements_apple_intel.txt
+++ b/requirements_apple_intel.txt
@@ -31,7 +31,7 @@ sse-starlette==1.6.5
tiktoken
# Mac wheels
-https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/metal/llama_cpp_python-0.3.8-cp311-cp311-macosx_15_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "24.0.0" and platform_release < "25.0.0" and python_version == "3.11"
-https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/metal/llama_cpp_python-0.3.8-cp311-cp311-macosx_14_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" and python_version == "3.11"
-https://github.com/oobabooga/exllamav3/releases/download/v0.0.1/exllamav3-0.0.1-py3-none-any.whl
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/metal/llama_cpp_binaries-0.2.0-cp311-cp311-macosx_15_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "24.0.0" and platform_release < "25.0.0" and python_version == "3.11"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/metal/llama_cpp_binaries-0.2.0-cp311-cp311-macosx_14_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" and python_version == "3.11"
+https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a3/exllamav3-0.0.1a3-py3-none-any.whl
https://github.com/oobabooga/exllamav2/releases/download/v0.2.8/exllamav2-0.2.8-py3-none-any.whl
diff --git a/requirements_apple_silicon.txt b/requirements_apple_silicon.txt
index 2048c99b..a7be282d 100644
--- a/requirements_apple_silicon.txt
+++ b/requirements_apple_silicon.txt
@@ -31,8 +31,8 @@ sse-starlette==1.6.5
tiktoken
# Mac wheels
-https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/metal/llama_cpp_python-0.3.8-cp311-cp311-macosx_15_0_arm64.whl; platform_system == "Darwin" and platform_release >= "24.0.0" and platform_release < "25.0.0" and python_version == "3.11"
-https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/metal/llama_cpp_python-0.3.8-cp311-cp311-macosx_14_0_arm64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" and python_version == "3.11"
-https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/metal/llama_cpp_python-0.3.8-cp311-cp311-macosx_13_0_arm64.whl; platform_system == "Darwin" and platform_release >= "22.0.0" and platform_release < "23.0.0" and python_version == "3.11"
-https://github.com/oobabooga/exllamav3/releases/download/v0.0.1/exllamav3-0.0.1-py3-none-any.whl
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/metal/llama_cpp_binaries-0.2.0-cp311-cp311-macosx_15_0_arm64.whl; platform_system == "Darwin" and platform_release >= "24.0.0" and platform_release < "25.0.0" and python_version == "3.11"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/metal/llama_cpp_binaries-0.2.0-cp311-cp311-macosx_14_0_arm64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" and python_version == "3.11"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/metal/llama_cpp_binaries-0.2.0-cp311-cp311-macosx_13_0_arm64.whl; platform_system == "Darwin" and platform_release >= "22.0.0" and platform_release < "23.0.0" and python_version == "3.11"
+https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a3/exllamav3-0.0.1a3-py3-none-any.whl
https://github.com/oobabooga/exllamav2/releases/download/v0.2.8/exllamav2-0.2.8-py3-none-any.whl
diff --git a/requirements_cpu_only.txt b/requirements_cpu_only.txt
index c7e2687c..2437c2ae 100644
--- a/requirements_cpu_only.txt
+++ b/requirements_cpu_only.txt
@@ -30,6 +30,6 @@ flask_cloudflared==0.0.14
sse-starlette==1.6.5
tiktoken
-# llama-cpp-python (CPU only, AVX2)
-https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.3.8+cpuavx2-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
-https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.3.8+cpuavx2-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
+# llama.cpp (CPU only, AVX2)
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/cpu/llama_cpp_binaries-0.2.0+cpuavx2-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/cpu/llama_cpp_binaries-0.2.0+cpuavx2-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
diff --git a/requirements_cpu_only_noavx2.txt b/requirements_cpu_only_noavx2.txt
index 2003c544..cbaa8e96 100644
--- a/requirements_cpu_only_noavx2.txt
+++ b/requirements_cpu_only_noavx2.txt
@@ -30,6 +30,6 @@ flask_cloudflared==0.0.14
sse-starlette==1.6.5
tiktoken
-# llama-cpp-python (CPU only, no AVX2)
-https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.3.8+cpuavx-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
-https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.3.8+cpuavx-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
+# llama.cpp (CPU only, no AVX2)
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/cpu/llama_cpp_binaries-0.2.0+cpuavx-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/cpu/llama_cpp_binaries-0.2.0+cpuavx-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
diff --git a/requirements_noavx2.txt b/requirements_noavx2.txt
index 60b71ac1..cce27aa2 100644
--- a/requirements_noavx2.txt
+++ b/requirements_noavx2.txt
@@ -31,21 +31,11 @@ flask_cloudflared==0.0.14
sse-starlette==1.6.5
tiktoken
-# llama-cpp-python (CPU only, no AVX2)
-https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.3.8+cpuavx-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
-https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.3.8+cpuavx-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
-
-# llama-cpp-python (CUDA, with GGML_CUDA_FORCE_MMQ)
-https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.3.8+cu124avx-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
-https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.3.8+cu124avx-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
-
-# llama-cpp-python (CUDA, without GGML_CUDA_FORCE_MMQ)
-https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda_tensorcores-0.3.8+cu124avx-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
-https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda_tensorcores-0.3.8+cu124avx-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
-
# CUDA wheels
-https://github.com/oobabooga/exllamav3/releases/download/v0.0.1/exllamav3-0.0.1+cu124.torch2.6.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
-https://github.com/oobabooga/exllamav3/releases/download/v0.0.1/exllamav3-0.0.1+cu124.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/textgen-webui/llama_cpp_binaries-0.2.0+cu124avx-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/textgen-webui/llama_cpp_binaries-0.2.0+cu124avx-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
+https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a3/exllamav3-0.0.1a3+cu124.torch2.6.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
+https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a3/exllamav3-0.0.1a3+cu124.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
https://github.com/oobabooga/exllamav2/releases/download/v0.2.8/exllamav2-0.2.8+cu124.torch2.6.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
https://github.com/oobabooga/exllamav2/releases/download/v0.2.8/exllamav2-0.2.8+cu124.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
https://github.com/oobabooga/exllamav2/releases/download/v0.2.8/exllamav2-0.2.8-py3-none-any.whl; platform_system == "Linux" and platform_machine != "x86_64"