###
This commit is contained in:
parent
9234aad01b
commit
9c02b1a454
38
main.py
38
main.py
|
|
@ -17,18 +17,13 @@ if "invoice_counter" not in st.session_state:
|
|||
st.session_state.invoice_counter = 0
|
||||
|
||||
# ==========================================
|
||||
# 1. API-CLIENT FÜR EXTERNE BACKENDS
|
||||
# 1. API-CLIENT FÜR DAS vLLM-BACKEND
|
||||
# ==========================================
|
||||
class LLMClient:
|
||||
"""Client für Ollama und vLLM (OpenAI-kompatibel) mit JSON-Schema-Unterstützung."""
|
||||
|
||||
def __init__(self, backend_type="vllm", model="gemma431b", base_url=None):
|
||||
self.backend_type = backend_type.lower()
|
||||
def __init__(self, model="gemma431b", base_url="http://10.192.0.140:8002"):
|
||||
self.model = model
|
||||
|
||||
if not base_url:
|
||||
base_url = "http://10.192.0.140:8002" if self.backend_type == "vllm" else "http://localhost:11434"
|
||||
|
||||
base_url = base_url.strip()
|
||||
if base_url.startswith("[") and ")" in base_url:
|
||||
match = re.search(r"\((https?://.*?)\)", base_url)
|
||||
|
|
@ -42,31 +37,7 @@ class LLMClient:
|
|||
{"role": "system", "content": system_prompt},
|
||||
{"role": "user", "content": prompt}
|
||||
]
|
||||
|
||||
if self.backend_type == "ollama":
|
||||
return self._ollama_request(messages, temperature, json_schema)
|
||||
else:
|
||||
return self._openai_compatible_request(messages, temperature, json_schema)
|
||||
|
||||
def _ollama_request(self, messages, temperature, json_schema):
|
||||
url = f"{self.base_url}/api/chat"
|
||||
payload = {
|
||||
"model": self.model,
|
||||
"messages": messages,
|
||||
"stream": False,
|
||||
"options": {
|
||||
"temperature": temperature
|
||||
}
|
||||
}
|
||||
if json_schema:
|
||||
payload["format"] = json_schema
|
||||
|
||||
try:
|
||||
response = requests.post(url, json=payload, timeout=120)
|
||||
response.raise_for_status()
|
||||
return response.json().get("message", {}).get("content", "")
|
||||
except Exception as e:
|
||||
return f"Ollama Error: {e}"
|
||||
return self._openai_compatible_request(messages, temperature, json_schema)
|
||||
|
||||
def _openai_compatible_request(self, messages, temperature, json_schema):
|
||||
url = f"{self.base_url}/v1/chat/completions"
|
||||
|
|
@ -361,8 +332,7 @@ def get_cached_llm():
|
|||
if "llm_instance" not in st.session_state:
|
||||
llm_url = os.environ.get('VLLM_URL', 'http://10.192.0.140:8002')
|
||||
llm_model = os.environ.get('LLM_MODEL', 'gemma431b')
|
||||
llm_backend = os.environ.get('LLM_BACKEND', 'vllm')
|
||||
st.session_state.llm_instance = LLMClient(backend_type=llm_backend, model=llm_model, base_url=llm_url)
|
||||
st.session_state.llm_instance = LLMClient(model=llm_model, base_url=llm_url)
|
||||
return st.session_state.llm_instance
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue