diff --git a/EdgeCraftRAG/docker_compose/intel/gpu/arc/README.md b/EdgeCraftRAG/docker_compose/intel/gpu/arc/README.md index 62baca9037..ec1b737195 100755 --- a/EdgeCraftRAG/docker_compose/intel/gpu/arc/README.md +++ b/EdgeCraftRAG/docker_compose/intel/gpu/arc/README.md @@ -34,7 +34,7 @@ EC-RAG supports vLLM deployment(default method) and local OpenVINO deployment fo #### Intel Arc A770 **OS**: Ubuntu Server 22.04.1 or newer (at least 6.2 LTS kernel) -**Driver & libraries**: Please refer to [Installing GPUs Drivers](https://dgpu-docs.intel.com/driver/client/overview.html#ubuntu-22.04) for detailed driver & libraries setup +**Driver & libraries**: Please refer to [Installing GPUs Drivers](https://dgpu-docs.intel.com/driver/installation-rolling.html#installing-gpu-drivers) for detailed driver & libraries setup **Available Inferencing Framework**: openVINO, vLLM ### 2. Access the Code diff --git a/EdgeCraftRAG/docker_compose/intel/gpu/arc/README_zh.md b/EdgeCraftRAG/docker_compose/intel/gpu/arc/README_zh.md index c1320d2ead..469b6ea2de 100644 --- a/EdgeCraftRAG/docker_compose/intel/gpu/arc/README_zh.md +++ b/EdgeCraftRAG/docker_compose/intel/gpu/arc/README_zh.md @@ -34,7 +34,7 @@ EC-RAG 支持 vLLM 部署（默认方式）以及面向 Intel Arc GPU 和 Core U #### Intel Arc A770 **操作系统**：Ubuntu Server 22.04.1 或更高版本（至少 6.2 LTS 内核） -**驱动与库**：详细驱动与库安装请参考 [Installing GPUs Drivers](https://dgpu-docs.intel.com/driver/client/overview.html#ubuntu-22.04) +**驱动与库**：详细驱动与库安装请参考 [Installing GPUs Drivers](https://dgpu-docs.intel.com/driver/installation-rolling.html#installing-gpu-drivers) **可用推理框架**：openVINO、vLLM ### 2. 获取代码 diff --git a/EdgeCraftRAG/docker_compose/intel/gpu/arc/compose.yaml b/EdgeCraftRAG/docker_compose/intel/gpu/arc/compose.yaml index bf44e6686c..21d8595f04 100644 --- a/EdgeCraftRAG/docker_compose/intel/gpu/arc/compose.yaml +++ b/EdgeCraftRAG/docker_compose/intel/gpu/arc/compose.yaml @@ -88,6 +88,7 @@ services: https_proxy: ${https_proxy} vLLM_ENDPOINT: ${vLLM_ENDPOINT:-http://${HOST_IP}:${VLLM_SERVICE_PORT_B60:-8086}} OVMS_ENDPOINT: ${OVMS_ENDPOINT:-http://${HOST_IP}:${OVMS_SERVICE_PORT:-8000}} + ECRAG_LLM_API_MODE: ${ECRAG_LLM_API_MODE:-chat} LLM_MODEL: ${LLM_MODEL} ENABLE_BENCHMARK: ${ENABLE_BENCHMARK:-false} MAX_MODEL_LEN: ${MAX_MODEL_LEN:-49152} diff --git a/EdgeCraftRAG/docs/API_Guide.md b/EdgeCraftRAG/docs/API_Guide.md index 2b0c318bc6..6e80956381 100644 --- a/EdgeCraftRAG/docs/API_Guide.md +++ b/EdgeCraftRAG/docs/API_Guide.md @@ -432,28 +432,6 @@ curl -X GET http://${HOST_IP}:16010/v1/data/documents \ -H "Content-Type: application/json" | jq '.' ``` -### Get all files - -```bash -curl -X GET http://${HOST_IP}:16010/v1/data/files \ - -H "Content-Type: application/json" | jq '.' -``` - -### Get a specific file - -```bash -curl -X GET http://${HOST_IP}:16010/v1/data/files/{name} \ - -H "Content-Type: application/json" | jq '.' -``` - -### Upload a file (from UI) - -```bash -curl -X POST "http://${HOST_IP}:16010/v1/data/file/{file_name}" \ - -H "Content-Type: multipart/form-data" \ - -F "file=@/path/to/your/document.pdf" | jq '.' -``` - --- ## Session Management diff --git a/EdgeCraftRAG/edgecraftrag/api/v1/chatqna.py b/EdgeCraftRAG/edgecraftrag/api/v1/chatqna.py index 01ddaabc65..4cf7d8c098 100644 --- a/EdgeCraftRAG/edgecraftrag/api/v1/chatqna.py +++ b/EdgeCraftRAG/edgecraftrag/api/v1/chatqna.py @@ -8,11 +8,11 @@ import requests from comps.cores.proto.api_protocol import ChatCompletionRequest from edgecraftrag.api_schema import RagOut -from edgecraftrag.base import GeneratorType +from edgecraftrag.base import GeneratorType, InferenceType from edgecraftrag.context import ctx from edgecraftrag.utils import chain_async_generators, serialize_contexts, serialize_node_with_score, stream_generator from fastapi import Body, FastAPI, HTTPException, status -from fastapi.responses import StreamingResponse +from fastapi.responses import JSONResponse, StreamingResponse chatqna_app = FastAPI() thread_pool = ThreadPoolExecutor(max_workers=16) @@ -220,3 +220,44 @@ async def save_session(sessionid, run_agent_gen): yield chunk or "" await asyncio.sleep(0) session_mgr.save_current_message(sessionid, "assistant", current_content) + +def _not_ready(reason: str): + return JSONResponse( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + content={"status": "not_ready", "reason": reason}, + ) + +# Lightweight readiness check before sending a real RAG request +@chatqna_app.get(path="/v1/ready") +async def get_ready(): + pipeline = ctx.get_pipeline_mgr().get_active_pipeline() + if pipeline is None or not pipeline.status.active: + return _not_ready("No active pipeline") + + generator = pipeline.get_generator(GeneratorType.CHATQNA) + if generator is not None and generator.inference_type == InferenceType.VLLM: + try: + response = requests.get(f"{generator.vllm_endpoint.rstrip('/')}/v1/models", timeout=2) + response.raise_for_status() + except Exception: + return _not_ready("LLM backend unavailable") + + try: + active_kbs = ctx.knowledgemgr.get_active_knowledge_base() + if not active_kbs: + return _not_ready("Retrieval unavailable") + request = ChatCompletionRequest(messages="ready") + request.user = active_kbs + result = await ctx.get_pipeline_mgr().run_retrieve(chat_request=request) + if result == -1: + return _not_ready("Retrieval unavailable") + except Exception: + return _not_ready("Retrieval unavailable") + + return { + "status": "ready", + "pipeline": pipeline.name, + "pipeline_active": pipeline.status.active, + "llm": "ready", + "retrieval": "ready", + } diff --git a/EdgeCraftRAG/edgecraftrag/components/agent.py b/EdgeCraftRAG/edgecraftrag/components/agent.py index 914edb6229..c5f2162dd0 100644 --- a/EdgeCraftRAG/edgecraftrag/components/agent.py +++ b/EdgeCraftRAG/edgecraftrag/components/agent.py @@ -12,6 +12,20 @@ from langgraph.config import get_stream_writer from pydantic import BaseModel, Field, model_serializer +from langchain_core.runnables.config import ensure_config, var_child_runnable_config +from langgraph._internal._runnable import RunnableCallable +_orig = RunnableCallable.ainvoke + +async def _patched(self, input, config=None, **kwargs): + token = var_child_runnable_config.set(config or ensure_config()) + try: + return await _orig(self, input, config, **kwargs) + finally: + var_child_runnable_config.reset(token) + + +RunnableCallable.ainvoke = _patched + class Retrieval(BaseModel): step: Any @@ -59,9 +73,9 @@ async def llm_generate(self, request: ChatCompletionRequest, streaming): response = await self._run_pipeline_generate(request) return response - async def llm_generate_astream_writer(self, request, prefix=None, suffix=None) -> str: + async def llm_generate_astream_writer(self, request, prefix=None, suffix=None, writer=None) -> str: response = "" - writer = get_stream_writer() + writer = _safe_writer(writer) first = True generator = await self.llm_generate(request, True) async for chunk in generator: @@ -123,7 +137,15 @@ def ser_model(self): return set -async def stream_writer(input): - writer = get_stream_writer() +def _safe_writer(writer=None): + if writer is not None: + return writer + try: + return get_stream_writer() + except RuntimeError: + return lambda chunk: None + +async def stream_writer(input, writer=None): + writer = _safe_writer(writer) async for chunk in stream_generator(input): writer(chunk) diff --git a/EdgeCraftRAG/edgecraftrag/components/agents/simple.py b/EdgeCraftRAG/edgecraftrag/components/agents/simple.py index 131afe63e8..0a7623ff39 100644 --- a/EdgeCraftRAG/edgecraftrag/components/agents/simple.py +++ b/EdgeCraftRAG/edgecraftrag/components/agents/simple.py @@ -124,7 +124,7 @@ def _build_graph(self): return qnagraph.compile() - async def retrieve(self, state: QnaState) -> dict: + async def retrieve(self, state: QnaState, writer=None) -> dict: # print(f"State Retrieve {state}") request = state.request request.messages = state.query @@ -137,7 +137,8 @@ async def retrieve(self, state: QnaState) -> dict: f"Retrieved {format_terminal_str(str(len(retrieved)), color='magenta', bold=True)} documents, Reranked to top {format_terminal_str(str(len(reranked)), color='magenta', bold=True)}.", ) await stream_writer( - f"\n\n🔍 **Retrieved {str(len(retrieved))} documents, Reranked to top {str(len(reranked))}**\n\n" + f"\n\n🔍 **Retrieved {str(len(retrieved))} documents, Reranked to top {str(len(reranked))}**\n\n", + writer=writer, ) new_retrieval = Retrieval(step=state.num_retrievals, query=state.query, retrieved=retrieved, reranked=reranked) @@ -146,9 +147,9 @@ async def retrieve(self, state: QnaState) -> dict: "retrievals": [*state.retrievals, new_retrieval], } - async def generate_query(self, state: QnaState) -> dict: + async def generate_query(self, state: QnaState, writer=None) -> dict: # print(f"State generate_query {state}") - await stream_writer('') + await stream_writer('', writer=writer) messages = [ {"role": ROLE.USER, "content": state.question}, @@ -157,7 +158,7 @@ async def generate_query(self, state: QnaState) -> dict: self._messages.extend(messages) self.conversation_history.extend(messages) - response = await self.llm_generate_astream_writer(state.request) + response = await self.llm_generate_astream_writer(state.request, writer=writer) message = {"role": ROLE.ASSISTANT, "content": response} self._messages.append(message) @@ -166,10 +167,10 @@ async def generate_query(self, state: QnaState) -> dict: "query": response, } - async def check_retrieved(self, state: QnaState) -> str: + async def check_retrieved(self, state: QnaState, writer=None) -> str: # print(f"State check_retrieved {state}") print("🤔", format_terminal_str("Evaluating if more information is needed", color="green")) - await stream_writer("🤔 **Evaluating if more information is needed...**\n\n") + await stream_writer("🤔 **Evaluating if more information is needed...**\n\n", writer=writer) # Format context for the next decision contexts = self.cfg.prompt_templates.contexts.format( @@ -191,27 +192,28 @@ async def check_retrieved(self, state: QnaState) -> str: ), ) await stream_writer( - f"\n\n⚠️ **Reached maximum retrievals: {self.max_retrievals}, stopping searching...**\n\n" + f"\n\n⚠️ **Reached maximum retrievals: {self.max_retrievals}, stopping searching...**\n\n", + writer=writer, ) return "stop" else: - response = await self.llm_generate_astream_writer(state.request) + response = await self.llm_generate_astream_writer(state.request, writer=writer) message = {"role": ROLE.ASSISTANT, "content": response} self._messages.append(message) self.conversation_history.append(message) if response.upper().startswith("NO"): print("✅", format_terminal_str("Information is sufficient, moving to next step\n", color="green")) - await stream_writer("\n\n✅ **Information is sufficient, moving to next step...**\n\n") + await stream_writer("\n\n✅ **Information is sufficient, moving to next step...**\n\n", writer=writer) return "stop" else: print("🔄", format_terminal_str("Need more information, generating new query ...", color="green")) - await stream_writer("\n\n🔄 **Need more information, generating new query...**\n\n") + await stream_writer("\n\n🔄 **Need more information, generating new query...**\n\n", writer=writer) return "continue" - async def generate_answer(self, state: QnaState) -> dict: + async def generate_answer(self, state: QnaState, writer=None) -> dict: # print(f"State generate_answer {state}") print("📝", format_terminal_str("Generating the final answer ...", color="cyan", bold=True)) - await stream_writer('') + await stream_writer('', writer=writer) plan_with_information = "" prev_step = "" for i, r in enumerate(state.retrievals): @@ -232,7 +234,7 @@ async def generate_answer(self, state: QnaState) -> dict: ] self.conversation_history.extend(self._messages) - response = await self.llm_generate_astream_writer(state.request) + response = await self.llm_generate_astream_writer(state.request, writer=writer) self.conversation_history.append({"role": ROLE.ASSISTANT, "content": response}) answer = self.postproc_answer(response, state) diff --git a/EdgeCraftRAG/edgecraftrag/components/generator.py b/EdgeCraftRAG/edgecraftrag/components/generator.py index fcad2433d8..be94b11b4b 100644 --- a/EdgeCraftRAG/edgecraftrag/components/generator.py +++ b/EdgeCraftRAG/edgecraftrag/components/generator.py @@ -16,10 +16,33 @@ from edgecraftrag.components.agents.utils import build_document_node_block from fastapi.responses import StreamingResponse from llama_index.llms.openai_like import OpenAILike +from llama_index.core.base.llms.types import ChatMessage, CompletionResponse, MessageRole from pydantic import model_serializer from unstructured.staging.base import elements_from_base64_gzipped_json +def _resolve_api_mode() -> str: + return os.getenv("ECRAG_LLM_API_MODE", "chat").strip().lower() + +def _build_freechat_messages(chat_request): + """Build a list of ChatMessage from a raw chat request message list.""" + messages = [] + for m in chat_request.messages: + if isinstance(m, dict): + messages.append(ChatMessage(role=m.get("role", "user"), content=m.get("content", ""))) + else: + messages.append(m) + return messages + +def _build_chat_messages(template, enable_think, chat_history, text_gen_context, final_query): + """Build the SYSTEM/USER ChatMessage list used in chat API mode.""" + system_content = template.format(input="", chat_history=chat_history, context=text_gen_context) + think_tag = "" if enable_think else " /no_think" + return [ + ChatMessage(role=MessageRole.SYSTEM, content=system_content), + ChatMessage(role=MessageRole.USER, content=str(final_query) + think_tag), + ] + def extract_urls(text): urls = [] words = text.split() @@ -88,14 +111,20 @@ def build_stream_response(status=None, content=None, error=None): return response -async def local_stream_generator(lock, llm, prompt_str, unstructured_str, benchmark=None, benchmark_index=None): +async def local_stream_generator(lock, llm, prompt_str, unstructured_str, benchmark=None, benchmark_index=None, use_chat=False): enable_benchmark = benchmark.is_enabled() if benchmark else False start_time = time.perf_counter() if enable_benchmark else None async with lock: if enable_benchmark: - response = await llm.astream_complete_with_bench(prompt_str) + if use_chat: + response = await llm.astream_chat_with_bench(prompt_str) + else: + response = await llm.astream_complete_with_bench(prompt_str) else: - response = await llm.astream_complete(prompt_str) + if use_chat: + response = await llm.astream_chat(prompt_str) + else: + response = await llm.astream_complete(prompt_str) try: async for r in response: yield r.delta or "" @@ -110,10 +139,13 @@ async def local_stream_generator(lock, llm, prompt_str, unstructured_str, benchm result_error = str(e)[start_idx:] yield f"code:0000{result_error}" -async def stream_generator(llm, prompt_str, unstructured_str, benchmark=None, benchmark_index=None): +async def stream_generator(llm, prompt_str, unstructured_str, benchmark=None, benchmark_index=None, use_chat=False): enable_benchmark = benchmark.is_enabled() if benchmark else False start_time = time.perf_counter() if enable_benchmark else None - response = await llm.astream_complete(prompt_str) + if use_chat: + response = await llm.astream_chat(prompt_str) + else: + response = await llm.astream_complete(prompt_str) try: async for r in response: yield r.delta or "" @@ -237,22 +269,24 @@ def __init__( self.remote_endpoint = self.vllm_endpoint def prompt_handler( - self, model_path, prompt_content=None, prompt_template_file=None, enable_think=False, enable_rag_retrieval=True + self, model_path, prompt_content=None, prompt_template_file=None, enable_think=False, enable_rag_retrieval=True, use_chat=None ): + if use_chat is None: + use_chat = _resolve_api_mode() == "chat" if prompt_content: - return get_prompt_template(model_path, prompt_content, prompt_template_file, enable_think) + return get_prompt_template(model_path, prompt_content, prompt_template_file, enable_think, use_chat) elif prompt_template_file is None: print("There is no template file, using the default template.") - prompt_template = get_prompt_template(model_path, prompt_content, prompt_template_file, enable_think) + prompt_template = get_prompt_template(model_path, prompt_content, prompt_template_file, enable_think, use_chat) return prompt_template else: if enable_rag_retrieval: resolve_prompt_template_path(prompt_template_file) else: prompt_content = "### User Guide ###You are a helpful assistant. Please respond to user inquiries with concise and professional answers.### Historical Content ###{chat_history}" - return get_prompt_template(model_path, prompt_content, prompt_template_file, enable_think) + return get_prompt_template(model_path, prompt_content, prompt_template_file, enable_think, use_chat) - return get_prompt_template(model_path, prompt_content, prompt_template_file, enable_think) + return get_prompt_template(model_path, prompt_content, prompt_template_file, enable_think, use_chat) def set_prompt(self, prompt): if "{context}" not in prompt: @@ -276,7 +310,7 @@ def clean_string(self, string): ret = ret.replace(*p) return ret - def query_transform(self, chat_request, retrieved_nodes, sub_questions=None): + def query_transform(self, chat_request, retrieved_nodes, sub_questions=None, use_chat=False): """Generate text_gen_context and prompt_str :param chat_request: Request object :param retrieved_nodes: List of retrieved nodes @@ -305,12 +339,18 @@ def query_transform(self, chat_request, retrieved_nodes, sub_questions=None): self.prompt_template_file, self.enable_think, self.enable_rag_retrieval, + use_chat=use_chat, ) if sub_questions: final_query = f"{query}\n\n### Sub-questions ###\nThe following list is how you should consider the answer, you MUST follow these steps when responding:\n\n{sub_questions}" else: final_query = query + if use_chat: + messages = _build_chat_messages( + self.original_template, self.enable_think, chat_history, text_gen_context, final_query + ) + return text_gen_context, messages prompt_str = self.prompt.format(input=final_query, chat_history=chat_history, context=text_gen_context) return text_gen_context, prompt_str @@ -322,7 +362,8 @@ async def run(self, chat_request, retrieved_nodes, node_parser_type, **kwargs): benchmark = kwargs.get("benchmark", None) benchmark_index = kwargs.get("benchmark_index", None) sub_questions = kwargs.get("sub_questions", None) - text_gen_context, prompt_str = self.query_transform(chat_request, retrieved_nodes, sub_questions=sub_questions) + use_chat = _resolve_api_mode() == "chat" + text_gen_context, payload = self.query_transform(chat_request, retrieved_nodes, sub_questions=sub_questions, use_chat=use_chat) # self.llm().config.update_generation_config(config) self.llm().config.update_generation_config(temperature=chat_request.temperature,top_p=chat_request.top_p, top_k=chat_request.top_k, typical_p=chat_request.typical_p, repetition_penalty=chat_request.repetition_penalty, do_sample=chat_request.temperature > 0.0) self.llm().config.max_new_tokens = chat_request.max_tokens @@ -332,13 +373,17 @@ async def run(self, chat_request, retrieved_nodes, node_parser_type, **kwargs): if chat_request.stream: # Asynchronous generator async def generator(): - async for chunk in local_stream_generator(self.lock, self.llm(), prompt_str, unstructured_str, benchmark, benchmark_index): + async for chunk in local_stream_generator(self.lock, self.llm(), payload, unstructured_str, benchmark, benchmark_index, use_chat=use_chat): yield chunk or "" await asyncio.sleep(0) return generator() else: - result = self.llm().complete(prompt_str) + if use_chat: + chat_response = self.llm().chat(payload) + result = CompletionResponse(text=chat_response.message.content or "") + else: + result = self.llm().complete(payload) return result async def run_remote(self, chat_request, retrieved_nodes, node_parser_type, **kwargs): @@ -346,7 +391,8 @@ async def run_remote(self, chat_request, retrieved_nodes, node_parser_type, **kw sub_questions = kwargs.get("sub_questions", None) benchmark = kwargs.get("benchmark", None) benchmark_index = kwargs.get("benchmark_index", None) - text_gen_context, prompt_str = self.query_transform(chat_request, retrieved_nodes, sub_questions=sub_questions) + use_chat = _resolve_api_mode() == "chat" + text_gen_context, payload = self.query_transform(chat_request, retrieved_nodes, sub_questions=sub_questions, use_chat=use_chat) api_base_suffix = "/v3" if self.inference_type == InferenceType.OVMS else "/v1" llm = OpenAILike( api_key="fake", @@ -358,6 +404,7 @@ async def run_remote(self, chat_request, retrieved_nodes, node_parser_type, **kw temperature=chat_request.temperature, streaming=chat_request.stream, repetition_penalty=chat_request.repetition_penalty, + is_chat_model=use_chat, ) unstructured_str = "" if node_parser_type == NodeParserType.UNSTRUCTURED: @@ -366,14 +413,18 @@ async def run_remote(self, chat_request, retrieved_nodes, node_parser_type, **kw # Asynchronous generator async def generator(): - async for chunk in stream_generator(llm, prompt_str, unstructured_str, benchmark, benchmark_index): + async for chunk in stream_generator(llm, payload, unstructured_str, benchmark, benchmark_index, use_chat=use_chat): yield chunk or "" await asyncio.sleep(0) return generator() else: - result = await llm.acomplete(prompt_str) - return result + if use_chat: + result = await llm.achat(payload) + return CompletionResponse(text=result.message.content or "") + else: + result = await llm.acomplete(payload) + return result async def run_vllm(self, chat_request, retrieved_nodes, node_parser_type, **kwargs): return await self.run_remote(chat_request, retrieved_nodes, node_parser_type, **kwargs) @@ -460,21 +511,27 @@ async def run_local(self, chat_request, retrieved_nodes, node_parser_type, **kwa ) self.llm().generate_kwargs = generate_kwargs self.llm().max_new_tokens = chat_request.max_tokens - prompt_str = chatcompletion_to_chatml(chat_request) + use_chat = _resolve_api_mode() == "chat" + payload = _build_freechat_messages(chat_request) if use_chat else chatcompletion_to_chatml(chat_request) if chat_request.stream: # Asynchronous generator async def generator(): - async for chunk in local_stream_generator(self.lock, self.llm(), prompt_str, ""): + async for chunk in local_stream_generator(self.lock, self.llm(), payload, "", use_chat=use_chat): yield chunk or "" await asyncio.sleep(0) return generator() else: - result = self.llm().complete(prompt_str) + if use_chat: + chat_response = self.llm().chat(payload) + result = CompletionResponse(text=chat_response.message.content or "") + else: + result = self.llm().complete(payload) return result async def run_remote(self, chat_request, retrieved_nodes, node_parser_type, **kwargs): + use_chat = _resolve_api_mode() == "chat" api_base_suffix = "/v3" if self.inference_type == InferenceType.OVMS else "/v1" llm = OpenAILike( api_key="fake", @@ -486,21 +543,29 @@ async def run_remote(self, chat_request, retrieved_nodes, node_parser_type, **kw temperature=chat_request.temperature, streaming=chat_request.stream, repetition_penalty=chat_request.repetition_penalty, + is_chat_model=use_chat, ) - prompt_str = chatcompletion_to_chatml(chat_request) + payload = _build_freechat_messages(chat_request) if use_chat else chatcompletion_to_chatml(chat_request) if chat_request.stream: # Asynchronous generator async def generator(): - gen = await llm.astream_complete(prompt_str) + if use_chat: + gen = await llm.astream_chat(payload) + else: + gen = await llm.astream_complete(payload) async for chunk in gen: yield chunk.delta or "" await asyncio.sleep(0) return generator() else: - result = await llm.acomplete(prompt_str) - return str(result) + if use_chat: + result = await llm.achat(payload) + return str(result.message.content or "") + else: + result = await llm.acomplete(payload) + return str(result) async def run_vllm(self, chat_request, retrieved_nodes, node_parser_type, **kwargs): return await self.run_remote(chat_request, retrieved_nodes, node_parser_type, **kwargs) diff --git a/EdgeCraftRAG/edgecraftrag/components/model.py b/EdgeCraftRAG/edgecraftrag/components/model.py index 3fec80ac82..b3c94aafbe 100644 --- a/EdgeCraftRAG/edgecraftrag/components/model.py +++ b/EdgeCraftRAG/edgecraftrag/components/model.py @@ -374,4 +374,23 @@ def complete_with_bench( generated_tokens = np.array(generation_result.tokens) completion = self._tokenizer.decode(generated_tokens) token = completion[0] - return CompletionResponse(text=token, raw={"model_output": token}) \ No newline at end of file + return CompletionResponse(text=token, raw={"model_output": token}) + + def chat_with_bench(self, messages: Any, **kwargs: Any) -> CompletionResponse: + """Chat endpoint with benchmark. + + Applies the model's own chat template via messages_to_prompt (local + tokenizer), then reuses complete_with_bench. + """ + prompt = self.messages_to_prompt(messages) + return self.complete_with_bench(prompt, formatted=True, **kwargs) + + def stream_chat_with_bench(self, messages: Any, **kwargs: Any) -> CompletionResponseGen: + """Streaming chat endpoint with benchmark.""" + prompt = self.messages_to_prompt(messages) + return self.stream_complete_with_bench(prompt, formatted=True, **kwargs) + + async def astream_chat_with_bench(self, messages: Any, **kwargs: Any) -> CompletionResponseAsyncGen: + """Async streaming chat endpoint with benchmark.""" + prompt = self.messages_to_prompt(messages) + return await self.astream_complete_with_bench(prompt, formatted=True, **kwargs) \ No newline at end of file diff --git a/EdgeCraftRAG/edgecraftrag/requirements.txt b/EdgeCraftRAG/edgecraftrag/requirements.txt index 74f2084355..810714fcc4 100644 --- a/EdgeCraftRAG/edgecraftrag/requirements.txt +++ b/EdgeCraftRAG/edgecraftrag/requirements.txt @@ -18,16 +18,17 @@ torchvision==0.23.0+cpu transformers==4.53.3 unstructured[all-docs]==0.18.27 werkzeug==3.1.3 -llama-index==0.14.13 +llama-index==0.14.23 pyarrow==22.0.0 -llama-index-embeddings-openvino==0.6.1 -llama-index-embeddings-openvino-genai==0.6.1 -llama-index-llms-openai==0.6.13 -llama-index-llms-openai-like==0.5.3 -llama-index-llms-openvino==0.5.1 +llama-index-embeddings-openvino==0.7.0 +llama-index-embeddings-openvino-genai==0.7.0 +llama-index-llms-openai==0.7.10 +llama-index-llms-openai-like==0.7.2 +llama-index-llms-openvino==0.6.0 llama-index-llms-openvino-genai==0.3.1 -llama-index-postprocessor-openvino-rerank==0.5.1 +llama-index-postprocessor-openvino-rerank==0.6.0 llama-index-readers-file==0.5.4 -llama-index-retrievers-bm25==0.6.5 -llama-index-vector-stores-faiss==0.5.2 -llama-index-vector-stores-milvus==0.9.6 \ No newline at end of file +llama-index-retrievers-bm25==0.7.1 +llama-index-vector-stores-faiss==0.6.0 +llama-index-vector-stores-milvus==0.9.6 +nncf==2.19.0 \ No newline at end of file diff --git a/EdgeCraftRAG/edgecraftrag/utils.py b/EdgeCraftRAG/edgecraftrag/utils.py index d6c2cef822..d33e40abb4 100644 --- a/EdgeCraftRAG/edgecraftrag/utils.py +++ b/EdgeCraftRAG/edgecraftrag/utils.py @@ -113,7 +113,7 @@ def iter_elements(cls, paragraph: Paragraph, opts: DocxPartitionerOptions) -> It yield Image(text="IMAGE", metadata=element_metadata) -def get_prompt_template(model_path, prompt_content=None, template_path=None, enable_think=False): +def get_prompt_template(model_path, prompt_content=None, template_path=None, enable_think=False, use_chat=False): model_path = _resolve_model_path(model_path) if prompt_content is not None: template = prompt_content @@ -122,6 +122,8 @@ def get_prompt_template(model_path, prompt_content=None, template_path=None, ena template = normalized_path.read_text(encoding=None) else: template = DEFAULT_TEMPLATE + if use_chat: + return template, template tokenizer = AutoTokenizer.from_pretrained(model_path, local_files_only=os.path.exists(model_path)) messages = [{"role": "system", "content": template}, {"role": "user", "content": "\n{input}\n"}] prompt_template = tokenizer.apply_chat_template( diff --git a/EdgeCraftRAG/kubernetes/helm/Chart.yaml b/EdgeCraftRAG/kubernetes/helm/Chart.yaml new file mode 100644 index 0000000000..c5fe831cff --- /dev/null +++ b/EdgeCraftRAG/kubernetes/helm/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: edgecraftrag +description: Helm chart for EdgeCraftRAG stack +type: application +version: 0.1.0 +appVersion: "25.11" diff --git a/EdgeCraftRAG/kubernetes/helm/README.md b/EdgeCraftRAG/kubernetes/helm/README.md new file mode 100644 index 0000000000..137091f31f --- /dev/null +++ b/EdgeCraftRAG/kubernetes/helm/README.md @@ -0,0 +1,94 @@ +# EdgeCraft RAG Helm Chart + +This doc intrudoces the Helm chart for deploying EdgeCraft RAG (ecrag) on a Kubernetes cluster. + +## Prerequisites + +- A running Kubernetes cluster. +- Helm installed. +- Required Docker images available in your registry or locally. + +## Configuration + +Before installing, you should configure the `edgecraftrag/values.yaml` file according to your environment. + +### Key Configurations + +1. **Images**: Set the registry and tag for `ecrag` and `vllm`. + ```yaml + image: + ecrag: + registry: + tag: + vllm: + registry: + tag: + ``` + +2. **Environment Variables**: Configure proxies and host IP. + ```yaml + env: + http_proxy: "http://proxy:port" + https_proxy: "http://proxy:port" + HOST_IP: "" + ``` + +3. **LLM Settings**: Adjust LLM model paths and parameters. + ```yaml + llm: + LLM_MODEL: "/path/to/model/inside/container" # Ensure this maps to paths.model + ``` + +4. **Persistant Paths**: Ensure the host paths exist for mounting. + ```yaml + paths: + model: /home/user/models + docs: /home/user/docs + ``` + +## Installation + +To install the chart, please use below command (`edgecraftrag` as an example) + +```bash +cd kubernetes/helm +helm install edgecraftrag ./ +``` + +If there're different clusters avaliable, please install the chart with specific kube config, e.g. : + +```bash +helm install edgecraftrag ./ --kubeconfig /home/user/.kube/nas.yaml +``` + +## Verification + +### Accessing the Web UI + +Once the service is running, you can access the UI via your browser. + +1. **Identify the Port**: + Check the `nodePort` configured in the `edgecraftrag/values.yaml` file. This is the external access port. + +2. **Identify the IP**: + Use the IP address of the Kubernetes node where the deployment is running. + * If running on your local machine (e.g., MicroK8s), use `localhost` or your machine's LAN IP. + * If running on a remote cluster, use that node's IP. + +3. **Open in Browser**: + Navigate to `http://:` + > Example: `http://192.168.1.5:31234` + +## Uninstallation + +To uninstall/delete the `edgecraftrag` deployment: + +```bash +helm uninstall edgecraftrag +``` + +If there're different clusters avaliable, please uninstall the chart with specific kube config, e.g. : + +```bash +helm uninstall edgecraftrag --kubeconfig /home/user/.kube/nas.yaml +``` diff --git a/EdgeCraftRAG/kubernetes/helm/README_zh.md b/EdgeCraftRAG/kubernetes/helm/README_zh.md new file mode 100644 index 0000000000..999830dc1e --- /dev/null +++ b/EdgeCraftRAG/kubernetes/helm/README_zh.md @@ -0,0 +1,94 @@ +# EdgeCraft RAG Helm Chart + +此文档将为您介绍如何使用Helm chart在Kubernetes集群上部署EdgeCraft RAG (ecrag)。 + +## 前置条件 + +- 您需要一个运行中的Kubernetes集群。 +- 您需要已经安装Helm。 +- 所需的Docker镜像已在您的镜像仓库或本地可用。 + +## 配置 + +安装前，请根据您的环境配置 `edgecraftrag/values.yaml` 文件。 + +### 关键配置 + +1. **镜像**：设置 `ecrag` 和 `vllm` 的镜像仓库和标签。 + ```yaml + image: + ecrag: + registry: + tag: + vllm: + registry: + tag: + ``` + +2. **环境变量**：配置代理和主机IP。 + ```yaml + env: + http_proxy: "http://proxy:port" + https_proxy: "http://proxy:port" + HOST_IP: "" + ``` + +3. **LLM设置**：调整LLM模型路径和参数。 + ```yaml + llm: + LLM_MODEL: "/path/to/model/inside/container" # 确保此路径映射到 paths.model + ``` + +4. **持久化路径**：确保主机挂载路径存在。 + ```yaml + paths: + model: /home/user/models + docs: /home/user/docs + ``` + +## 安装 + +请使用如下命令安装helm（以`edgecraftrag`作为发布名为例）： + +```bash +cd kubernetes/helm +helm install edgecraftrag ./edgecraftrag +``` + +如果有不同的集群可用，请使用指定的kube config安装chart，例如： + +```bash +helm install edgecraftrag ./edgecraftrag --kubeconfig /home/user/.kube/nas.yaml +``` + +## 验证 + +### 访问Web界面 + +服务运行后，您可以通过浏览器访问UI。 + +1. **确认端口**： + 查看 `edgecraftrag/values.yaml` 文件中配置的 `nodePort`。这是外部访问端口。 + +2. **确认IP**： + 使用部署所运行的Kubernetes节点的IP地址。 + * 如果在本地机器运行（如MicroK8s），使用 `localhost` 或您机器的局域网IP。 + * 如果在远程集群运行，使用该节点的IP。 + +3. **在浏览器中打开**： + 访问 `http://:` + > 示例：`http://192.168.1.5:31234` + +## 卸载 + +卸载/删除部署的`edgecraftrag`： + +```bash +helm uninstall edgecraftrag +``` + +如果有不同的集群可用，请使用指定的kube config卸载chart，例如： + +```bash +helm uninstall edgecraftrag --kubeconfig /home/user/.kube/nas.yaml +``` diff --git a/EdgeCraftRAG/kubernetes/helm/templates/configmap-env.yaml b/EdgeCraftRAG/kubernetes/helm/templates/configmap-env.yaml new file mode 100644 index 0000000000..0fe7f105fb --- /dev/null +++ b/EdgeCraftRAG/kubernetes/helm/templates/configmap-env.yaml @@ -0,0 +1,36 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: edgecraftrag-env +data: + # Common environment variables + no_proxy: "{{ .Values.env.no_proxy }}" + http_proxy: "{{ .Values.env.http_proxy }}" + https_proxy: "{{ .Values.env.https_proxy }}" + HOST_IP: "{{ .Values.env.HOST_IP }}" + ENABLE_BENCHMARK: "{{ .Values.env.ENABLE_BENCHMARK }}" + CHAT_HISTORY_ROUND: "{{ .Values.env.CHAT_HISTORY_ROUND }}" + METADATA_DATABASE_URL: "{{ .Values.env.METADATA_DATABASE_URL }}" + MEGA_SERVICE_PORT: "{{ .Values.ports.mega }}" + PIPELINE_SERVICE_HOST_IP: edgecraftrag-server + PIPELINE_SERVICE_PORT: "{{ .Values.ports.pipeline }}" + UI_SERVICE_PORT: "{{ .Values.ports.ui.port }}" + VLLM_SERVICE_PORT_B60: "{{ .Values.ports.vllm }}" + + # llm-serving-xpu specific environment variables + LLM_MODEL: "{{ .Values.llm.LLM_MODEL }}" + DTYPE: "{{ .Values.llm.DTYPE }}" + ZE_AFFINITY_MASK: "{{ .Values.llm.ZE_AFFINITY_MASK }}" + ENFORCE_EAGER: "{{ .Values.llm.ENFORCE_EAGER }}" + TRUST_REMOTE_CODE: "{{ .Values.llm.TRUST_REMOTE_CODE }}" + DISABLE_SLIDING_WINDOW: "{{ .Values.llm.DISABLE_SLIDING_WINDOW }}" + GPU_MEMORY_UTIL: "{{ .Values.llm.GPU_MEMORY_UTIL }}" + NO_ENABLE_PREFIX_CACHING: "{{ .Values.llm.NO_ENABLE_PREFIX_CACHING }}" + MAX_NUM_BATCHED_TOKENS: "{{ .Values.llm.MAX_NUM_BATCHED_TOKENS }}" + MAX_MODEL_LEN: "{{ .Values.llm.MAX_MODEL_LEN }}" + DISABLE_LOG_REQUESTS: "{{ .Values.llm.DISABLE_LOG_REQUESTS }}" + BLOCK_SIZE: "{{ .Values.llm.BLOCK_SIZE }}" + QUANTIZATION: "{{ .Values.llm.QUANTIZATION }}" + TP: "{{ .Values.llm.TP }}" + DP: "{{ .Values.llm.DP }}" + diff --git a/EdgeCraftRAG/kubernetes/helm/templates/daemonset-edgecraftrag-server.yaml b/EdgeCraftRAG/kubernetes/helm/templates/daemonset-edgecraftrag-server.yaml new file mode 100644 index 0000000000..e7a68336bb --- /dev/null +++ b/EdgeCraftRAG/kubernetes/helm/templates/daemonset-edgecraftrag-server.yaml @@ -0,0 +1,58 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: edgecraftrag-server +spec: + selector: + matchLabels: + app: edgecraftrag-server + template: + metadata: + labels: + app: edgecraftrag-server + spec: + securityContext: + runAsUser: 1000 + runAsGroup: 1000 + supplementalGroups: + - {{ .Values.gpu.groups.video }} + - {{ .Values.gpu.groups.render }} + containers: + - name: edgecraftrag-server + image: "{{ .Values.image.ecrag.registry }}/edgecraftrag-server:{{ .Values.image.ecrag.tag }}" + imagePullPolicy: IfNotPresent + envFrom: + - configMapRef: + name: edgecraftrag-env + env: + - name: PIPELINE_SERVICE_HOST_IP + value: "0.0.0.0" + ports: + - containerPort: {{ .Values.ports.pipeline }} + volumeMounts: + - name: model-path + mountPath: /home/user/models + - name: docs-path + mountPath: /home/user/docs + - name: tmpfile-path + mountPath: /home/user/ui_cache + - name: prompt-path + mountPath: /templates/custom + - name: dri-device + mountPath: /dev/dri + volumes: + - name: model-path + hostPath: + path: "{{ .Values.paths.model }}" + - name: docs-path + hostPath: + path: "{{ .Values.paths.docs }}" + - name: tmpfile-path + hostPath: + path: "{{ .Values.paths.tmpfile }}" + - name: prompt-path + hostPath: + path: "{{ .Values.paths.prompt }}" + - name: dri-device + hostPath: + path: /dev/dri diff --git a/EdgeCraftRAG/kubernetes/helm/templates/daemonset-llm-serving-xpu.yaml b/EdgeCraftRAG/kubernetes/helm/templates/daemonset-llm-serving-xpu.yaml new file mode 100644 index 0000000000..5534993d87 --- /dev/null +++ b/EdgeCraftRAG/kubernetes/helm/templates/daemonset-llm-serving-xpu.yaml @@ -0,0 +1,58 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: llm-serving-xpu +spec: + selector: + matchLabels: + app: llm-serving-xpu + template: + metadata: + labels: + app: llm-serving-xpu + spec: + securityContext: + runAsUser: 1000 + runAsGroup: 1000 + supplementalGroups: + - {{ .Values.gpu.groups.video }} + - {{ .Values.gpu.groups.render }} + containers: + - name: llm-serving-xpu + image: "{{ .Values.image.vllm.registry }}/llm-scaler-vllm:{{ .Values.image.vllm.tag }}" + imagePullPolicy: IfNotPresent + command: + - "/bin/bash" + - "-c" + - "cd /workspace/vllm/models && source /opt/intel/oneapi/setvars.sh --force && \ + VLLM_OFFLOAD_WEIGHTS_BEFORE_QUANT=1 TORCH_LLM_ALLREDUCE=1 VLLM_USE_V1=1 \ + CCL_ZE_IPC_EXCHANGE=pidfd VLLM_ALLOW_LONG_MAX_MODEL_LEN=1 VLLM_WORKER_MULTIPROC_METHOD=spawn \ + python3 -m vllm.entrypoints.openai.api_server \ + --model $LLM_MODEL --dtype $DTYPE --enforce-eager --port $VLLM_SERVICE_PORT_B60 \ + --trust-remote-code --disable-sliding-window --gpu-memory-util $GPU_MEMORY_UTIL \ + --no-enable-prefix-caching --max-num-batched-tokens $MAX_NUM_BATCHED_TOKENS \ + --disable-log-requests --max-model-len $MAX_MODEL_LEN --block-size $BLOCK_SIZE \ + --quantization $QUANTIZATION -tp=$TP -dp=$DP" + envFrom: + - configMapRef: + name: edgecraftrag-env + ports: + - containerPort: {{ .Values.ports.vllm }} + securityContext: + privileged: true + volumeMounts: + - name: model-path + mountPath: /workspace/vllm/models + - name: dri-device + mountPath: /dev/dri + volumes: + - name: model-path + hostPath: + path: "{{ .Values.paths.model }}" + - name: dri-device + hostPath: + path: /dev/dri + tolerations: + - key: "gpu" + operator: "Exists" + effect: "NoSchedule" diff --git a/EdgeCraftRAG/kubernetes/helm/templates/deployment-ecrag.yaml b/EdgeCraftRAG/kubernetes/helm/templates/deployment-ecrag.yaml new file mode 100644 index 0000000000..45f22326eb --- /dev/null +++ b/EdgeCraftRAG/kubernetes/helm/templates/deployment-ecrag.yaml @@ -0,0 +1,45 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ecrag +spec: + replicas: {{ .Values.replica.ecrag }} + selector: + matchLabels: + app: ecrag + template: + metadata: + labels: + app: ecrag + spec: + containers: + - name: ecrag + image: "{{ .Values.image.ecrag.registry }}/edgecraftrag:{{ .Values.image.ecrag.tag }}" + imagePullPolicy: IfNotPresent + envFrom: + - configMapRef: + name: edgecraftrag-env + ports: + - containerPort: {{ .Values.ports.mega }} + volumeMounts: + - name: model-path + mountPath: /home/user/models + - name: docs-path + mountPath: /home/user/docs + - name: tmpfile-path + mountPath: /home/user/ui_cache + - name: prompt-path + mountPath: /templates/custom + volumes: + - name: model-path + hostPath: + path: "{{ .Values.paths.model }}" + - name: docs-path + hostPath: + path: "{{ .Values.paths.docs }}" + - name: tmpfile-path + hostPath: + path: "{{ .Values.paths.tmpfile }}" + - name: prompt-path + hostPath: + path: "{{ .Values.paths.prompt }}" diff --git a/EdgeCraftRAG/kubernetes/helm/templates/deployment-edgecraftrag-ui.yaml b/EdgeCraftRAG/kubernetes/helm/templates/deployment-edgecraftrag-ui.yaml new file mode 100644 index 0000000000..ce4ab4018e --- /dev/null +++ b/EdgeCraftRAG/kubernetes/helm/templates/deployment-edgecraftrag-ui.yaml @@ -0,0 +1,45 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: edgecraftrag-ui +spec: + replicas: {{ .Values.replica.ecrag_ui }} + selector: + matchLabels: + app: edgecraftrag-ui + template: + metadata: + labels: + app: edgecraftrag-ui + spec: + containers: + - name: edgecraftrag-ui + image: "{{ .Values.image.ecrag.registry }}/edgecraftrag-ui:{{ .Values.image.ecrag.tag }}" + imagePullPolicy: IfNotPresent + envFrom: + - configMapRef: + name: edgecraftrag-env + ports: + - containerPort: {{ .Values.ports.ui.port }} + volumeMounts: + - name: model-path + mountPath: /home/user/models + - name: docs-path + mountPath: /home/user/docs + - name: tmpfile-path + mountPath: /home/user/ui_cache + - name: prompt-path + mountPath: /templates/custom + volumes: + - name: model-path + hostPath: + path: "{{ .Values.paths.model }}" + - name: docs-path + hostPath: + path: "{{ .Values.paths.docs }}" + - name: tmpfile-path + hostPath: + path: "{{ .Values.paths.tmpfile }}" + - name: prompt-path + hostPath: + path: "{{ .Values.paths.prompt }}" diff --git a/EdgeCraftRAG/kubernetes/helm/templates/service-ecrag.yaml b/EdgeCraftRAG/kubernetes/helm/templates/service-ecrag.yaml new file mode 100644 index 0000000000..93ee1d73d4 --- /dev/null +++ b/EdgeCraftRAG/kubernetes/helm/templates/service-ecrag.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Service +metadata: + name: ecrag +spec: + selector: + app: ecrag + ports: + - protocol: TCP + port: {{ .Values.ports.mega }} + targetPort: {{ .Values.ports.mega }} diff --git a/EdgeCraftRAG/kubernetes/helm/templates/service-edgecraftrag-server.yaml b/EdgeCraftRAG/kubernetes/helm/templates/service-edgecraftrag-server.yaml new file mode 100644 index 0000000000..6f04b40f20 --- /dev/null +++ b/EdgeCraftRAG/kubernetes/helm/templates/service-edgecraftrag-server.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Service +metadata: + name: edgecraftrag-server +spec: + selector: + app: edgecraftrag-server + ports: + - protocol: TCP + port: {{ .Values.ports.pipeline }} + targetPort: {{ .Values.ports.pipeline }} diff --git a/EdgeCraftRAG/kubernetes/helm/templates/service-edgecraftrag-ui.yaml b/EdgeCraftRAG/kubernetes/helm/templates/service-edgecraftrag-ui.yaml new file mode 100644 index 0000000000..cb02247dbe --- /dev/null +++ b/EdgeCraftRAG/kubernetes/helm/templates/service-edgecraftrag-ui.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Service +metadata: + name: edgecraftrag-ui +spec: + type: NodePort + selector: + app: edgecraftrag-ui + ports: + - protocol: TCP + port: {{ .Values.ports.ui.port }} + targetPort: {{ .Values.ports.ui.port }} + nodePort: {{ .Values.ports.ui.nodePort }} diff --git a/EdgeCraftRAG/kubernetes/helm/templates/service-llm-serving-xpu.yaml b/EdgeCraftRAG/kubernetes/helm/templates/service-llm-serving-xpu.yaml new file mode 100644 index 0000000000..e8bf327f7a --- /dev/null +++ b/EdgeCraftRAG/kubernetes/helm/templates/service-llm-serving-xpu.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Service +metadata: + name: llm-serving-xpu +spec: + selector: + app: llm-serving-xpu + ports: + - protocol: TCP + port: {{ .Values.ports.vllm }} + targetPort: {{ .Values.ports.vllm }} diff --git a/EdgeCraftRAG/kubernetes/helm/values.yaml b/EdgeCraftRAG/kubernetes/helm/values.yaml new file mode 100644 index 0000000000..203ce236f6 --- /dev/null +++ b/EdgeCraftRAG/kubernetes/helm/values.yaml @@ -0,0 +1,57 @@ +image: + ecrag: + registry: opea + tag: latest + vllm: + registry: intel + tag: 1.1-preview + +replica: + ecrag: 1 + ecrag_ui: 1 + +env: + no_proxy: "" + http_proxy: "" + https_proxy: "" + HOST_IP: "" + ENABLE_BENCHMARK: false + CHAT_HISTORY_ROUND: 0 + METADATA_DATABASE_URL: "" + +llm: + LLM_MODEL: "" + DTYPE: float16 + ZE_AFFINITY_MASK: 0,1 + ENFORCE_EAGER: 1 + TRUST_REMOTE_CODE: 1 + DISABLE_SLIDING_WINDOW: 1 + GPU_MEMORY_UTIL: 0.9 + NO_ENABLE_PREFIX_CACHING: 1 + MAX_NUM_BATCHED_TOKENS: 8192 + MAX_MODEL_LEN: 49152 + DISABLE_LOG_REQUESTS: 1 + BLOCK_SIZE: 64 + QUANTIZATION: sym_int4 + TP: 1 + DP: 1 + + +ports: + pipeline: 16010 + mega: 16011 + ui: + port: 8082 + nodePort: 30082 + vllm: 8086 + +paths: + model: /home/user/models + docs: /home/user/docs + tmpfile: /home/user/ui_cache + prompt: /templates/custom + +gpu: + groups: + video: 44 + render: 991 diff --git a/EdgeCraftRAG/tests/test_compose_on_arc.sh b/EdgeCraftRAG/tests/test_compose_on_arc.sh new file mode 100755 index 0000000000..cdb2cd7e31 --- /dev/null +++ b/EdgeCraftRAG/tests/test_compose_on_arc.sh @@ -0,0 +1,128 @@ +#!/bin/bash +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +set -e +source ./common.sh + +IMAGE_REPO=${IMAGE_REPO:-"opea"} +IMAGE_TAG=${IMAGE_TAG:-"latest"} +echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" +echo "TAG=IMAGE_TAG=${IMAGE_TAG}" +export REGISTRY=${IMAGE_REPO} +export TAG=${IMAGE_TAG} + +WORKPATH=$(dirname "$PWD") +LOG_PATH="$WORKPATH/tests" + +ip_address=$(hostname -I | awk '{print $1}') +HOST_IP=$ip_address + +COMPOSE_FILE="compose.yaml" +EC_RAG_SERVICE_PORT=16010 + +export MODEL_PATH="${HOME}/models" +# MODEL_PATH="$WORKPATH/models" +DOC_PATH="$WORKPATH/tests" +UI_UPLOAD_PATH="$WORKPATH/tests" +MAX_MODEL_LEN=5000 + +HF_ENDPOINT=https://hf-mirror.com + + +function build_docker_images() { + opea_branch=${opea_branch:-"main"} + cd $WORKPATH/docker_image_build + git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git + pushd GenAIComps + echo "GenAIComps test commit is $(git rev-parse HEAD)" + docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile . + popd && sleep 1s + + echo "Build all the images with --no-cache, check docker_image_build.log for details..." + service_list="edgecraftrag edgecraftrag-server edgecraftrag-ui" + docker compose -f build.yaml build --no-cache > ${LOG_PATH}/docker_image_build.log + + docker images && sleep 1s +} + +function start_services() { + export UI_UPLOAD_PATH=${UI_UPLOAD_PATH} + + cd $WORKPATH/docker_compose/intel/gpu/arc + source set_env.sh + # Start Docker Containers + docker compose -f $COMPOSE_FILE up -d > ${LOG_PATH}/start_services_with_compose.log + sleep 20 +} + +function validate_rag() { + cd $WORKPATH/tests + + # setup pipeline + validate_services \ + "${HOST_IP}:${EC_RAG_SERVICE_PORT}/v1/settings/pipelines" \ + "active" \ + "pipeline" \ + "edgecraftrag-server" \ + '@configs/test_pipeline_local_llm.json' + + # add data to knowledge base + validate_knowledge + + # query + validate_services \ + "${HOST_IP}:${EC_RAG_SERVICE_PORT}/v1/chatqna" \ + "1234567890" \ + "query" \ + "edgecraftrag-server" \ + '{"messages":"What is the test id?"}' +} + +function validate_megaservice() { + # Curl the Mega Service + validate_services \ + "${HOST_IP}:16011/v1/chatqna" \ + "1234567890" \ + "query" \ + "edgecraftrag-server" \ + '{"messages":"What is the test id?"}' +} + +function stop_docker() { + cd $WORKPATH/docker_compose/intel/gpu/arc + docker compose -f $COMPOSE_FILE down +} + + +function main() { + mkdir -p $LOG_PATH + + echo "::group::stop_docker" + stop_docker + echo "::endgroup::" + + echo "::group::build_docker_images" + if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi + echo "::endgroup::" + + echo "::group::start_services" + start_services + echo "::endgroup::" + + echo "::group::validate_rag" + validate_rag + echo "::endgroup::" + + echo "::group::validate_megaservice" + validate_megaservice + echo "::endgroup::" + + echo "::group::stop_docker" + stop_docker + echo y | docker system prune + echo "::endgroup::" + +} + +main diff --git a/EdgeCraftRAG/tests/test_compose_vllm_on_arc.sh b/EdgeCraftRAG/tests/test_compose_vllm_on_arc.sh new file mode 100755 index 0000000000..ada4cdb7da --- /dev/null +++ b/EdgeCraftRAG/tests/test_compose_vllm_on_arc.sh @@ -0,0 +1,168 @@ +#!/bin/bash +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +set -e +source ./common.sh + +IMAGE_REPO=${IMAGE_REPO:-"opea"} +IMAGE_TAG=${IMAGE_TAG:-"latest"} +echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" +echo "TAG=IMAGE_TAG=${IMAGE_TAG}" +export REGISTRY=${IMAGE_REPO} +export TAG=${IMAGE_TAG} + +WORKPATH=$(dirname "$PWD") +LOG_PATH="$WORKPATH/tests" + +ip_address=$(hostname -I | awk '{print $1}') +HOST_IP=$ip_address + +COMPOSE_FILE="compose.yaml" +EC_RAG_SERVICE_PORT=16010 + +MODEL_PATH="${HOME}/models" +# MODEL_PATH="$WORKPATH/models" +DOC_PATH="$WORKPATH/tests" +UI_UPLOAD_PATH="$WORKPATH/tests" + +HF_ENDPOINT=https://hf-mirror.com +VLLM_SERVICE_PORT_A770=8086 +TP=1 +vLLM_ENDPOINT="http://${HOST_IP}:${VLLM_SERVICE_PORT_A770}" +LLM_MODEL="Qwen/Qwen3-8B" +LLM_MODEL_PATH="${MODEL_PATH}/${LLM_MODEL}" +VLLM_IMAGE_TAG="0.8.3-b20" + +function build_docker_images() { + opea_branch=${opea_branch:-"main"} + cd $WORKPATH/docker_image_build + git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git + pushd GenAIComps + echo "GenAIComps test commit is $(git rev-parse HEAD)" + docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile . + popd && sleep 1s + + echo "Pull intelanalytics/ipex-llm-serving-xpu image" + docker pull intelanalytics/ipex-llm-serving-xpu:${VLLM_IMAGE_TAG} + + echo "Build all the images with --no-cache, check docker_image_build.log for details..." + docker compose -f build.yaml build --no-cache > ${LOG_PATH}/docker_image_build.log + + docker images && sleep 1s +} + +function start_services() { + cd $WORKPATH/docker_compose/intel/gpu/arc + source set_env.sh + # Start Docker Containers + docker compose --profile a770 -f $COMPOSE_FILE up -d > ${LOG_PATH}/start_services_with_compose.log + echo "ipex-llm-serving-xpu is booting, please wait." + sleep 30s + n=0 + until [[ "$n" -ge 100 ]]; do + docker logs ipex-llm-serving-xpu-770 > ${LOG_PATH}/ipex-llm-serving-xpu-container.log 2>&1 + if grep -q "Starting vLLM API server on http://0.0.0.0:" ${LOG_PATH}/ipex-llm-serving-xpu-container.log; then + break + fi + sleep 6s + n=$((n+1)) + done +} + +function validate_services() { + local URL="$1" + local EXPECTED_RESULT="$2" + local SERVICE_NAME="$3" + local DOCKER_NAME="$4" + local INPUT_DATA="$5" + + echo "[ $SERVICE_NAME ] Validating $SERVICE_NAME service..." + local RESPONSE=$(curl -s -w "%{http_code}" -o ${LOG_PATH}/${SERVICE_NAME}.log -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL") + while [ ! -f ${LOG_PATH}/${SERVICE_NAME}.log ]; do + sleep 1 + done + local HTTP_STATUS="${RESPONSE: -3}" + local CONTENT=$(cat ${LOG_PATH}/${SERVICE_NAME}.log) + + if [ "$HTTP_STATUS" -eq 200 ]; then + echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..." + else + echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS" + docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log + exit 1 + fi + sleep 1s +} + +function validate_rag() { + cd $WORKPATH/tests + + # setup pipeline + validate_services \ + "${HOST_IP}:${EC_RAG_SERVICE_PORT}/v1/settings/pipelines" \ + "active" \ + "pipeline" \ + "edgecraftrag-server" \ + '@configs/test_pipeline_ipex_vllm.json' + + # add data to knowledge base + validate_knowledge + + # query + validate_services \ + "${HOST_IP}:${EC_RAG_SERVICE_PORT}/v1/chatqna" \ + "1234567890" \ + "query" \ + "ipex-llm-serving-xpu-770" \ + '{"messages":"What is the test id?","max_tokens":5}' +} + +function validate_megaservice() { + # Curl the Mega Service + validate_services \ + "${HOST_IP}:16011/v1/chatqna" \ + "1234567890" \ + "query" \ + "ipex-llm-serving-xpu-770" \ + '{"messages":"What is the test id?","max_tokens":5}' +} + +function stop_docker() { + cd $WORKPATH/docker_compose/intel/gpu/arc + export MODEL_PATH="${HOME}/models" + docker compose -f $COMPOSE_FILE down +} + + +function main() { + mkdir -p $LOG_PATH + + echo "::group::stop_docker" + stop_docker + echo "::endgroup::" + + echo "::group::build_docker_images" + if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi + echo "::endgroup::" + + echo "::group::start_services" + start_services + echo "::endgroup::" + + echo "::group::validate_rag" + validate_rag + echo "::endgroup::" + + echo "::group::validate_megaservice" + validate_megaservice + echo "::endgroup::" + + echo "::group::stop_docker" + stop_docker + echo y | docker system prune + echo "::endgroup::" + +} + +main diff --git a/EdgeCraftRAG/tests/test_compose_vllm_on_arc_b60.sh b/EdgeCraftRAG/tests/test_compose_vllm_on_arc_b60.sh new file mode 100755 index 0000000000..d7245a9ca9 --- /dev/null +++ b/EdgeCraftRAG/tests/test_compose_vllm_on_arc_b60.sh @@ -0,0 +1,169 @@ +#!/bin/bash +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +set -e +source ./common.sh + +IMAGE_REPO=${IMAGE_REPO:-"opea"} +IMAGE_TAG=${IMAGE_TAG:-"latest"} +echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" +echo "TAG=IMAGE_TAG=${IMAGE_TAG}" +export REGISTRY=${IMAGE_REPO} +export TAG=${IMAGE_TAG} + +WORKPATH=$(dirname "$PWD") +LOG_PATH="$WORKPATH/tests" + +ip_address=$(hostname -I | awk '{print $1}') +HOST_IP=$ip_address + +COMPOSE_FILE="compose.yaml" +EC_RAG_SERVICE_PORT=16010 + +MODEL_PATH="${HOME}/models" +# MODEL_PATH="$WORKPATH/models" +DOC_PATH="$WORKPATH/tests" +UI_UPLOAD_PATH="$WORKPATH/tests" + +HF_ENDPOINT=https://hf-mirror.com +VLLM_SERVICE_PORT_B60=8086 +TP=1 +vLLM_ENDPOINT="http://${HOST_IP}:${VLLM_SERVICE_PORT_B60}" +LLM_MODEL="Qwen/Qwen3-8B" +VLLM_IMAGE_TAG="1.1-preview" +DP=1 +ZE_AFFINITY_MASK=1 + +function build_docker_images() { + opea_branch=${opea_branch:-"main"} + cd $WORKPATH/docker_image_build + git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git + pushd GenAIComps + echo "GenAIComps test commit is $(git rev-parse HEAD)" + docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile . + popd && sleep 1s + + echo "Pull intel/llm-scaler-vllm image" + docker pull intel/llm-scaler-vllm:${VLLM_IMAGE_TAG} + + echo "Build all the images with --no-cache, check docker_image_build.log for details..." + docker compose -f build.yaml build --no-cache > ${LOG_PATH}/docker_image_build.log + + docker images && sleep 1s +} + +function start_services() { + cd $WORKPATH/docker_compose/intel/gpu/arc + source set_env.sh + # Start Docker Containers + docker compose --profile b60 -f $COMPOSE_FILE up -d > ${LOG_PATH}/start_services_with_compose.log + echo "ipex-serving-xpu is booting, please wait." + sleep 30s + n=0 + until [[ "$n" -ge 100 ]]; do + docker logs ipex-serving-xpu-container > ${LOG_PATH}/ipex-serving-xpu-container.log 2>&1 + if grep -q "Starting vLLM API server" ${LOG_PATH}/ipex-serving-xpu-container.log; then + break + fi + sleep 6s + n=$((n+1)) + done +} + +function validate_services() { + local URL="$1" + local EXPECTED_RESULT="$2" + local SERVICE_NAME="$3" + local DOCKER_NAME="$4" + local INPUT_DATA="$5" + + echo "[ $SERVICE_NAME ] Validating $SERVICE_NAME service..." + local RESPONSE=$(curl -s -w "%{http_code}" -o ${LOG_PATH}/${SERVICE_NAME}.log -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL") + while [ ! -f ${LOG_PATH}/${SERVICE_NAME}.log ]; do + sleep 1 + done + local HTTP_STATUS="${RESPONSE: -3}" + local CONTENT=$(cat ${LOG_PATH}/${SERVICE_NAME}.log) + + if [ "$HTTP_STATUS" -eq 200 ]; then + echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..." + else + echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS" + docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log + exit 1 + fi + sleep 1s +} + +function validate_rag() { + cd $WORKPATH/tests + + # setup pipeline + validate_services \ + "${HOST_IP}:${EC_RAG_SERVICE_PORT}/v1/settings/pipelines" \ + "active" \ + "pipeline" \ + "edgecraftrag-server" \ + '@configs/test_pipeline_ipex_vllm.json' + + # add data to knowledge base + validate_knowledge + + # query + validate_services \ + "${HOST_IP}:${EC_RAG_SERVICE_PORT}/v1/chatqna" \ + "1234567890" \ + "query" \ + "ipex-serving-xpu-container" \ + '{"messages":"What is the test id?","max_tokens":5}' +} + +function validate_megaservice() { + # Curl the Mega Service + validate_services \ + "${HOST_IP}:16011/v1/chatqna" \ + "1234567890" \ + "query" \ + "ipex-serving-xpu-container" \ + '{"messages":"What is the test id?","max_tokens":5}' +} + +function stop_docker() { + cd $WORKPATH/docker_compose/intel/gpu/arc + export MODEL_PATH="${HOME}/models" + docker compose -f $COMPOSE_FILE down +} + + +function main() { + mkdir -p $LOG_PATH + + echo "::group::stop_docker" + stop_docker + echo "::endgroup::" + + echo "::group::build_docker_images" + if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi + echo "::endgroup::" + + echo "::group::start_services" + start_services + echo "::endgroup::" + + echo "::group::validate_rag" + validate_rag + echo "::endgroup::" + + echo "::group::validate_megaservice" + validate_megaservice + echo "::endgroup::" + + echo "::group::stop_docker" + stop_docker + echo y | docker system prune + echo "::endgroup::" + +} + +main diff --git a/EdgeCraftRAG/tools/README.md b/EdgeCraftRAG/tools/README.md index e5fde632bd..0f53bd6734 100644 --- a/EdgeCraftRAG/tools/README.md +++ b/EdgeCraftRAG/tools/README.md @@ -300,6 +300,7 @@ Examples: ```bash ./tools/model_download.sh vllm ./tools/model_download.sh ov Qwen/Qwen3-8B /data/models +./tools/model_download.sh ov OpenVINO/Qwen3-8B-int4-ov /data/models ``` Environment behavior: @@ -307,6 +308,13 @@ Environment behavior: - if a virtual environment is already active, it is reused - otherwise, the script creates/activates `ecrag_venv` automatically (same style as `quick_start.sh`) - missing `python3-venv` / `pip` prerequisites are installed automatically when supported by the system package manager +- `DOWNLOAD_EMBEDDING_RERANKER` controls embedding/reranker download checks: + - `1` (default): keep current embedding/reranker download/check logic + - `0`: skip embedding/reranker download checks (LLM processing still runs based on mode) + +```bash +DOWNLOAD_EMBEDDING_RERANKER=0 ./tools/model_download.sh ov OpenVINO/Qwen3-8B-int4-ov /data/models +``` ## 3.2 Direct Startup Scripts diff --git a/EdgeCraftRAG/tools/README_zh.md b/EdgeCraftRAG/tools/README_zh.md index da4c2d34c4..3ec1a1d27f 100644 --- a/EdgeCraftRAG/tools/README_zh.md +++ b/EdgeCraftRAG/tools/README_zh.md @@ -287,6 +287,7 @@ source workspace/bootstrap.env ```bash ./tools/model_download.sh vllm ./tools/model_download.sh ov Qwen/Qwen3-8B /data/models +./tools/model_download.sh ov OpenVINO/Qwen3-8B-int4-ov /data/models ``` 环境行为说明： @@ -294,6 +295,13 @@ source workspace/bootstrap.env - 若当前已激活虚拟环境，会优先复用 - 若未激活虚拟环境，脚本会自动创建并激活 `ecrag_venv`（与 `quick_start.sh` 一致） - 若缺失 `python3-venv` 或 `pip`，脚本会在支持的包管理器上自动安装所需前置依赖 +- `DOWNLOAD_EMBEDDING_RERANKER` 用于控制 embedding/reranker 下载检查： + - `1`（默认）：保持当前 embedding/reranker 下载/检查逻辑 + - `0`：跳过 embedding/reranker 下载检查（LLM 仍按当前 mode 执行） + +```bash +DOWNLOAD_EMBEDDING_RERANKER=0 ./tools/model_download.sh ov OpenVINO/Qwen3-8B-int4-ov /data/models +``` ## 3.2 直接启动脚本 diff --git a/EdgeCraftRAG/tools/model_download.sh b/EdgeCraftRAG/tools/model_download.sh index ae167ae016..00c9ea430f 100755 --- a/EdgeCraftRAG/tools/model_download.sh +++ b/EdgeCraftRAG/tools/model_download.sh @@ -561,6 +561,22 @@ ensure_llm_model_for_ov() { fi echo "[Model Check] OpenVINO LLM model missing: ${ov_llm_dir}" + + # If user passes an OpenVINO model id/name (contains 'OpenVINO'), + # download it directly to the final OV target directory and skip conversion. + if [[ "${LLM_MODEL}" == *OpenVINO* ]]; then + echo "[Model Check] Detected OpenVINO model id/name '${LLM_MODEL}', downloading directly..." + download_model "${LLM_MODEL}" "${ov_llm_dir}" + + if ! openvino_model_exists "${ov_llm_dir}"; then + echo "[Model Check] ERROR: Download completed but OpenVINO model is incomplete: ${ov_llm_dir}" + exit 1 + fi + + echo "[Model Check] OpenVINO LLM model ready: ${ov_llm_dir}" + return 0 + fi + resolved_llm_src_dir=$(prepare_source_model "${LLM_MODEL}" "${llm_src_dir}" "${SOURCE_MODEL_PATH}") echo "[Model Check] Converting LLM model '${LLM_MODEL}' to ${OV_CONVERSION_METHOD^^} OpenVINO..." ensure_openvino_tooling diff --git a/EdgeCraftRAG/ui/docker/Dockerfile.ui b/EdgeCraftRAG/ui/docker/Dockerfile.ui index 1296f51c98..5956e6f9fb 100644 --- a/EdgeCraftRAG/ui/docker/Dockerfile.ui +++ b/EdgeCraftRAG/ui/docker/Dockerfile.ui @@ -19,7 +19,10 @@ FROM nginx:stable-alpine COPY --from=vue-app /home/user/ui/dist /usr/share/nginx/html -COPY ./ui/vue/nginx.conf /etc/nginx/nginx.conf +COPY ./ui/docker/nginx.conf /etc/nginx/nginx.conf.template +COPY ./ui/docker/replace-env.sh /docker-entrypoint.d/99-render-nginx-conf.sh + +RUN chmod +x /docker-entrypoint.d/99-render-nginx-conf.sh EXPOSE 8082 diff --git a/EdgeCraftRAG/ui/vue/nginx.conf b/EdgeCraftRAG/ui/docker/nginx.conf similarity index 79% rename from EdgeCraftRAG/ui/vue/nginx.conf rename to EdgeCraftRAG/ui/docker/nginx.conf index 8b6701e78a..9e91bdeec0 100644 --- a/EdgeCraftRAG/ui/vue/nginx.conf +++ b/EdgeCraftRAG/ui/docker/nginx.conf @@ -14,7 +14,7 @@ http { keepalive_timeout 90; server { - listen 8082; + listen ${UI_SERVICE_PORT}; server_name _; location / { @@ -25,7 +25,7 @@ http { } location /v1/ { - proxy_pass http://edgecraftrag-server:16010; + proxy_pass http://${PIPELINE_SERVICE_HOST_IP}:${PIPELINE_SERVICE_PORT}; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_http_version 1.1; proxy_read_timeout 600s; @@ -33,14 +33,14 @@ http { } location /home/user/ { - proxy_pass http://edgecraftrag-server:16010; + proxy_pass http://${PIPELINE_SERVICE_HOST_IP}:${PIPELINE_SERVICE_PORT}; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_http_version 1.1; proxy_set_header Connection ""; } location /v1/chatqna { - proxy_pass http://ecrag:16011; + proxy_pass http://${MEGA_SERVICE_HOST_IP}:${MEGA_SERVICE_PORT}; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_http_version 1.1; proxy_set_header Connection ""; @@ -48,7 +48,7 @@ http { proxy_cache off; proxy_read_timeout 600s; gzip off; - proxy_set_header Host ecrag:16011; + proxy_set_header Host ${MEGA_SERVICE_HOST_IP}:${MEGA_SERVICE_PORT}; proxy_set_header X-Accel-Buffering no; proxy_set_header X-Real-IP $remote_addr; chunked_transfer_encoding on; diff --git a/EdgeCraftRAG/ui/docker/replace-env.sh b/EdgeCraftRAG/ui/docker/replace-env.sh new file mode 100644 index 0000000000..96053bec3a --- /dev/null +++ b/EdgeCraftRAG/ui/docker/replace-env.sh @@ -0,0 +1,27 @@ +#!/bin/sh + +set -eu + +NGINX_TEMPLATE="/etc/nginx/nginx.conf.template" +NGINX_CONF="/etc/nginx/nginx.conf" + +: "${UI_SERVICE_PORT:=8082}" +: "${PIPELINE_SERVICE_HOST_IP:=edgecraftrag-server}" +: "${PIPELINE_SERVICE_PORT:=16010}" +: "${MEGA_SERVICE_HOST_IP:=ecrag}" +: "${MEGA_SERVICE_PORT:=16011}" + +export UI_SERVICE_PORT +export PIPELINE_SERVICE_HOST_IP +export PIPELINE_SERVICE_PORT +export MEGA_SERVICE_HOST_IP +export MEGA_SERVICE_PORT + +envsubst '${UI_SERVICE_PORT} ${PIPELINE_SERVICE_HOST_IP} ${PIPELINE_SERVICE_PORT} ${MEGA_SERVICE_HOST_IP} ${MEGA_SERVICE_PORT}' \ + < "$NGINX_TEMPLATE" \ + > "$NGINX_CONF" + +if grep -n '\${[A-Za-z_][A-Za-z0-9_]*}' "$NGINX_CONF"; then + echo "[replace-env] Unresolved nginx template variables remain in $NGINX_CONF" >&2 + exit 1 +fi \ No newline at end of file diff --git a/EdgeCraftRAG/ui/vue/src/i18n/en.ts b/EdgeCraftRAG/ui/vue/src/i18n/en.ts index addcee16be..8ad229933d 100644 --- a/EdgeCraftRAG/ui/vue/src/i18n/en.ts +++ b/EdgeCraftRAG/ui/vue/src/i18n/en.ts @@ -42,6 +42,7 @@ export default { no: "No", manual: "Create Now", inputTip: "Please enter ", + jsonInvalid: "Please enter a valid JSON object or array", connect: "Connect", detail: "View details", retry: "Retry", diff --git a/EdgeCraftRAG/ui/vue/src/i18n/zh.ts b/EdgeCraftRAG/ui/vue/src/i18n/zh.ts index 0e8a914a6a..ab600e0040 100644 --- a/EdgeCraftRAG/ui/vue/src/i18n/zh.ts +++ b/EdgeCraftRAG/ui/vue/src/i18n/zh.ts @@ -41,6 +41,7 @@ export default { no: "否", manual: "去创建", inputTip: "请输入 ", + jsonInvalid: "请输入有效的 JSON 对象或数组", connect: "连接", detail: "查看详情", retry: "重试", diff --git a/EdgeCraftRAG/ui/vue/src/views/chatbot/components/Chatbot/Chat.vue b/EdgeCraftRAG/ui/vue/src/views/chatbot/components/Chatbot/Chat.vue index 0f66cf8bb4..50114f70c7 100644 --- a/EdgeCraftRAG/ui/vue/src/views/chatbot/components/Chatbot/Chat.vue +++ b/EdgeCraftRAG/ui/vue/src/views/chatbot/components/Chatbot/Chat.vue @@ -10,7 +10,6 @@ ([]); const inputKeywords = ref(""); const scrollContainer = ref(null); const messageComponent = ref(null); -let resizeObserver: ResizeObserver | null = null; -const messageRef = ref(null); const inResponse = ref(false); const imgVisible = ref(false); const imageSrc = ref(""); @@ -162,6 +159,7 @@ const resizeObserverRef = ref(null); const enableKB = ref(true); const isCreatingNewSession = ref(false); const shouldIgnoreRouteChange = ref(false); +let throttledHandleScroll: ReturnType | null = null; const inputRef = ref(); const handleEnvUrl = () => { @@ -455,7 +453,13 @@ const initResizeObserver = () => { resizeObserverRef.value = new ResizeObserver(handleResize); resizeObserverRef.value.observe(messageComponent.value); - const throttledHandleScroll = throttle(handleScroll, 100); + if (throttledHandleScroll) { + scrollContainer.value?.removeEventListener( + "scroll", + throttledHandleScroll, + ); + } + throttledHandleScroll = throttle(handleScroll, 100); scrollContainer.value?.addEventListener("scroll", throttledHandleScroll); } @@ -536,11 +540,15 @@ onMounted(() => { }); onBeforeUnmount(() => { - if (resizeObserver && messageComponent.value) { - resizeObserver.unobserve(messageComponent.value); - resizeObserver = null; + if (resizeObserverRef.value) { + resizeObserverRef.value.disconnect(); + resizeObserverRef.value = null; + } + if (throttledHandleScroll) { + scrollContainer.value?.removeEventListener("scroll", throttledHandleScroll); + throttledHandleScroll.cancel(); + throttledHandleScroll = null; } - scrollContainer.value?.removeEventListener("scroll", handleScroll); }); onUnmounted(() => { diff --git a/EdgeCraftRAG/ui/vue/src/views/chatbot/components/Chatbot/MessageItem.vue b/EdgeCraftRAG/ui/vue/src/views/chatbot/components/Chatbot/MessageItem.vue index 5dd96b1174..1e459aa72a 100644 --- a/EdgeCraftRAG/ui/vue/src/views/chatbot/components/Chatbot/MessageItem.vue +++ b/EdgeCraftRAG/ui/vue/src/views/chatbot/components/Chatbot/MessageItem.vue @@ -1,5 +1,5 @@