DEVOCEAN

from langserve import RemoteRunnable

# Request query to Remote LLM using FastAPI
def generate_response_remote(query):    
    # single-query retriever 셋팅    
    retriever = load_embedding_singlequery()
    
    #Set-up remote LLM    
    llm = RemoteRunnable(url="http://localhost:8000/llm")
    template = """Use the following pieces of information to answer the user's question.    
    If you don't know the answer, just say that you don't know, don't try to make up an answer.    
    Context: {context}    
    Question: {question}    
    Only return the Korean answer below and nothing else.    
    Korean answer:    """    
    prompt = PromptTemplate(        
        template=template,        
        input_variables=['context', 'question'])

    qa_chain = (        
        {            
            'context': retriever,            
            'question': RunnablePassthrough()        
        }        
        | prompt        
        | llm    
    )
    
    query = query    
    print('input query: ' + query)    
    #generate answer    
    output = qa_chain.invoke(query)    
    print(output)    
    return output

st.title("OpenLLM AI Chatbot")
user_input = st.text_input("여기에 질문을 입력하세요")
if user_input:    
    #response = generate_response(user_input)    
    #response = generate_response_rerank(user_input)    
    #response = generate_response_multiquery(user_input)
    response = generate_response_remote(user_input)
    
    st.write("AI Chatbot 답변: ")    
    st.write(response)

from fastapi import FastAPI
from fastapi.responses import RedirectResponse
from langserve import add_routes
from fastapi.middleware.cors import CORSMiddleware
from typing import List, Union
from langserve.pydantic_v1 import BaseModel, Field
from langchain_core.messages import HumanMessage, AIMessage, SystemMessage
# user-defined llm.py
from llm import llm as chain

# Set FastAPI
app = FastAPI()

# Set Cross-Origin Resource Sharing(CORS) config for FastAPI
app.add_middleware(    
    CORSMiddleware,    
    allow_origins=["*"],    
    allow_credentials=True,    
    allow_methods=["*"],    
    allow_headers=["*"],    
    expose_headers = ["*"],
)

# Re-direct to llm server when get event is triggered via chrome browser
@app.get("/")
async def redirect_root_to_llmserver():    
    return RedirectResponse("/llm/playground")

# Set routes to call llm with LlamaCpp located in llm.py
add_routes(app, chain, path="/llm")

if __name__ == "__main__":    
    import uvicorn

    uvicorn.run(app, host="0.0.0.0", port=8000)

from langchain_community.llms import LlamaCpp
# LLM 경로 지정
MODEL_PATH = "C:\Work\GenAI\FoundationModel\llama-2–7b-chat\llama-2-7b-chat.Q4_K_M.gguf"
#MODEL_PATH = "C:\Work\GenAI\FoundationModel\Llama-3-8B-Instruct\Meta-Llama-3-8B-Instruct.Q4_K_M.gguf"

llm = LlamaCpp(    
    model_path=MODEL_PATH,    
    temperature=0.2,    
    top_p=1,    
    max_tokens=3072,    
    verbose=True,    
    n_ctx=3072,
)