DEVOCEAN

import os
from docx import Document
import glob

# 검색할 디렉토리 경로
directory_path = '...'

# 지정된 디렉토리에서 모든 .docx 파일 찾기
docx_files = glob.glob(os.path.join(directory_path, '*.docx'))

# 새로운 Document 객체 생성 (모든 문서를 병합할 객체)
merged_document = Document()

# 각 DOCX 파일에 대해 반복
for file_path in docx_files:
    # 문서 열기
    document = Document(file_path)

    # 현재 문서의 모든 문단에 대해 반복
    for para in document.paragraphs:
        # 새 문서에 문단 추가
        merged_document.add_paragraph(para.text)

# 병합된 문서 저장
merged_document.save("/merged_document.docx")
print(f"해당 텍스트는 {len(merged_document.paragraphs)} paragraphs가 있습니다.")

# 문서의 각 문단을 순회하며 텍스트 출력
import tiktoken
import pandas as pd
from tqdm import tqdm

fgd_text = ''
for para in tqdm(merged_document.paragraphs):
    text = para.text
    fgd_text += text
print('문서를 모두 읽어왔습니다!')

from langchain.text_splitter import RecursiveCharacterTextSplitter

text_splitter = RecursiveCharacterTextSplitter(
      chunk_size = 512
      ,chunk_overlap = 100
)

documents = text_splitter.split_text(fgd_text)
documents_content = text_splitter.create_documents(documents)

from langchain_openai import OpenAIEmbeddings
from langchain.vectorstores import Chroma
Chroma().delete_collection() # 메모리에 로드된 기존 데이터 삭제

embedding = OpenAIEmbeddings(model='text-embedding-3-large')
# vectorstore = Chroma.from_texts(documents, embedding = embedding,persist_directory = 'db')
vectorstore = Chroma.from_documents(documents_content, embedding = embedding,persist_directory = 'db')
vectorstore.persist()

from langchain.prompts import ChatPromptTemplate
from langchain_openai import ChatOpenAI
from langchain import hub

from langchain.schema.runnable import RunnablePassthrough
from langchain.schema.output_parser import StrOutputParser

prompt = hub.pull("rlm/rag-prompt")

llm = ChatOpenAI(model_name='gpt-4-turbo-preview', temperature=0.1)

def format_docs(docs):
    return "\n---\n".join(doc.page_content for doc in docs)
    # join : 구분자를 기준으로 스트링 리스트를 하나의 스트링으로 연결

rag_chain = (
    {"context": retriever | format_docs, "question": RunnablePassthrough()}
    # retriever : question을 받아서 context 검색: document 반환
    # format_docs : document 형태를 받아서 텍스트로 변환
    # RunnablePassthrough(): 체인의 입력을 그대로 저장
    | prompt
    | llm
    | StrOutputParser()
)

from langchain.storage import InMemoryStore
from langchain.schema import Document
from langchain.retrievers import ParentDocumentRetriever
from langchain.text_splitter import RecursiveCharacterTextSplitter

embedding = OpenAIEmbeddings(model='text-embedding-3-large')

# 큰 청크와 작은 청크 분리
child_splitter = RecursiveCharacterTextSplitter(chunk_size=150, chunk_overlap=20)
parent_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=100)

Chroma().delete_collection() # 메모리에 로드된 기존 데이터 삭제

vectorstore = Chroma(embedding_function = embedding)
store = InMemoryStore()

parent_child_retriever = ParentDocumentRetriever(
    vectorstore = vectorstore,
    docstore = store,
    child_splitter = child_splitter,
    parent_splitter = parent_splitter
)

rag_chain = (
    {"context": parent_child_retriever | format_docs, "question": RunnablePassthrough()}
    | prompt
    | llm
    | StrOutputParser()
)

questions = [
    "FGD에 참석한 고객들은 자급제 단말에 대해 어떻게 생각하나요?",
    "자급제 단말은 SKT 유지/해지에 어떤 영향을 주나요?",
    "멤버십 혜택은 SKT 유지/해지에 어떤 영향을 주나요?"

]
result = rag_chain.batch(questions)
for i, ans in enumerate(result):
    ans = ans.replace('.','.\n')
    print(f"Question: {questions[i]}")
    print(f"Answer: {ans}")
    print('---')

from langchain.retrievers import BM25Retriever, EnsembleRetriever


bm25_retriever = BM25Retriever.from_documents(pages)
bm25_retriever.k = 5

vectorstore = Chroma.from_documents(pages, embedding)
retriever = vectorstore.as_retriever()

ensemble_retriever = EnsembleRetriever(
    retrievers=[bm25_retriever, retriever], weights=[0.5, 0.5]
)

## 요약

from langchain.chains import MapReduceDocumentsChain, ReduceDocumentsChain
from langchain_text_splitters import CharacterTextSplitter
from langchain.chains.combine_documents.stuff import StuffDocumentsChain
from langchain.chains.llm import LLMChain
from langchain_core.prompts import PromptTemplate
from langchain import hub

llm = ChatOpenAI(temperature=0, model = 'gpt-4')

# Map
map_template = """다음의 문서들이 있다
{docs}
이 문서들을 바탕으로, 주요 주제들을 파악하라

답변:"""
map_prompt = PromptTemplate.from_template(map_template)
map_chain = LLMChain(llm=llm, prompt=map_prompt)

# Reduce
reduce_template = """다음은 요약문들이다:
{docs}
이를 바탕으로 주요 주제에 대한 최종 통합 요약으로 정제하라.

답변:"""
reduce_prompt = PromptTemplate.from_template(reduce_template)
# Run chain
reduce_chain = LLMChain(llm=llm, prompt=reduce_prompt)

# Takes a list of documents, combines them into a single string, and passes this to an LLMChain
combine_documents_chain = StuffDocumentsChain(
    llm_chain=reduce_chain, document_variable_name="docs"
)

# Combines and iteratively reduces the mapped documents
reduce_documents_chain = ReduceDocumentsChain(
    # This is final chain that is called.
    combine_documents_chain=combine_documents_chain,
    # If documents exceed context for `StuffDocumentsChain`
    collapse_documents_chain=combine_documents_chain,
    # The maximum number of tokens to group documents into.
    token_max=4000,
)

combine_documents_chain = StuffDocumentsChain(
    llm_chain=reduce_chain, document_variable_name="docs"
)

reduce_documents_chain = ReduceDocumentsChain(
    combine_documents_chain=combine_documents_chain,
    collapse_documents_chain=combine_documents_chain,
    token_max=4000,
)

from langchain.text_splitter import RecursiveCharacterTextSplitter
map_reduce_chain = MapReduceDocumentsChain(
    llm_chain=map_chain,
    reduce_documents_chain=reduce_documents_chain,
    document_variable_name="docs",
    return_intermediate_steps=False,
)

text_splitter = CharacterTextSplitter.from_tiktoken_encoder(
    chunk_size = 2000, chunk_overlap = 100
    ,separator = '.'
)

split_docs = text_splitter.split_text(fgd_text)
documents_content = text_splitter.create_documents(split_docs)