DEVOCEAN

pipenv install

!cat ../Pipfile

from dotenv import load_dotenv
import os

load_dotenv(verbose=True)

# Create your own .env file in root path and add OPENAI_API_KEY
os.environ['OPENAI_API_KEY'] = os.getenv("OPENAI_API_KEY")

from langchain_community.document_loaders import WebBaseLoader
from langchain_text_splitters import RecursiveCharacterTextSplitter
from langchain_openai import OpenAIEmbeddings
from langchain_community.vectorstores import FAISS
import os

embeddings = OpenAIEmbeddings(model="text-embedding-ada-002")

loader = WebBaseLoader(
    "https://blog.langchain.dev/langchain-v0-1-0/"
)
documents = loader.load()

for document in documents:
    document.metadata['filename'] = document.metadata['source']


# save & load data
vdb_path = "db/faiss/langchain_docs"
if(os.path.exists(vdb_path)):
    print(f'load faiss {vdb_path}')
    vector_store = FAISS.load_local(vdb_path, embeddings, allow_dangerous_deserialization=True)  
else:     
    text_splitter = RecursiveCharacterTextSplitter(
        chunk_size = 1250,
        chunk_overlap = 100,
        length_function = len,
        is_separator_regex = False
    )
    split_docs = text_splitter.split_documents(documents)

    vector_store = FAISS.from_documents(split_docs, embeddings)
    vector_store.save_local(vdb_path)

load faiss db/faiss/langchain_docs

from operator import itemgetter

from langchain import hub
from langchain_core.runnables import RunnablePassthrough
from langchain_core.output_parsers import StrOutputParser
from langchain_community.chat_models import ChatOpenAI

retriever = vector_store.as_retriever()
prompt = hub.pull("rlm/rag-prompt")
llm = ChatOpenAI(model_name="gpt-3.5-turbo", temperature=0)

def format_docs(args):
    docs = args["source_docs"]
    return "\n\n".join(doc.page_content for doc in docs)

rag_chain = (
    RunnablePassthrough.assign(
        question=itemgetter("question")
    )
    | {
        "source_docs" : itemgetter("question") | retriever,
        "question": itemgetter("question"),
    }
    | {
        "source_docs": itemgetter("source_docs"), 
        "answer": {
            "context": format_docs,
            "question": itemgetter("question"),        
        } | prompt | llm | StrOutputParser()
    }
)

rag_chain.invoke({"question" : "What are the major changes in v 0.1.0?"})

/Users/1111999/.local/share/virtualenvs/llmops-ESENPtay/lib/python3.11/site-packages/langchain_core/_api/deprecation.py:119: LangChainDeprecationWarning: The class `ChatOpenAI` was deprecated in LangChain 0.0.10 and will be removed in 0.3.0. An updated version of the class exists in the langchain-openai package and should be used instead. To use it run `pip install -U langchain-openai` and import as `from langchain_openai import ChatOpenAI`.
  warn_deprecated(





{'source_docs': [Document(page_content='versioning policy for a little over a month now.langchain itself, however, still remained on 0.0.x versions. Having all releases on minor version 0 created a few challenges:Users couldn’t be confident that updating would not have breaking changeslangchain became bloated and unstable as we took a “maintain everything” approach to reduce breaking changes and deprecation notificationsHowever, starting today with the release of langchain 0.1.0, all future releases will follow a new versioning standard. Specifically:Any breaking changes to the public API will result in a minor version bump (the second digit)Any bug fixes or new features will result in a patch version bump (the third digit)We hope that this, combined with the previous architectural changes, will:Communicate clearly if breaking changes are made, allowing developers to update with confidenceGive us an avenue for officially deprecating and deleting old code, reducing bloatMore responsibly deal with integrations (whose SDKs are often changing as rapidly as LangChain)Even after we release a 0.2 version, we will commit to maintaining a branch of 0.1, but will only patch critical bug fixes. See more towards the end of this post on our plans for that.While re-architecting the', metadata={'source': 'https://blog.langchain.dev/langchain-v0-1-0/', 'title': 'LangChain v0.1.0', 'language': 'en', 'filename': 'https://blog.langchain.dev/langchain-v0-1-0/'}, _lc_kwargs={'page_content': 'versioning policy for a little over a month now.langchain itself, however, still remained on 0.0.x versions. Having all releases on minor version 0 created a few challenges:Users couldn’t be confident that updating would not have breaking changeslangchain became bloated and unstable as we took a “maintain everything” approach to reduce breaking changes and deprecation notificationsHowever, starting today with the release of langchain 0.1.0, all future releases will follow a new versioning standard. Specifically:Any breaking changes to the public API will result in a minor version bump (the second digit)Any bug fixes or new features will result in a patch version bump (the third digit)We hope that this, combined with the previous architectural changes, will:Communicate clearly if breaking changes are made, allowing developers to update with confidenceGive us an avenue for officially deprecating and deleting old code, reducing bloatMore responsibly deal with integrations (whose SDKs are often changing as rapidly as LangChain)Even after we release a 0.2 version, we will commit to maintaining a branch of 0.1, but will only patch critical bug fixes. See more towards the end of this post on our plans for that.While re-architecting the', 'metadata': {'source': 'https://blog.langchain.dev/langchain-v0-1-0/', 'title': 'LangChain v0.1.0', 'language': 'en', 'filename': 'https://blog.langchain.dev/langchain-v0-1-0/'}}),
  Document(page_content='Today we’re excited to announce the release of langchain 0.1.0, our first stable version. It is fully backwards compatible, comes in both Python and JavaScript, and comes with improved focus through both functionality and documentation. A stable version of LangChain helps us earn developer trust and gives us the ability to evolve the library systematically and safely.Python GitHub DiscussionPython v0.1.0 GuidesJS v0.1.0 GuidesYouTube WalkthroughIntroductionLangChain has been around for a little over a year and has changed a lot as it’s grown to become the default framework for building LLM applications. As we previewed a month ago, we recently decided to make significant changes to the\xa0 LangChain package architecture in order to better organize the project and strengthen the foundation.\xa0Specifically we made two large architectural changes: separating out langchain-core and separating out partner packages (either into langchain-community or standalone partner packages) from langchain.\xa0As a reminder, langchain-core contains the main abstractions, interfaces, and core functionality. This code is stable and has been following a stricter versioning policy for a little over a month now.langchain itself, however, still remained on 0.0.x', metadata={'source': 'https://blog.langchain.dev/langchain-v0-1-0/', 'title': 'LangChain v0.1.0', 'language': 'en', 'filename': 'https://blog.langchain.dev/langchain-v0-1-0/'}, _lc_kwargs={'page_content': 'Today we’re excited to announce the release of langchain 0.1.0, our first stable version. It is fully backwards compatible, comes in both Python and JavaScript, and comes with improved focus through both functionality and documentation. A stable version of LangChain helps us earn developer trust and gives us the ability to evolve the library systematically and safely.Python GitHub DiscussionPython v0.1.0 GuidesJS v0.1.0 GuidesYouTube WalkthroughIntroductionLangChain has been around for a little over a year and has changed a lot as it’s grown to become the default framework for building LLM applications. As we previewed a month ago, we recently decided to make significant changes to the\xa0 LangChain package architecture in order to better organize the project and strengthen the foundation.\xa0Specifically we made two large architectural changes: separating out langchain-core and separating out partner packages (either into langchain-community or standalone partner packages) from langchain.\xa0As a reminder, langchain-core contains the main abstractions, interfaces, and core functionality. This code is stable and has been following a stricter versioning policy for a little over a month now.langchain itself, however, still remained on 0.0.x', 'metadata': {'source': 'https://blog.langchain.dev/langchain-v0-1-0/', 'title': 'LangChain v0.1.0', 'language': 'en', 'filename': 'https://blog.langchain.dev/langchain-v0-1-0/'}}),
  Document(page_content="new types of agentsImproving our production ingestion capabilitiesRemoving old and unused functionalityImportantly, even though we are excited about removing some of the old and legacy code to make langchain slimmer and more focused, we also want to maintain support for people who are still using the old version. That is why we will maintain 0.1 as a stable branch (patching in critical bug fixes) for at least 3 months after 0.2 release. We plan to do this for every stable release from here on out.And if you've been wanting to get started contributing, there's never been a better time. We recently added a good getting started issue on GitHub if you're looking for a place to start.One More ThingA large part of LangChain v0.1.0 is stability and focus on the core areas outlined above. Now that we've identified the areas people love about LangChain, we can work on adding more advanced and complete tooling there.One of the main things people love about LangChain is it's support for agents. Most agents are largely defined as running an LLM in some sort of a loop. So far, the only way we've had to do that is with AgentExecutor. We've added a lot of parameters and functionality to AgentExecutor, but its still just one way of running a", metadata={'source': 'https://blog.langchain.dev/langchain-v0-1-0/', 'title': 'LangChain v0.1.0', 'language': 'en', 'filename': 'https://blog.langchain.dev/langchain-v0-1-0/'}, _lc_kwargs={'page_content': "new types of agentsImproving our production ingestion capabilitiesRemoving old and unused functionalityImportantly, even though we are excited about removing some of the old and legacy code to make langchain slimmer and more focused, we also want to maintain support for people who are still using the old version. That is why we will maintain 0.1 as a stable branch (patching in critical bug fixes) for at least 3 months after 0.2 release. We plan to do this for every stable release from here on out.And if you've been wanting to get started contributing, there's never been a better time. We recently added a good getting started issue on GitHub if you're looking for a place to start.One More ThingA large part of LangChain v0.1.0 is stability and focus on the core areas outlined above. Now that we've identified the areas people love about LangChain, we can work on adding more advanced and complete tooling there.One of the main things people love about LangChain is it's support for agents. Most agents are largely defined as running an LLM in some sort of a loop. So far, the only way we've had to do that is with AgentExecutor. We've added a lot of parameters and functionality to AgentExecutor, but its still just one way of running a", 'metadata': {'source': 'https://blog.langchain.dev/langchain-v0-1-0/', 'title': 'LangChain v0.1.0', 'language': 'en', 'filename': 'https://blog.langchain.dev/langchain-v0-1-0/'}}),
  Document(page_content='bug fixes. See more towards the end of this post on our plans for that.While re-architecting the package towards a path to a stable 0.1 release, we took the opportunity to talk to hundreds of developers about why they use LangChain and what they love about it. This input guided our direction and focus. We also used it as an opportunity to bring parity to the Python and JavaScript versions in the core areas outlined below. 💡While certain integrations and more tangential chains may be language specific, core abstractions and key functionality are implemented equally in both the Python and JavaScript packages.We want to share what we’ve heard and our plan to continually improve LangChain. We hope that sharing these learnings will increase transparency into our thinking and decisions, allowing others to better use, understand, and contribute to LangChain. After all, a huge part of LangChain is our community – both the user base and the 2000+ contributors – and we want everyone to come along for the journey.\xa0Third Party IntegrationsOne of the things that people most love about LangChain is how easy we make it to get started building on any stack. We have almost 700 integrations, ranging from LLMs to vector stores to tools for agents', metadata={'source': 'https://blog.langchain.dev/langchain-v0-1-0/', 'title': 'LangChain v0.1.0', 'language': 'en', 'filename': 'https://blog.langchain.dev/langchain-v0-1-0/'}, _lc_kwargs={'page_content': 'bug fixes. See more towards the end of this post on our plans for that.While re-architecting the package towards a path to a stable 0.1 release, we took the opportunity to talk to hundreds of developers about why they use LangChain and what they love about it. This input guided our direction and focus. We also used it as an opportunity to bring parity to the Python and JavaScript versions in the core areas outlined below. 💡While certain integrations and more tangential chains may be language specific, core abstractions and key functionality are implemented equally in both the Python and JavaScript packages.We want to share what we’ve heard and our plan to continually improve LangChain. We hope that sharing these learnings will increase transparency into our thinking and decisions, allowing others to better use, understand, and contribute to LangChain. After all, a huge part of LangChain is our community – both the user base and the 2000+ contributors – and we want everyone to come along for the journey.\xa0Third Party IntegrationsOne of the things that people most love about LangChain is how easy we make it to get started building on any stack. We have almost 700 integrations, ranging from LLMs to vector stores to tools for agents', 'metadata': {'source': 'https://blog.langchain.dev/langchain-v0-1-0/', 'title': 'LangChain v0.1.0', 'language': 'en', 'filename': 'https://blog.langchain.dev/langchain-v0-1-0/'}})],
 'answer': 'The major changes in version 0.1.0 include a new versioning standard where breaking changes result in a minor version bump and bug fixes or new features result in a patch version bump. This release aims to communicate clearly about breaking changes, reduce bloat, and responsibly deal with integrations. The release also brings stability, improved focus, and compatibility in both Python and JavaScript versions.'}

from ragas.testset.generator import TestsetGenerator
from ragas.testset.evolutions import simple, reasoning, multi_context
from langchain_openai import ChatOpenAI, OpenAIEmbeddings

# generator with openai models
generator_llm = ChatOpenAI(model="gpt-3.5-turbo-16k")
critic_llm = ChatOpenAI(model="gpt-4")
embeddings = OpenAIEmbeddings()

generator = TestsetGenerator.from_langchain(
    generator_llm,
    critic_llm,
    embeddings
)

# generate testset
testset = generator.generate_with_langchain_docs(documents, test_size=10, distributions={simple: 0.5, reasoning: 0.25, multi_context: 0.25})

Generating: 100%|██████████| 10/10 [05:02<00:00, 30.28s/it]   

test_df = testset.to_pandas()
test_questions = test_df["question"].values.tolist()
test_groundtruths = test_df["ground_truth"].values.tolist()

testset.to_pandas()

.dataframe tbody tr th {
    vertical-align: top;
}

.dataframe thead th {
    text-align: right;
}

from datasets import Dataset

answers = []
contexts = []

for question in test_questions:
  response = rag_chain.invoke({"question": question})
  answers.append(response['answer'])
  contexts.append([context.page_content for context in response['source_docs']])

response_dataset = Dataset.from_dict({
    "question" : test_questions,
    "answer" : answers,
    "contexts" : contexts,
    "ground_truth" : test_groundtruths
})

response_dataset.to_pandas()

.dataframe tbody tr th {
    vertical-align: top;
}

.dataframe thead th {
    text-align: right;
}

from ragas import evaluate

from ragas.metrics import (
    faithfulness,
    answer_relevancy,
    answer_correctness,
    context_recall,
    context_precision,
)

metrics = [
    faithfulness,
    answer_relevancy,
    context_recall,
    context_precision,
    answer_correctness,
]

results = evaluate(response_dataset, metrics, raise_exceptions=False)

results

Evaluating: 100%|██████████| 50/50 [00:20<00:00,  2.50it/s]
{'faithfulness': 1.0000, 'answer_relevancy': 0.9790, 'context_recall': 1.0000, 'context_precision': 0.8444, 'answer_correctness': 0.6330}