🔒[private user] 🔒[private user] sure, here is my code snippet. Was thinking that maybe simply because it is a pipeline it treats each component as separate. However, it would be much more beneficial to see the pipeline as whole trace.
def _load_index(storage_dir) -> Optional[VectorStoreIndex]:
if os.path.exists(storage_dir):
storage_context = StorageContext.from_defaults(
persist_dir=storage_dir) # rebuild storage context
index = load_index_from_storage(storage_context) # load index
return index
else:
documents = CSVReader(concat_rows=False).load_data(
Path('My document.csv'))
nodes = Settings.node_parser.get_nodes_from_documents(documents)
storage_context = StorageContext.from_defaults()
storage_context.docstore.add_documents(nodes)
index = VectorStoreIndex(nodes, storage_context=storage_context)
index.storage_context.persist(persist_dir=storage_dir)
return index
# index
rules_index = _load_index(os.path.join(
'src', 'pipelines', 'index_storage'))
# similarity store
similarity_store = SingleStoreVectorStore(
table_name="product_embeddings",
content_field="content",
metadata_field="metadata",
vector_field="vector",
timeout=30,
database="products"
)
similarity_index = VectorStoreIndex.from_vector_store(
vector_store=similarity_store)
similarity_query_engine = similarity_index.as_query_engine(
response_mode="compact", use_async=False
)
rules_query_engine = rules_index.as_query_engine(
response_mode="compact", use_async=False
)
plain_product_tmpl = PromptTemplate(
"The following item: '{name}' has following components: {components}.")
llm = OpenAI(model="gpt-4o-mini", temperature=0)
similarity_retriever = similarity_index.as_retriever(similarity_top_k=5)
rules_retriever = rules_index.as_retriever(similarity_top_k=3)
# here are simply my custom components that inherit from CustomQueryComponent
classifier = AlternativeClassifierResponse(llm=llm)
suggestor = SuggestorComponent(llm=llm)
sibling_rules_retriever = AlternativeSiblingRuleComponent(
retriever=rules_retriever)
# and here is the pipeline
p = QueryPipeline(verbose=True)
p.add_modules(
{
"similarity_retriever": similarity_retriever,
"sibling_rules_retriever": sibling_rules_retriever,
"classifier": classifier,
"suggestor": suggestor,
"plain_product_tmpl": plain_product_tmpl
}
)
# here is creation of DAG pipeline
p.add_link("plain_product_tmpl", "similarity_retriever")
p.add_link("similarity_retriever", "suggestor", dest_key="product_nodes")
p.add_link("plain_product_tmpl", "suggestor", dest_key="query_str")
p.add_link("suggestor", "sibling_rules_retriever", dest_key="brick_candidates")
p.add_link("sibling_rules_retriever", "classifier", dest_key="rule_nodes")
p.add_link("suggestor", "classifier", dest_key="brick_candidates")
p.add_link("plain_product_tmpl", "classifier", dest_key="query_str")
was generally trying to base on example from docs of llama-index https://docs.llamaindex.ai/en/stable/module_guides/querying/pipeline/usage_pattern/
