chore: sync app/ and src/ from GitHub

app/app.py

@@ -1,3 +1,4 @@
+import json
 import csv, sys
 from datetime import datetime
 from pathlib import Path
@@ -8,13 +9,12 @@ import markdown
 ROOT_FOLDER = Path(__file__).resolve().parent.parent
 
 sys.path.append(str(ROOT_FOLDER))
-import pickle
+import sys
 import os
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src'))
-from src.retrieval_helpers import enrich_search_results,enrich_bm25_search_results
-from src.semantic import load_vector_store
+from src.semantic import load_vector_store, enrich_search_results
 from src.rag_pipeline import run_rag
-from src.bm25 import load
+from src.bm25 import load, search
 from src.hybrid import HybridRetriever
 
 from dotenv import load_dotenv
@@ -40,7 +40,6 @@ TOP_K = 5
 
 HF_TOKEN = os.getenv('HF_TOKEN')
 
-from datasets import load_dataset
 from huggingface_hub import snapshot_download, login
 
 # ─── Custom CSS ───────────────────────────────────────────────────────────────
@@ -49,46 +48,26 @@ with open('./app/styles.css', "r") as f:
 
 st.markdown(f"<style>{css}</style>", unsafe_allow_html=True)
 
-@st.cache_resource
-def load_hf_dataset():
-    return load_dataset(
-        "McAuley-Lab/Amazon-Reviews-2023",
-        "raw_meta_Grocery_and_Gourmet_Food",
-        trust_remote_code=True,
-        token=HF_TOKEN
-    )
-
 VECTOR_STORE_DIR = ROOT / "data" / "processed"
 
 @st.cache_resource
 def load_vector_store_cached():
-
     login(token=HF_TOKEN, add_to_git_credential=False)
     VECTOR_STORE_DIR.mkdir(parents=True, exist_ok=True)
 
-    # Skip download if already cached on disk
-    if not any(VECTOR_STORE_DIR.iterdir()):
-        snapshot_path = Path(snapshot_download(
-            repo_id="rishadaz/amazon_retriever-storage",
-            repo_type="dataset",
-            local_dir=str(VECTOR_STORE_DIR),
-            token=HF_TOKEN,
-        ))
-    else:
-        snapshot_path = VECTOR_STORE_DIR
-
-    mini_index_path = snapshot_path / "tokenisation" / "bm25_index_mini.pkl"
-    embeddings_dir  = snapshot_path / "embeddings"
-
-    if not mini_index_path.exists():
-        raise FileNotFoundError(f"BM25 index not found at {mini_index_path}")
-    if not embeddings_dir.exists():
-        raise FileNotFoundError(f"Embeddings dir not found at {embeddings_dir}")
+    snapshot_path = snapshot_download(
+        repo_id="rishadaz/amazon_retriever-storage",
+        repo_type="dataset",
+        local_dir=str(VECTOR_STORE_DIR),
+        split='full',
+        token=HF_TOKEN,
+    )
 
-    with open(mini_index_path, "rb") as f:
-        bm25_retriever = pickle.load(f)
+    mini_index_path = Path(snapshot_path) / "tokenisation" / "bm25_index_mini.pkl"
+    embeddings_dir  = Path(snapshot_path) / "embeddings"
 
-    vector_store = load_vector_store(embeddings_dir)
+    vector_store    = load_vector_store(embeddings_dir)
+    bm25_retriever  = load(mini_index_path)
 
     return vector_store, bm25_retriever
 
@@ -97,14 +76,12 @@ def load_vector_store_cached():
 # read the mini versions of the files we have provided in the repo
 
 data_source = os.getenv('DATA_SOURCE')
-
+print(f"Running with data source {data_source}")
 # note: remote has the full generated corpus and 
 # embeddings which can take a long time to download and 
 # the app might become heavy too and slow down 
 # processing. For development pls use the smaller "local" corpus
 
-HF_DATASET = load_hf_dataset()
-
 if data_source == 'local':
     MINI_INDEX_PATH  = ROOT / "data" / "processed" / "tokenisation" / "bm25_index_mini.pkl"
 
@@ -124,7 +101,7 @@ def bm25_search(query: str, top_k: int = 3) -> list[dict]:
     Returns top_k review-level results (may include multiple reviews per ASIN).
     """
 
-    results = enrich_bm25_search_results(retriever, query, top_k, HF_DATASET['full'])
+    results = search(retriever, query, top_k)
     return results
 
 
@@ -136,7 +113,7 @@ def semantic_search(query: str, top_k: int = 3) -> list[dict]:
     Returns top_k review-level results (scores are cosine similarities, 0–1).
     """
    
-    results = enrich_search_results(vector_store, query, top_k, HF_DATASET['full'])
+    results = enrich_search_results(vector_store, query, top_k)
     return results
 
 hybrid_retriever = HybridRetriever(
@@ -148,7 +125,7 @@ hybrid_retriever = HybridRetriever(
     )
 
 def llm_retriever(query: str, top_k: int = 5):
-    answer, docs = run_rag(hybrid_retriever, query=query, hf_dataset=HF_DATASET['full'])
+    answer, docs = run_rag(hybrid_retriever, query=query)
     return answer, docs
 
 
@@ -177,18 +154,25 @@ def log_feedback(query: str, mode: str, asin: str, title: str, vote: str) -> Non
             "vote":      vote,
         })
 
-def render_product(ind, item):
-    reviews     = item.get("reviews",{})
-    title       = item["title"]
+def render_product(ind, item, mode):
+    item = dict(item)
+    if "reviews" in item.keys():
+        reviews     = item.get("reviews",{})
+    elif "top_reviews" in item.keys():
+        reviews     = item.get("top_reviews",{})
+    else:
+        reviews = []
+    title       = item.get("title","")
     avg_rating  = item["average_rating"]
     n_reviews   = len(reviews)
     # total_reviews = item.get('total_reviews', n_reviews)
     rating_number = item.get('rating_number', 0)
     asin        = item['parent_asin']
     review_word = "review" if n_reviews == 1 else "reviews"
-    large_images = item.get('images', {}).get('large', [])
-    image_html = f'<img src="{large_images[0]}" style="width:100%;max-width:200px;border-radius:8px;margin-bottom:8px;" />' if large_images else ''
+    large_image = item.get('image', "")
+    image_html = f'<img src="{large_image}" style="width:100%;max-width:200px;border-radius:8px;margin-bottom:8px;" />' if large_image else f'<image src="" />'
     raw_price = item.get('price')
+    score = item.get('score',None) if 'score' in item else item.get('hybrid_score',None)
     try:
         price_val = float(str(raw_price).replace('$', '').replace(',', '').strip())
         price_html = f'<span style="color:#2ecc71;font-weight:600">${price_val:.2f}</span>'
@@ -197,7 +181,11 @@ def render_product(ind, item):
 
 
     # ── Product card header ───────────────────────────────────────────
-    score_badge = f'<span class="score-badge">similarity score: {float(item["score"]):.2f}</span>' if 'score' in item else "<span/>"
+    score_badge = f'<span class="score-badge">{mode} score: {float(score):.2f}</span>' if score else "<span/>"
+    if 'retrieval_source' in item:
+        source_badge = f'<span class="score-badge">Source: {item['retrieval_source']}</span>'
+    else:
+        source_badge = '<span />'
     
     st.markdown(
         f"""
@@ -207,7 +195,7 @@ def render_product(ind, item):
             <span class="stars">{stars(avg_rating)}</span>
             &nbsp;<small style="color:#888">{avg_rating:.1f}/5 avg ({rating_number:,} ratings)</small>
             &nbsp;&nbsp;
-            {score_badge}
+            {score_badge} {source_badge}
             {"&nbsp;&nbsp;" + price_html if price_html else ""}
         </div>
         """,
@@ -248,13 +236,13 @@ def render_product(ind, item):
 
 
 
-def render_results(results: list[dict], mode: str, query: str) -> None:
+def render_results(results: list[dict], mode: str) -> None:
     if not results:
         st.info("No results returned.")
         return
     
     for ind, item in enumerate(results):
-        render_product(ind,item)
+        render_product(ind,item, mode)
         
 # ─── App layout ───────────────────────────────────────────────────────────────
 st.markdown(
@@ -319,7 +307,7 @@ with tab_search:
             if mode == "BM25"
             else st.session_state.semantic_results
         )
-        render_results(results, mode=mode.lower(), query=st.session_state.last_query)
+        render_results(results, mode=mode.lower())
 
 # ─── LLM Tab ──────────────────────────────────────────────────────────────────
 with tab_llm:
@@ -343,12 +331,8 @@ with tab_llm:
         st.markdown("#### 📦 Retrieved Products")
         docs = st.session_state.get("llm_docs", [])
         if docs:
-            # Build scrollable card list in one HTML block
-            cards_html = "<div class='doc-sidebar'>"
-            for i, doc in enumerate(docs, 1):
-                render_product(i,doc)
-            cards_html += "</div>"
-            st.markdown(cards_html, unsafe_allow_html=True)
+            docs = [json.loads(json.dumps(obj.metadata, default=str)) for obj in docs]
+            render_results(docs, mode='hybrid')
         else: 
             st.markdown("<p style='color:#aaa;'>No documents retrieved.</p>", unsafe_allow_html=True)
 

src/bm25.py

@@ -26,7 +26,7 @@ from langchain_core.documents import Document
 ROOT_FOLDER = Path(__file__).resolve().parent.parent
 
 sys.path.append(str(ROOT_FOLDER))
-from src.utils import simple_tokenize
+from src.utils import simple_tokenize, extract_image
 from src.eda_helpers import get_best_reviews
 
 
@@ -162,22 +162,6 @@ Details:
 Top Reviews (showing {n_reviews}):
     {review_lines}"""
 
-
-def _extract_image_url(images: Any) -> str:
-    """
-    Extract the best available image URL from the images field.
-    The field is a dict with keys: thumb, large, hi_res, variant — each a list.
-    Prefers 'large', falls back to 'thumb', then 'hi_res'. Returns "" if none found.
-    """
-    if not images or not isinstance(images, dict):
-        return ""
-    for key in ("large", "thumb", "hi_res"):
-        urls = images.get(key)
-        if isinstance(urls, list) and urls and urls[0]:
-            return urls[0]
-    return ""
-
-
 def build_document(product: dict, top_reviews: list[dict]) -> Document | None:
     """
     Build one LangChain Document for a single product row from the metadata Dataset.
@@ -201,7 +185,7 @@ def build_document(product: dict, top_reviews: list[dict]) -> Document | None:
         "details":        details_dict,
         "average_rating": product.get("average_rating"),
         "rating_number":  product.get("rating_number"),
-        "image_url":      _extract_image_url(product.get("images")),
+        "image":      extract_image(product),
         "top_reviews":    top_reviews,
     }
 
@@ -389,33 +373,18 @@ def search(
     # Get raw BM25 scores for ALL documents
     scores = retriever.vectorizer.get_scores(tokenized_query)  # np.ndarray, len = n_docs
     
-    # Get top-k doc indices by score
-    top_indices = sorted(range(len(scores)), key=lambda i: scores[i], reverse=True)[:top_k]
-    
-    results = []
-    for idx in top_indices:
-        doc = retriever.docs[idx]          # retriever.docs holds the original Document list
-        m   = doc.metadata
-        top_reviews = m.get("top_reviews", [])
-
-        rated = [r["rating"] for r in top_reviews if r.get("rating") is not None]
-        avg_rating = round(sum(rated) / len(rated), 1) if rated else 0.0
-
-        if top_reviews and top_reviews[0].get("text"):
-            snippet = top_reviews[0]["text"][:300]
-        else:
-            snippet = m.get("description", "")[:300]
-
-        results.append({
-            "asin":        m.get("parent_asin", ""),
-            "title":       m.get("title", ""),
-            "text":        snippet,
-            "rating":      avg_rating,
-            "score":       float(scores[idx]),   
-            "top_reviews": top_reviews,
-        })
-
-    return results
+    # Get indices of top-k scores
+    top_indices = sorted(
+        range(len(scores)),
+        key=scores.__getitem__,
+        reverse=True
+    )[:top_k]
+
+    # Collect metadata for top results, including score
+    return [
+        {**retriever.docs[i].metadata, "score": scores[i]}
+        for i in top_indices
+    ]
 
 
 # ── notebook entry point ──────────────────────────────────────────────────────

src/rag_pipeline.py

@@ -21,8 +21,6 @@ from langchain_core.output_parsers import StrOutputParser
 from langchain_core.prompts import ChatPromptTemplate
 from langchain_core.runnables import RunnableLambda, RunnablePassthrough
 from langchain_huggingface import ChatHuggingFace, HuggingFaceEndpoint
-from src.retrieval_helpers import _format_docs
-
 # ---------------------------------------------------------------------------
 # Logging
 # ---------------------------------------------------------------------------
@@ -203,7 +201,6 @@ def run_rag(
     max_new_tokens: int = DEFAULT_MAX_NEW_TOKENS,
     provider: str = "auto",
     verbose: bool = False,
-    hf_dataset = None
 ) -> str:
     """
     Execute a full RAG pipeline and return the model's answer.
@@ -295,10 +292,5 @@ def run_rag(
     logger.info("Invoking RAG chain for query: %r", query)
     answer: str = rag_chain.invoke(query)
     logger.debug("RAG answer: %s", answer)
-    
-    if hf_dataset:
-        docs = _format_docs(retrieved_docs, hf_dataset)
-    else:
-        docs = retrieved_docs
 
-    return answer, docs
+    return answer, retrieved_docs

src/semantic.py

@@ -17,6 +17,7 @@ Typical usage
 
 import logging
 from typing import Any
+
 import torch
 import json, os, sys
 from pathlib import Path
@@ -31,6 +32,7 @@ ROOT_FOLDER = Path(__file__).resolve().parent.parent
 
 sys.path.append(str(ROOT_FOLDER))
 from src.eda_helpers import get_best_reviews
+from src.utils import decode_ratings, extract_image
 
 logger = logging.getLogger(__name__)
 
@@ -141,6 +143,8 @@ def create_document(product, reviews: Dataset) -> Document | None:
         "rating_number":  product.get("rating_number"),
         # --- categorical (filterable) ---
         "main_category":  product.get("main_category", ""),
+        "title":  product.get("title", ""),
+        "image":  extract_image(product),
         "categories":     product.get("categories") or [],
         # --- free-form (display only; coerce to str for FAISS compatibility) ---
         "details":        str(product.get("details") or ""),
@@ -257,28 +261,31 @@ def build_and_save_vector_store(
 # Search
 # ---------------------------------------------------------------------------
 
-def semantic_search(
-    query: str,
-    vector_store: FAISS,
-    k: int = DEFAULT_TOP_K,
-    filter = None,
-) -> list[Document]:
+def enrich_search_results(vector_store, query: str, k: int, filter=None):
     """
-    Run a semantic similarity search against a pre-built *vector_store*.
-
+    Perform similarity search and enrich results with HuggingFace dataset metadata.
+    
     Args:
-        query:        Natural-language search query.
-        vector_store: A FAISS store built with :func:`build_vector_store`.
-        k:            Number of results to return.
-        filter:       Optional metadata filter dict, e.g.
-                      ``{"main_category": "Electronics"}``.
-
+        vector_store: LangChain vector store instance
+        query: Search query string
+        k: Number of results to return
+        filter: Filter dict for similarity search
+    
     Returns:
-        Ordered list of the *k* most relevant Documents.
+        List of enriched metadata objects as dicts
     """
     results = vector_store.similarity_search_with_score(query, k=k, filter=filter)
-    logger.info("'%s' -> %d results", query, len(results))
-    return results
+
+    enriched_results = []
+
+    for doc, score in results:
+        metadata_object = {**doc.metadata}  # start with all doc metadata
+        metadata_object['score'] = float(score)
+        metadata_object['reviews'] = decode_ratings(doc.page_content) or []
+
+        enriched_results.append(metadata_object)
+
+    return [json.loads(json.dumps(obj, default=str)) for obj in enriched_results]
 
 # ---------------------------------------------------------------------------
 # Read existing vector store

src/utils.py

@@ -1,4 +1,5 @@
 import re
+import json
 import nltk
 from nltk.corpus import stopwords
 
@@ -17,4 +18,53 @@ def simple_tokenize(text):
     text = re.sub(r"[^a-z0-9\s]", "", text)
     tokens = text.split()
     tokens = [t for t in tokens if t not in STOPWORDS]
-    return tokens
+    return tokens
+
+def extract_image(row):
+    """
+    Return the first large image URL from the HF dataset row, or None.
+ 
+    Expected structure (adjust key names to match your dataset):
+        row["images"] = {"large": ["https://...", ...], ...}
+    or a JSON-encoded string of the same shape.
+    """
+    images = row.get("images")
+    if images is None:
+        return None
+ 
+    # Some datasets store this column as a JSON string
+    if isinstance(images, str):
+        try:
+            images = json.loads(images)
+        except json.JSONDecodeError:
+            return None
+ 
+    if not isinstance(images, dict):
+        return None
+ 
+    large = images.get("large")
+    if large and isinstance(large, list) and len(large) > 0:
+        return large[0]
+ 
+    return None
+
+def decode_ratings(page_content):
+    block_pattern = r'\[\d\.0★\].*'
+    matches = re.findall(block_pattern, page_content)
+    if matches:
+        pattern = r'\[(\d\.0)★\]\s*(.*?)\s*—\s*(.*)'
+        parsed = []
+
+        for r in matches[:3]:
+            match = re.match(pattern, r)
+            if match:
+                rating, title, text = match.groups()
+                parsed.append({
+                    'rating': float(rating),
+                    'title': title.strip(),
+                    'text': text.strip()
+                })
+
+        return(parsed)
+    else:
+        return {}