create .gitignore in image folder, and include base64 image in parquet, app.py use base64 image to display

pipeline/process_videos.py

@@ -1,3 +1,5 @@
+import base64
+
 import cv2
 import pandas as pd
 from PIL import Image
@@ -12,6 +14,8 @@ DATAFRAME_PATH = DATA_DIR / "dataset.parquet"
 
 
 def process_videos() -> None:
+    IMAGES_DIR.mkdir(exist_ok=True, parents=True)
+    (IMAGES_DIR / ".gitignore").write_text("*")
     "Runs clip on video frames, saves results to a parquet file"
     clip_wrapper = ClipWrapper()
     results = []
@@ -27,19 +31,21 @@ def process_videos() -> None:
         ):
             image_path = extracted_images_dir / f"{frame_idx}.jpg"
             image.save(image_path)
+            with open(image_path, "rb") as f:
+                encoded_image = base64.b64encode(f.read()).decode()
             results.append(
                 [
                     video_id,
                     frame_idx,
                     timestamp_secs,
-                    str(image_path.relative_to(REPO_ROOT)),
+                    encoded_image,
                     *clip_vector,
                 ]
             )
         complete_file.touch()
     df = pd.DataFrame(
         results,
-        columns=["video_id", "frame_idx", "timestamp", "image_path"]
+        columns=["video_id", "frame_idx", "timestamp", "base64_image"]
         + [f"dim_{i}" for i in range(MODEL_DIM)],
     )
     print(f"Saving data to {DATAFRAME_PATH}")

video_semantic_search/app.py

@@ -30,6 +30,7 @@ class SemanticSearcher:
                 video_id=row["video_id"],
                 frame_idx=row["frame_idx"],
                 timestamp=row["timestamp"],
+                base64_image=row["base64_image"],
                 score=score,
             )
             for score, (_, row) in zip(D[0], self.metadata.iloc[I[0]].iterrows())
@@ -45,10 +46,12 @@ class SearchResult:
     video_id: str
     frame_idx: int
     timestamp: float
+    base64_image: str
     score: float
 
 
 def get_video_url(video_id: str, timestamp: float) -> str:
+    timestamp = max(0, timestamp - 3)  # Show 3 seconds before the query
     return f"https://www.youtube.com/watch?v={video_id}&t={int(timestamp)}"
 
 
@@ -87,19 +90,10 @@ def display_search_results(results: list[SearchResult]) -> None:
                 """,
                 unsafe_allow_html=True,
             )
-
-            # Display the embedded YouTube video
-            # st.video(get_video_url(result.video_id), start_time=int(result.timestamp))
-            # st.image(f"data/images/{result.video_id}/{result.frame_idx}.jpg")
-            with open(
-                f"data/images/{result.video_id}/{result.frame_idx}.jpg", "rb"
-            ) as f:
-                image = f.read()
-                encoded = base64.b64encode(image).decode()
             st.markdown(
                 f"""
                 <a href="{get_video_url(result.video_id, result.timestamp)}">
-                <img src="data:image/jpeg;base64,{encoded}" alt="frame {result.frame_idx}" width="100%">
+                <img src="data:image/jpeg;base64,{result.base64_image}" alt="frame {result.frame_idx} timestamp {int(result.timestamp)}" width="100%">
                 </a>
                 """,
                 unsafe_allow_html=True,