"use client" import { useCallback, useEffect, useRef, useState, useTransition } from "react" import { useInterval } from "usehooks-ts" import Webcam from "react-webcam" import AutoSizer from "react-virtualized-auto-sizer" import { see } from "./engine/see" import { Progress } from "./interface/progress" export function Observe({ onObserve, }: { onObserve: (observation: string, image: string) => void }) { const [_isPending, startTransition] = useTransition() const [img, setImg] = useState("") const webcamRef = useRef(null) const [isInitialized, setInitialized] = useState(false) const [frameNumber, setFrameNumber] = useState(0) const [isBusy, setBusy] = useState(false) const [lastObservation, setLastObservation] = useState("Nothing to see yet.") const [lastObservedAt, setLastObservedAt] = useState(Date.now()) const defaultWidth = 1280 const defaultHeight = 1024 // 720 // minimum wait time between calls const minimumWaitTimeInSec = 10 // in case we need to record a video, check the last part of // https://blog.openreplay.com/capture-real-time-images-and-videos-with-react-webcam/ const capture = useCallback(() => { if (!webcamRef.current) { return } const imageSrc = webcamRef.current.getScreenshot() if (!imageSrc) { return } setImg(imageSrc) setFrameNumber(frameNumber + 1) return imageSrc }, [webcamRef]) // note: for some strange reason, the webcam (at least on macOS) // has a "fade in effect", which means in the first few seconds, // eg. if we capture at 800ms, if will be darker than normal useEffect(() => { if (webcamRef.current && img && !isInitialized) { setInitialized(true) } }, [webcamRef.current, img, isInitialized]) const observe = () => { if (isBusy) { // console.log("we are already predicting: skippping turn") return } const currentTimeInMs = Date.now() const elapsedTimeInMs = currentTimeInMs - lastObservedAt const elapsedTimeInSec = elapsedTimeInMs / 1000 if (elapsedTimeInSec < minimumWaitTimeInSec) { // console.log("minimum wait time between calls not reached: skipping turn") return } setBusy(true) // console.log("Capturing new frame from webcam..") startTransition(async () => { const imageBase64 = capture() if (!imageBase64) { console.log("Failed to capture a new frame") setTimeout(() => { setBusy(false) setLastObservedAt(Date.now()) }, 2000) return } const prompt = `What do you see here?` console.log("JULIAN: disabled watch") // console.log("Calling IDEFICS..") const newObservation = await see({ prompt, imageBase64 }) // console.log("New observation: ", newObservation) if (newObservation && newObservation !== lastObservation) { // console.log("update!") setLastObservation(newObservation || "") onObserve(newObservation || "", imageBase64) } setLastObservedAt(Date.now()) // comment to disable the infinite loop! setBusy(false) }) // console.log("observation ended!") } useInterval(() => { observe() }, 1000) return ( {({ height, width }) => ( <> )} ) }