improve UI + add caching

package-lock.json

@@ -8,6 +8,7 @@
       "name": "video-quest",
       "version": "0.0.0",
       "dependencies": {
+        "@gorgonjs/gorgon": "^1.4.1",
         "@huggingface/agents": "^0.0.4",
         "@huggingface/inference": "^2.6.1",
         "@radix-ui/react-accordion": "^1.1.2",
@@ -189,6 +190,11 @@
         "react-dom": ">=16.8.0"
       }
     },
+    "node_modules/@gorgonjs/gorgon": {
+      "version": "1.4.1",
+      "resolved": "https://registry.npmjs.org/@gorgonjs/gorgon/-/gorgon-1.4.1.tgz",
+      "integrity": "sha512-XUTvRODad+uD89CVoLQEi3aOaJC/x9+KqLBKil4a+hKlrDRc6TAoEofn/Kje/S4Q+ylwJRbhZnb98QgiSZxIqw=="
+    },
     "node_modules/@huggingface/agents": {
       "version": "0.0.4",
       "resolved": "https://registry.npmjs.org/@huggingface/agents/-/agents-0.0.4.tgz",

package.json

@@ -9,6 +9,7 @@
     "lint": "next lint"
   },
   "dependencies": {
+    "@gorgonjs/gorgon": "^1.4.1",
     "@huggingface/agents": "^0.0.4",
     "@huggingface/inference": "^2.6.1",
     "@radix-ui/react-accordion": "^1.1.2",

src/app/games/city.ts

@@ -1,3 +1,4 @@
+import { edu } from "@/lib/fonts"
 import { Game } from "./types"
 
 const actions = [
@@ -43,6 +44,7 @@ const initialActionnables = [
 export const game: Game = {
   title: "City",
   type: "city",
+  className:  edu.className,
   initialSituation,
   initialActionnables,
   getScenePrompt: (situation?: string) => [

src/app/games/doom.ts

@@ -1,3 +1,4 @@
+import { orbitron } from "@/lib/fonts"
 import { Game } from "./types"
 
 const initialSituation = [
@@ -16,6 +17,7 @@ const initialActionnables = [
 export const game: Game = {
   title: "Doom",
   type: "doom",
+  className: orbitron.className,
   initialSituation,
   initialActionnables,
   getScenePrompt: (situation?: string) => [

src/app/games/dungeon.ts

@@ -1,3 +1,4 @@
+import { amatic } from "@/lib/fonts"
 import { Game, Scene } from "./types"
 
 const actions = [
@@ -47,6 +48,7 @@ const initialActionnables = [
 export const game: Game = {
   title: "Dungeon",
   type: "dungeon",
+  className: amatic.className,
   initialSituation,
   initialActionnables,
   getScenePrompt: (situation?: string) => [

src/app/games/pirates.ts

@@ -1,3 +1,4 @@
+import { lugrasimo } from "@/lib/fonts"
 import { Game } from "./types"
 
 const actions = [
@@ -54,6 +55,7 @@ const initialSituation = [
 export const game: Game = {
   title: "Pirates",
   type: "pirates",
+  className: lugrasimo.className,
   initialSituation,
   initialActionnables,
   getScenePrompt: (situation?: string) => [

src/app/games/types.ts

@@ -1,3 +1,5 @@
+import { FontName } from "@/lib/fonts"
+
 export type GameType = 'pirates' | 'city' | 'dungeon' | 'doom'
 
 export interface Scene {
@@ -8,6 +10,7 @@ export interface Scene {
 export interface Game {
   title: string
   type: GameType
+  className: string
   initialSituation: string
   initialActionnables: string[]
   getScenePrompt: (situation?: string) => string[]

src/app/main.tsx

@@ -1,7 +1,8 @@
 "use client"
 
 import { useEffect, useRef, useState, useTransition } from "react"
-import { useSearchParams } from 'next/navigation'
+import { usePathname, useRouter, useSearchParams } from "next/navigation"
+
 
 import { ImageRenderer } from "@/components/business/image-renderer"
 
@@ -16,7 +17,7 @@ import {
 import { render } from "./render"
 
 import { RenderedScene } from "./types"
-import { GameType } from "./games/types"
+import { Game, GameType } from "./games/types"
 import { defaultGame, games, getGame } from "./games"
 import { getBackground } from "@/app/queries/getBackground"
 import { getDialogue } from "@/app/queries/getDialogue"
@@ -30,11 +31,15 @@ export default function Main() {
     maskBase64: "",
     segments:[]
   })
+  const router = useRouter()
+  const pathname = usePathname()
   const searchParams = useSearchParams()
  
   const requestedGame = (searchParams.get('game') as GameType) || defaultGame
-  console.log("requestedGame:", requestedGame)
+  console.log("requestedGame: " + requestedGame)
   const gameRef = useRef<GameType>(requestedGame)
+  console.log("gameRef.current: " + gameRef.current)
+  const [game, setGame] = useState<Game>(getGame(gameRef.current))
   const [situation, setSituation] = useState("")
   const [scene, setScene] = useState("")
   const [dialogue, setDialogue] = useState("")
@@ -46,12 +51,7 @@ export default function Main() {
     setLoading(true)
 
     await startTransition(async () => {
-
-      // console.log(`getting agent..`)
-      // note: we use a ref so that it can be changed in the background
-      const type = gameRef?.current
-      console.log("type:", type)
-      const game = getGame(type)
+      console.log("Rendering a scene for " + game.type)
 
       // console.log(`rendering scene..`)
       const newRendered = await render(
@@ -65,10 +65,9 @@ export default function Main() {
         ).slice(0, 6) // too many can slow us down it seems
       )
 
-      // detect if something changed in the background
-      if (type !== gameRef?.current) {
-        console.log("agent type changed! reloading scene")
-        setTimeout(() => { loadNextScene() }, 0)
+      // detect if type game type changed while we were busy
+      if (game.type !== gameRef?.current) {
+        console.log("game type changed! aborting..")
         return
       } 
 
@@ -130,23 +129,44 @@ export default function Main() {
 
   const clickables = Array.from(new Set(rendered.segments.map(s => s.label)).values())
 
+  const handleSelectGame = (newGameType: GameType) => {
+    gameRef.current = newGameType
+    setGame(getGame(newGameType))
+    setRendered({
+      assetUrl: "", 
+      error: "",
+      maskBase64: "",
+      segments:[]
+    })
+    setLoading(true)
+
+    const current = new URLSearchParams(Array.from(searchParams.entries()))
+    current.set("game", newGameType)
+    const search = current.toString()
+    const query = search ? `?${search}` : ""
+
+    // for some reason, this doesn't work?!
+    router.replace(`${pathname}${query}`, { })
+    
+    // workaround.. but it is strange that router.replace doesn't work..
+    let newurl = window.location.protocol + "//" + window.location.host + window.location.pathname + '?' + search.toString()
+    window.history.pushState({path: newurl}, '', newurl)
+  }
+
   return (
-    <div className="flex flex-col w-full pt-4">
+    <div
+      className={[
+        "flex flex-col w-full pt-4",
+        getGame(gameRef.current).className // apply the game theme
+      ].join(" ")}
+    >
       <div className="flex flex-col space-y-3 px-2">
         <div className="flex flex-row items-center space-x-3">
           <label className="flex">Select a story:</label>
           <Select
             defaultValue={gameRef.current}
-            onValueChange={(value) => {
-              gameRef.current = value as GameType
-              setRendered({
-                assetUrl: "", 
-                error: "",
-                maskBase64: "",
-                segments:[]
-              })
-            }}>
-            <SelectTrigger className="w-[180px]">
+            onValueChange={(value) => { handleSelectGame(value as GameType) }}>
+            <SelectTrigger className="text-xl w-[180px]">
               <SelectValue placeholder="Type" />
             </SelectTrigger>
             <SelectContent>
@@ -156,7 +176,7 @@ export default function Main() {
             </SelectContent>
           </Select>
         </div>
-        <p className="text-xl">This experimental demo uses shared ressources: each scene may take more than 45s to load.</p>
+        <p className="text-xl">A stable diffusion exploration game. Click on an item to explore a new scene!</p>
         <div className="flex flex-row">
           <div className="text-xl mr-2">🔎 Clickable items:</div>
           {clickables.map((clickable, i) => 
@@ -172,6 +192,7 @@ export default function Main() {
         onUserAction={handleUserAction}
         onUserHover={setHoveredActionnable}
         isLoading={isLoading}
+        type={game.type}
       />
       <p className="text-xl">{dialogue}</p>
     </div>

src/app/page.tsx

@@ -4,15 +4,22 @@ import Head from "next/head"
 
 import Main from "./main"
 
+// https://nextjs.org/docs/pages/building-your-application/optimizing/fonts 
+
 export default async function IndexPage({ params: { ownerId } }: { params: { ownerId: string }}) {
   return (
-    <div>
+    <>
       <Head>
+        <link rel="preconnect" href="https://fonts.gstatic.com" crossOrigin="anonymous" />
+        <link rel="preconnect" href="https://fonts.googleapis.com" crossOrigin="anonymous" />
         <meta name="viewport" content="width=device-width, initial-scale=0.86, maximum-scale=5.0, minimum-scale=0.86" />
       </Head>
-      <main className="dark fixed inset-0 flex flex-col items-center bg-stone-900 text-stone-10 overflow-y-scroll">
+      <main className={
+        `dark fixed inset-0 flex flex-col items-center
+         bg-stone-900 text-stone-10 overflow-y-scroll
+        `}>
         <Main />
       </main>
-    </div>
+    </>
   )
 }

src/app/render.ts

@@ -1,56 +1,63 @@
 "use server"
 
+import Gorgon from "@gorgonjs/gorgon"
+
 import { RenderedScene } from "./types"
 
 // note: there is no / at the end in the variable
 // so we have to add it ourselves if needed
 const apiUrl = process.env.RENDERING_ENGINE_API
 
+
+const cacheDurationInSec = 30 * 60 // 30 minutes
+
 export async function render(prompt: string, actionnables: string[] = []) {
-  let defaulResult: RenderedScene = {
-    assetUrl: "",
-    maskBase64: "",
-    error: "",
-    segments: []
-  }
-
-  try {
-    console.log(`calling ${apiUrl}/render with prompt: ${prompt}`)
-    const res = await fetch(`${apiUrl}/render`, {
-      method: "POST",
-      headers: {
-        Accept: "application/json",
-        "Content-Type": "application/json",
-        // Authorization: `Bearer ${process.env.VC_SECRET_ACCESS_TOKEN}`,
-      },
-      body: JSON.stringify({
-        prompt,
-        // nbFrames: 8 and nbSteps: 15 --> ~10 sec generation
-        nbFrames: 1, // when nbFrames is 1, we will only generate static images
-        nbSteps: 20,
-        actionnables,
-        segmentation: "firstframe", // one day we will remove this param, to make it automatic
-      }),
-       cache: 'no-store',
-     // we can also use this (see https://vercel.com/blog/vercel-cache-api-nextjs-cache)
-     // next: { revalidate: 1 }
-    })
-
-    // console.log("res:", res)
-    // The return value is *not* serialized
-    // You can return Date, Map, Set, etc.
-    
-    // Recommendation: handle errors
-    if (res.status !== 200) {
-      // This will activate the closest `error.js` Error Boundary
-      throw new Error('Failed to fetch data')
+  return await Gorgon.get(`render/${JSON.stringify(prompt, actionnables)}`, async () => {
+    let defaulResult: RenderedScene = {
+      assetUrl: "",
+      maskBase64: "",
+      error: "",
+      segments: []
+    }
+
+    try {
+      console.log(`calling ${apiUrl}/render with prompt: ${prompt}`)
+      const res = await fetch(`${apiUrl}/render`, {
+        method: "POST",
+        headers: {
+          Accept: "application/json",
+          "Content-Type": "application/json",
+          // Authorization: `Bearer ${process.env.VC_SECRET_ACCESS_TOKEN}`,
+        },
+        body: JSON.stringify({
+          prompt,
+          // nbFrames: 8 and nbSteps: 15 --> ~10 sec generation
+          nbFrames: 1, // when nbFrames is 1, we will only generate static images
+          nbSteps: 20,
+          actionnables,
+          segmentation: "firstframe", // one day we will remove this param, to make it automatic
+        }),
+        cache: 'no-store',
+      // we can also use this (see https://vercel.com/blog/vercel-cache-api-nextjs-cache)
+      // next: { revalidate: 1 }
+      })
+
+      // console.log("res:", res)
+      // The return value is *not* serialized
+      // You can return Date, Map, Set, etc.
+      
+      // Recommendation: handle errors
+      if (res.status !== 200) {
+        // This will activate the closest `error.js` Error Boundary
+        throw new Error('Failed to fetch data')
+      }
+      
+      const response = (await res.json()) as RenderedScene
+      // console.log("response:", response)
+      return response
+    } catch (err) {
+      console.error(err)
+      return defaulResult
     }
-    
-    const response = (await res.json()) as RenderedScene
-    // console.log("response:", response)
-    return response
-  } catch (err) {
-    console.error(err)
-    return defaulResult
-  }
-}
+  }, cacheDurationInSec * 1000)
+}

src/components/business/image-renderer.tsx

@@ -2,6 +2,7 @@ import { useEffect, useRef, useState } from "react"
 
 import { ImageSegment, RenderedScene } from "@/app/types"
 import { ProgressBar } from "../misc/progress"
+import { GameType } from "@/app/games/types"
 
 export const ImageRenderer = ({
   rendered: {
@@ -11,12 +12,14 @@ export const ImageRenderer = ({
   },
   onUserAction,
   onUserHover,
-  isLoading = false,
+  isLoading,
+  type,
 }: {
   rendered: RenderedScene
   onUserAction: (actionnable: string) => void
   onUserHover: (actionnable: string) => void
-  isLoading?: boolean
+  isLoading: boolean
+  type: GameType
 }) => {
   const timeoutRef = useRef<any>()
   const imgRef = useRef<HTMLImageElement | null>(null)
@@ -24,7 +27,8 @@ export const ImageRenderer = ({
   const contextRef = useRef<CanvasRenderingContext2D | null>(null)
   const [actionnable, setActionnable] = useState<string>("")
   const [progressPercent, setProcessPercent] = useState(0)
-  const showLoaderRef = useRef(true)
+  const progressRef = useRef(0)
+  const isLoadingRef = useRef(isLoading)
 
   useEffect(() => {
     if (maskBase64) {
@@ -147,49 +151,30 @@ export const ImageRenderer = ({
     }
   };
 
-  useEffect(() => {
-    clearTimeout(timeoutRef.current)
-    let progress = 0
-    setProcessPercent(0)
-    // note: when everything is fine, it takes about 45 seconds to render a new scene
-
-    const computeProgress = async () => {
-      if (!showLoaderRef.current) {
-        console.log("Asked to hide the loader")
-        progress = 100
-        setProcessPercent(100)
-        clearTimeout(timeoutRef.current)
-        return
-      }
+  const updateProgressBar = () => {
+    const duration = 1000 // 1 sec
+    const frequency = 200 // 200ms
+    const nbUpdatesPerSec = duration / frequency // 5x per second
 
-      // console.log("still loading..")
-
-      // console.log("updating progress")
-      progress = progress + 1
-      setProcessPercent(progress)
-
-      timeoutRef.current = setTimeout(() => {
-        computeProgress()
-      }, 1000)
-    }
+    // normally it takes 45, and we will try to go below,
+    // but to be safe let's set the counter a 1 min
+    const nbSeconds = 45 // 1 min
+    const amountInPercent =  100 / (nbUpdatesPerSec * nbSeconds) // 0.333
 
-    computeProgress()
-  }, [assetUrl])
+    progressRef.current = Math.min(100, progressRef.current + amountInPercent)
+    setProcessPercent(progressRef.current)
+  }
 
-  
   useEffect(() => {
-    showLoaderRef.current = isLoading || !assetUrl
-  }, [isLoading, assetUrl])
-  
+    clearInterval(timeoutRef.current)
+    isLoadingRef.current = isLoading
+    progressRef.current = 0
+    setProcessPercent(0)
+    if (isLoading) {
+      timeoutRef.current = setInterval(updateProgressBar, 200)
+    }
+  }, [isLoading, assetUrl, type])
 
-  if (!assetUrl) {
-    return <div className="flex w-full pt-8 items-center justify-center text-center">
-      <ProgressBar
-        text="⌛"
-        progressPercentage={progressPercent}
-      />
-    </div>
-  }
 
   /*
         <img
@@ -226,26 +211,37 @@ export const ImageRenderer = ({
   return (
     <div className={[
       "w-full py-8 px-2",
-      isLoading ? "animate-pulse" : ""
+      // isLoading ? "animate-pulse" : ""
     ].join(" ")
     }>
       <div className="relative w-full">
-        <img
-          src={assetUrl}
-          // src={"data:image/png;base64," + maskBase64}
-          ref={imgRef}
-          width="1024px"
-          height="512px"
-          className={
-            [
-             //  "absolute top-0 left-0",
-              actionnable && !isLoading ? "cursor-pointer" : ""
-            ].join(" ")
-          }
-          onMouseDown={(event) => handleMouseEvent(event, true)}
-          onMouseMove={handleMouseEvent}
+        {assetUrl
+          ? <img
+            src={assetUrl}
+            // src={"data:image/png;base64," + maskBase64}
+            ref={imgRef}
+            width="1024px"
+            height="512px"
+            className={
+              [
+              //  "absolute top-0 left-0",
+                actionnable && !isLoading ? "cursor-pointer" : ""
+              ].join(" ")
+            }
+            onMouseDown={(event) => handleMouseEvent(event, true)}
+            onMouseMove={handleMouseEvent}
+          />
+          : null}
+      </div>
+
+    {isLoading
+      ? <div className="fixed flex w-20 h-20 bottom-8 right-0 mr-8">
+        <ProgressBar
+          text="⌛"
+          progressPercentage={progressPercent}
         />
       </div>
+      : null}
     </div>
   )
 }

src/components/misc/progress.tsx

@@ -1,6 +1,6 @@
 "use client"
 
-import { CircularProgressbar } from "react-circular-progressbar"
+import { CircularProgressbar, buildStyles } from "react-circular-progressbar"
 import "react-circular-progressbar/dist/styles.css"
 
 export function ProgressBar ({
@@ -23,20 +23,20 @@ export function ProgressBar ({
         text={text || ""}   
 
         // Width of circular line relative to total width of component, a value from 0-100. Default: 8.
-        strokeWidth={8}
+        strokeWidth={12}
 
-        /*
+        
           // As a convenience, you can use buildStyles to configure the most common style changes:
 
           styles={buildStyles({
             // Rotation of path and trail, in number of turns (0-1)
-            rotation: 0.25,
+            rotation: 0,
 
             // Whether to use rounded or flat corners on the ends - can use 'butt' or 'round'
-            strokeLinecap: 'butt',
+            strokeLinecap: 'round',
 
             // Text size
-            textSize: '16px',
+            textSize: '40px',
 
             // How long animation takes to go from one percentage to another, in seconds
             pathTransitionDuration: 0.5,
@@ -45,12 +45,12 @@ export function ProgressBar ({
             // pathTransition: 'none',
 
             // Colors
-            pathColor: `rgba(62, 152, 199, ${percentage / 100})`,
+            // pathColor: `rgba(62, 152, 199, ${percentage / 100})`,
             textColor: '#f88',
             trailColor: '#d6d6d6',
             backgroundColor: '#3e98c7',
           })}
-          */
+      
       />
     </div>
   )

src/lib/fonts.ts

@@ -0,0 +1,49 @@
+import { Inter, Edu_SA_Beginner, Orbitron, Amatic_SC } from "next/font/google"
+import localFont from "next/font/local"
+
+export const inter = Inter({
+  subsets: ["latin"],
+  variable: "--font-inter",
+})
+
+export const edu = Edu_SA_Beginner({
+  subsets: ["latin"],
+  variable: "--font-edu",
+})
+
+export const orbitron = Orbitron({
+  subsets: ["latin"],
+  variable: "--font-orbitron",
+})
+
+export const amatic = Amatic_SC({
+  subsets: ["latin"],
+  weight: "400",
+  variable: "--font-amatic"
+})
+
+export const lugrasimo = localFont({
+  src: "../fonts/Lugrasimo-Regular.woff2",
+  variable: "--font-lugrasimo"
+})
+
+// https://fonts.google.com/specimen/Amatic+SC
+// https://fonts.google.com/specimen/Orbitron
+// https://fonts.google.com/specimen/Edu+SA+Beginner
+// https://fonts.google.com/specimen/Tektur
+
+// https://nextjs.org/docs/pages/building-your-application/optimizing/fonts 
+// If loading a variable font, you don"t need to specify the font weight
+export const fontList = [
+  inter,
+  edu,
+  orbitron,
+  amatic,
+  lugrasimo,
+]
+
+export const classNames = fontList.map(font => font.className)
+
+export const className = classNames.join(" ")
+
+export type FontName = "font-inter" | "font-sans" | "font-edu" | "font-orbitron" | "font-amatic" | "font-lugrasimo"

tailwind.config.js

@@ -6,6 +6,7 @@ module.exports = {
     './components/**/*.{ts,tsx}',
     './app/**/*.{ts,tsx}',
     './src/**/*.{ts,tsx}',
+    './src/lib/fonts.ts'
 	],
   theme: {
     container: {
@@ -16,6 +17,13 @@ module.exports = {
       },
     },
     extend: {
+      fontFamily: {
+        sans: ['var(--font-inter)'],
+        edu: ['var(--font-edu)'],
+        orbitron: ['var(--font-orbitron)'],
+        amatic: ['var(--font-amatic)'],
+        lugrasimo: ['var(--font-lugrasimo)'],
+      },
       keyframes: {
         "accordion-down": {
           from: { height: 0 },