experimenting with things

README.md

@@ -36,7 +36,8 @@ Rendering config:
 - `RENDERING_HF_INFERENCE_ENDPOINT_URL`: necessary if you decide to use a custom inference endpoint
 - `RENDERING_REPLICATE_API_MODEL_VERSION`: url to the VideoChain API server
 - `RENDERING_HF_INFERENCE_ENDPOINT_URL`: optional, default to nothing
-- `RENDERING_HF_INFERENCE_API_MODEL`: optional, defaults to "stabilityai/stable-diffusion-xl-base-1.0"
+- `RENDERING_HF_INFERENCE_API_BASE_MODEL`: optional, defaults to "stabilityai/stable-diffusion-xl-base-1.0"
+- `RENDERING_HF_INFERENCE_API_REFINER_MODEL`: optional, defaults to "stabilityai/stable-diffusion-xl-refiner-1.0"
 - `RENDERING_REPLICATE_API_MODEL`: optional, defaults to "stabilityai/sdxl"
 - `RENDERING_REPLICATE_API_MODEL_VERSION`: optional, in case you want to change the version
 

package.json

@@ -42,6 +42,7 @@
     "eslint": "8.45.0",
     "eslint-config-next": "13.4.10",
     "html2canvas": "^1.4.1",
+    "konva": "^9.2.2",
     "lucide-react": "^0.260.0",
     "next": "13.4.10",
     "openai": "^4.10.0",
@@ -51,6 +52,7 @@
     "react-circular-progressbar": "^2.1.0",
     "react-dom": "18.2.0",
     "react-icons": "^4.11.0",
+    "react-konva": "^18.2.10",
     "react-virtualized-auto-sizer": "^1.0.20",
     "replicate": "^0.17.0",
     "sbd": "^1.0.19",

src/app/engine/render.ts

@@ -169,6 +169,10 @@ export async function newRender({
       
       // note: there is no "refiner" step yet for custom inference endpoint
       // you probably don't need it anyway, as you probably want to deploy an all-in-one model instead for perf reasons
+      
+      // update: right now it is not possible to use it from the Inference API either:
+      // "Model type not found or pipeline not implemented"
+      /*
       if (renderingEngine === "INFERENCE_API") {
         try {
           const refinerModelUrl = `https://api-inference.huggingface.co/models/${huggingFaceInferenceApiRefinerModel}`
@@ -213,7 +217,8 @@ export async function newRender({
         } catch (err) {
           console.log(`Refiner step failed, but this is not a blocker. Error details: ${err}`)
         }
-      }
+      } */
+
 
       return {
         renderId: uuidv4(),

src/app/interface/page/index.tsx

@@ -1,4 +1,4 @@
-import { allLayouts } from "@/app/layouts"
+import { allLayoutAspectRatios, allLayouts } from "@/app/layouts"
 import { useStore } from "@/app/store"
 import { cn } from "@/lib/utils"
 import { useEffect, useRef } from "react"
@@ -9,7 +9,7 @@ export function Page({ page }: { page: number }) {
   // const prompt = useStore(state => state.prompt)
 
   const LayoutElement = (allLayouts as any)[layouts[page]]
-
+  const aspectRatio = ((allLayoutAspectRatios as any)[layouts[page]] as string) || "aspect-[250/297]"
   /*
   const [canLoad, setCanLoad] = useState(false)
   useEffect(() => {
@@ -36,11 +36,7 @@ export function Page({ page }: { page: number }) {
       ref={pageRef}
       className={cn(
         `w-full`,
-        // we are trying to reach a "book" look
-        // we are using aspect-[297/210] because it matches A4 (297mm x 210mm)
-        // `aspect-[210/297]`,
-        `aspect-[250/297]`,
-
+        aspectRatio,
         `transition-all duration-100 ease-in-out`,
         `border border-stone-200`,
         `shadow-2xl`,

src/app/interface/top-menu/index.tsx

@@ -31,7 +31,8 @@ const layoutIcons: Partial<Record<LayoutName, StaticImageData>> = {
   Layout0: layoutPreview0,
   Layout1: layoutPreview1,
   Layout2: layoutPreview2,
-  Layout3: layoutPreview3
+  Layout3: layoutPreview3,
+  Layout4: undefined,
 }
 
 export function TopMenu() {

src/app/layouts/index.tsx

@@ -252,21 +252,104 @@ export function Layout3() {
   )
 }
 
+// squares + vertical
+export function Layout4() {
+  return (
+    <Grid className="grid-cols-8 grid-rows-8">
+      <div className="bg-zinc-100 col-start-1 col-end-7 row-start-1 row-end-3">
+        <Panel
+          panel={0}
+          width={512}
+          height={1024}
+        />
+      </div>
+      <div className="bg-zinc-100 col-start-3 col-end-9 row-start-3 row-end-4">
+        <Panel
+          panel={1}
+          width={1024}
+          height={768}
+        />
+      </div>
+      <div className="bg-stone-100 col-start-2 col-end-8 row-start-4 row-end-6">
+        <Panel
+          panel={2}
+          width={768}
+          height={1024}
+        />
+      </div>
+      <div className="bg-slate-100 col-start-1 col-end-9 row-start-6 row-end-8">
+        <Panel
+          panel={3}
+          width={1024}
+          height={512}
+        />
+      </div>
+    </Grid>
+  )
+}
+
+// squares + horizontal
+export function Layout5() {
+  return (
+    <Grid className="grid-cols-4 grid-rows-4">
+      <div className="bg-zinc-100">
+        <Panel
+          panel={0}
+          width={1024}
+          height={1024}
+        />
+      </div>
+      <div className="bg-zinc-100">
+        <Panel
+          panel={1}
+          width={1024}
+          height={1024}
+        />
+      </div>
+      <div className="bg-stone-100">
+        <Panel
+          panel={2}
+          width={1024}
+          height={1024}
+        />
+      </div>
+      <div className="bg-slate-100">
+        <Panel
+          panel={3}
+          width={1024}
+          height={1024}
+        />
+      </div>
+    </Grid>
+  )
+}
+
 // export const layouts = { Layout1, Layout2_todo, Layout3_todo, Layout4_todo, Layout2, Layout3 }
 export const allLayouts = {
   random: <></>,
   Layout0,
   Layout1,
   Layout2,
-  Layout3 
+  Layout3,
+  Layout4
 }
 
 export const allLayoutLabels = {
   random: "Random layout",
-  Layout0: "Layout 0",
-  Layout1: "Layout 1",
-  Layout2: "Layout 2",
-  Layout3: "Layout 3",
+  Layout0: "Grid 0",
+  Layout1: "Grid 1",
+  Layout2: "Grid 2",
+  Layout3: "Grid 3",
+  // Layout4: "Blocks 1",
+}
+
+// note for reference: A4 (297mm x 210mm)
+export const allLayoutAspectRatios = {
+  Layout0: "aspect-[250/297]",
+  Layout1: "aspect-[250/297]",
+  Layout2: "aspect-[250/297]",
+  Layout3: "aspect-[250/297]",
+  // Layout4: "aspect-[1/3]",
 }
 
 export type LayoutName = keyof typeof allLayouts 

src/app/queries/getStory.ts

@@ -16,6 +16,9 @@ export const getStory = async ({
   prompt: string;
 }): Promise<LLMResponse> => {
 
+  // In case you need to quickly debug the RENDERING engine you can uncomment this:
+  // return mockLLMResponse
+
   const query = createLlamaPrompt([
     {
       role: "system",

src/app/queries/mockLLMResponse.ts

@@ -0,0 +1,24 @@
+import { LLMResponse } from "@/types"
+
+export const mockLLMResponse: LLMResponse = [
+  {
+    "panel": 1,
+    "instructions": "Close-up of cat's face, looking straight at reader with a smirk on its face",
+    "caption": "Feline mischief"
+  },
+  {
+    "panel": 2,
+    "instructions": "Medium shot of cat sniffing a glass of milk, with a surprised expression",
+    "caption": "Uh oh, what's this?"
+  },
+  {
+    "panel": 3,
+    "instructions": "Wide shot of cat knocking over the glass of milk, with a crazed look in its eyes",
+    "caption": "Cat-astrophe!"
+  },
+  {
+    "panel": 4,
+    "instructions": "Close-up of cat's face, looking satisfied with a milk moustache",
+    "caption": "Mission accomplished"
+  }
+]

src/components/ui/dialog.tsx

@@ -11,10 +11,9 @@ const Dialog = DialogPrimitive.Root
 const DialogTrigger = DialogPrimitive.Trigger
 
 const DialogPortal = ({
-  className,
   ...props
 }: DialogPrimitive.DialogPortalProps) => (
-  <DialogPrimitive.Portal className={cn(className)} {...props} />
+  <DialogPrimitive.Portal {...props} />
 )
 DialogPortal.displayName = DialogPrimitive.Portal.displayName
 

tailwind.config.js

@@ -57,6 +57,14 @@ module.exports = {
       },
       screens: {
         'print': { 'raw': 'print' },
+      },
+      gridTemplateColumns: {
+        '12': 'repeat(12, minmax(0, 1fr))',
+        '16': 'repeat(16, minmax(0, 1fr))',
+      },
+      gridTemplateRows: {
+        '12': 'repeat(12, minmax(0, 1fr))',
+        '16': 'repeat(16, minmax(0, 1fr))',
       }
     },
   },