add a new experimental feature

package-lock.json

@@ -69,6 +69,7 @@
         "tailwindcss-animate": "^1.0.6",
         "ts-node": "^10.9.1",
         "typescript": "^5.4.5",
+        "use-file-picker": "^2.1.2",
         "usehooks-ts": "2.9.1",
         "uuid": "^9.0.0",
         "zustand": "^4.4.1"
@@ -4320,6 +4321,17 @@
         "node": "^10.12.0 || >=12.0.0"
       }
     },
+    "node_modules/file-selector": {
+      "version": "0.2.4",
+      "resolved": "https://registry.npmjs.org/file-selector/-/file-selector-0.2.4.tgz",
+      "integrity": "sha512-ZDsQNbrv6qRi1YTDOEWzf5J2KjZ9KMI1Q2SGeTkCJmNNW25Jg4TW4UMcmoqcg4WrAyKRcpBXdbWRxkfrOzVRbA==",
+      "dependencies": {
+        "tslib": "^2.0.3"
+      },
+      "engines": {
+        "node": ">= 10"
+      }
+    },
     "node_modules/fill-range": {
       "version": "7.0.1",
       "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz",
@@ -9992,6 +10004,20 @@
         }
       }
     },
+    "node_modules/use-file-picker": {
+      "version": "2.1.2",
+      "resolved": "https://registry.npmjs.org/use-file-picker/-/use-file-picker-2.1.2.tgz",
+      "integrity": "sha512-ZEIzRi1wXeIXDWr5i55gRBVER8rTkSGskDUY94bciTTAZJHlBnOTRLL/LDYjgz6d+US3yELHnRvtBhLxFGtB0A==",
+      "dependencies": {
+        "file-selector": "0.2.4"
+      },
+      "engines": {
+        "node": ">=12"
+      },
+      "peerDependencies": {
+        "react": ">=16"
+      }
+    },
     "node_modules/use-sidecar": {
       "version": "1.1.2",
       "resolved": "https://registry.npmjs.org/use-sidecar/-/use-sidecar-1.1.2.tgz",

package.json

@@ -70,6 +70,7 @@
     "tailwindcss-animate": "^1.0.6",
     "ts-node": "^10.9.1",
     "typescript": "^5.4.5",
+    "use-file-picker": "^2.1.2",
     "usehooks-ts": "2.9.1",
     "uuid": "^9.0.0",
     "zustand": "^4.4.1"

src/app/interface/about/index.tsx

@@ -8,8 +8,8 @@ import { Login } from "../login"
 const APP_NAME = `AI Comic Factory`
 const APP_DOMAIN = `aicomicfactory.app`
 const APP_URL = `https://aicomicfactory.app`
-const APP_VERSION = `1.3`
-const APP_RELEASE_DATE = `April 2024`
+const APP_VERSION = `1.4`
+const APP_RELEASE_DATE = `May 2024`
 
 const ExternalLink = ({ url, children }: { url: string; children: ReactNode }) => {
   return (

src/app/interface/bottom-bar/bottom-bar.tsx

@@ -1,4 +1,5 @@
 import { startTransition, useEffect, useState } from "react"
+import { useFilePicker } from 'use-file-picker'
 
 import { useStore } from "@/app/store"
 import { Button } from "@/components/ui/button"
@@ -14,6 +15,7 @@ import { useLocalStorage } from "usehooks-ts"
 import { localStorageKeys } from "../settings-dialog/localStorageKeys"
 import { defaultSettings } from "../settings-dialog/defaultSettings"
 import { getParam } from "@/lib/getParam"
+import { Input } from "@/components/ui/input"
 
 function BottomBar() {
   // deprecated, as HTML-to-bitmap didn't work that well for us
@@ -32,12 +34,15 @@ function BottomBar() {
   const allStatus = Object.values(panelGenerationStatus)
   const remainingImages = allStatus.reduce((acc, s) => (acc + (s ? 1 : 0)), 0)
 
+  const currentClap = useStore(s => s.currentClap)
+  
   const upscaleQueue = useStore(s => s.upscaleQueue)
   const renderedScenes = useStore(s => s.renderedScenes)
   const removeFromUpscaleQueue = useStore(s => s.removeFromUpscaleQueue)
   const setRendered = useStore(s => s.setRendered)
   const [isUpscaling, setUpscaling] = useState(false)
 
+  const loadClap = useStore(s => s.loadClap)
   const downloadClap = useStore(s => s.downloadClap)
 
   const [hasGeneratedAtLeastOnce, setHasGeneratedAtLeastOnce] = useLocalStorage<boolean>(
@@ -87,6 +92,27 @@ function BottomBar() {
     }
   }, [hasFinishedGeneratingImages, hasGeneratedAtLeastOnce])
 
+  const { openFilePicker, filesContent } = useFilePicker({
+    accept: '.clap',
+    readAs: "ArrayBuffer"
+  })
+  const fileData = filesContent[0]
+
+  useEffect(() => {
+    const fn = async () => {
+      if (fileData?.name) {
+        try {
+          const blob = new Blob([fileData.content])
+          await loadClap(blob)
+        } catch (err) {
+          console.error("failed to load the Clap file:", err)
+        }
+      }
+    }
+    fn()
+  }, [fileData?.name])
+
+
   return (
     <div className={cn(
       `print:hidden`,
@@ -152,21 +178,25 @@ function BottomBar() {
            </Button>
         </div>
           */}
+          {canSeeBetaFeatures ? <Button
+            onClick={openFilePicker}
+            disabled={remainingImages > 0}
+          >Load</Button> : null}
           {canSeeBetaFeatures ? <Button
             onClick={downloadClap}
-            disabled={!prompt?.length || remainingImages > 0}
+            disabled={!prompt?.length || remainingImages > 0 || !currentClap}
           >
-          {remainingImages ? `${allStatus.length - remainingImages}/${allStatus.length} ⌛` : `Save .clap`}
+          {remainingImages ? `${allStatus.length - remainingImages}/${allStatus.length} ⌛` : `Save`}
         </Button> : null}
           <Button
             onClick={handlePrint}
             disabled={!prompt?.length}
           >
             <span className="hidden md:inline">{
-            remainingImages ? `${allStatus.length - remainingImages}/${allStatus.length} panels ⌛` : `Save PDF`
+            remainingImages ? `${allStatus.length - remainingImages}/${allStatus.length} panels ⌛` : `Get PDF`
             }</span>
             <span className="inline md:hidden">{
-              remainingImages ? `${allStatus.length - remainingImages}/${allStatus.length} ⌛` : `Save`
+              remainingImages ? `${allStatus.length - remainingImages}/${allStatus.length} ⌛` : `PDF`
             }</span>
         </Button>
         <Share />

src/app/interface/panel/index.tsx

@@ -286,6 +286,15 @@ export function Panel({
   useEffect(() => {
     if (!prompt.length) { return }
 
+    const renderedScene: RenderedScene | undefined = useStore.getState().renderedScenes[panelIndex]
+
+    // I'm trying to find a rule to handle the case were we load a .clap file
+    // I think we should trash all the Panel objects for this to work properly 
+    if (renderedScene && renderedScene.status === "pregenerated" && renderedScene.assetUrl) {
+      console.log(`loading a pre-generated panel..`)
+      return
+    }
+
     startImageGeneration({ prompt, width, height, nbFrames, revision })
 
     clearTimeout(timeoutRef.current)
@@ -456,7 +465,13 @@ export function Panel({
           height={height}
           alt={rendered.alt}
           className={cn(
-            `comic-panel w-full h-full object-cover max-w-max`,
+            `comic-panel w-full h-full`,
+            `object-cover`,
+
+            // I think we can remove this to improve compatibility,
+            // in case the generate image isn't exactly the same size
+            // `max-w-max`,
+
             // showCaptions ? `-mt-11` : ''
             )}
         />}

src/app/interface/share/index.tsx

@@ -119,10 +119,10 @@ ${comicFileMd}`;
             disabled={!prompt?.length}
           >
             <span className="hidden md:inline">{
-            remainingImages ? `${allStatus.length - remainingImages}/${allStatus.length} panels ⌛` : `Save PDF`
+            remainingImages ? `${allStatus.length - remainingImages}/${allStatus.length} panels ⌛` : `Get PDF`
             }</span>
             <span className="inline md:hidden">{
-              remainingImages ? `${allStatus.length - remainingImages}/${allStatus.length} ⌛` : `Save`
+              remainingImages ? `${allStatus.length - remainingImages}/${allStatus.length} ⌛` : `PDF`
             }</span>
         </Button>
           </p>

src/app/interface/top-menu/index.tsx

@@ -79,6 +79,7 @@ export function TopMenu() {
     requestedStoryPrompt
   )
 
+  // TODO should be in the store
   const [draftPromptA, setDraftPromptA] = useState(lastDraftPromptA)
   const [draftPromptB, setDraftPromptB] = useState(lastDraftPromptB)
   const draftPrompt = `${draftPromptA}||${draftPromptB}`
@@ -242,6 +243,7 @@ export function TopMenu() {
         <div className="flex flex-row flex-grow w-full">
           <div className="flex flex-row flex-grow w-full">
             <Input
+              id="top-menu-input-story-prompt"
               placeholder="1. Story (eg. detective dog)"
               className={cn(
                 `w-1/2 rounded-r-none`,
@@ -260,6 +262,7 @@ export function TopMenu() {
               value={draftPromptB}
             />
             <Input
+              id="top-menu-input-style-prompt"
               placeholder="2. Style (eg 'rain, shiba')"
               className={cn(
                 `w-1/2`,

src/app/main.tsx

@@ -121,6 +121,13 @@ export default function Main() {
     // console.log(`main.tsx: asked to re-generate!!`)
     if (!prompt) { return }
 
+    // a quick and dirty hack to skip prompt regeneration,
+    // unless the prompt has really changed
+    if (prompt === useStore.getState().currentClap?.meta.description) {
+      console.log(`loading a pre-generated comic, so skipping prompt regeneration..`)
+      return
+    }
+
     // if the prompt or preset changed, we clear the cache
     // this part is important, otherwise when trying to change the prompt
     // we wouldn't still have remnants of the previous comic

src/app/store/index.ts

@@ -1,14 +1,16 @@
 "use client"
 
 import { create } from "zustand"
-import { ClapProject, newClap, newSegment, serializeClap } from "@aitube/clap"
+import { ClapProject, ClapSegment, ClapSegmentFilteringMode, filterSegments, newClap, newSegment, parseClap, serializeClap } from "@aitube/clap"
 
 import { FontName } from "@/lib/fonts"
 import { Preset, PresetName, defaultPreset, getPreset, getRandomPreset } from "@/app/engine/presets"
 import { RenderedScene } from "@/types"
-import { LayoutName, defaultLayout, getRandomLayoutName } from "../layouts"
 import { getParam } from "@/lib/getParam"
 
+import { LayoutName, defaultLayout, getRandomLayoutName } from "../layouts"
+import { putTextInInput } from "@/lib/putTextInInput"
+
 export const useStore = create<{
   prompt: string
   font: FontName
@@ -71,9 +73,17 @@ export const useStore = create<{
   // setPage: (page: HTMLDivElement) => void
 
   generate: (prompt: string, presetName: PresetName, layoutName: LayoutName) => void
-
   convertComicToClap: () => Promise<ClapProject>
-
+  convertClapToComic: (clap: ClapProject) => Promise<{
+    currentNbPanels: number
+    prompt: string
+    storyPrompt: string
+    stylePrompt: string
+    panels: string[]
+    renderedScenes: Record<string, RenderedScene>
+    captions: string[]
+  }>
+  loadClap: (blob: Blob) => Promise<void>
   downloadClap: () => Promise<void>
 }>((set, get) => ({
   prompt:
@@ -406,6 +416,7 @@ export const useStore = create<{
       layouts,
     })
   },
+  
   convertComicToClap: async (): Promise<ClapProject> => {
     const {
       currentNbPanels,
@@ -497,8 +508,120 @@ export const useStore = create<{
     return clap
   },
 
+  convertClapToComic: async (clap: ClapProject): Promise<{
+    currentNbPanels: number
+    prompt: string
+    storyPrompt: string
+    stylePrompt: string
+    panels: string[]
+    renderedScenes: Record<string, RenderedScene>
+    captions: string[]
+  }> => {
+
+    const prompt = clap.meta.description
+    const [stylePrompt, storyPrompt] = prompt.split("||").map(x => x.trim())
+
+    const panels: string[] = []
+    const renderedScenes: Record<string, RenderedScene> = {}
+    const captions: string[] = []
+
+    const panelGenerationStatus: Record<number, boolean> = {}
+
+    const cameraShots = clap.segments.filter(s => s.category === "camera")
+
+    const shots = cameraShots.map(cameraShot => ({
+      camera: cameraShot,
+      storyboard: filterSegments(
+        ClapSegmentFilteringMode.START,
+        cameraShot,
+        clap.segments,
+        "storyboard"
+      ).at(0) as (ClapSegment | undefined),
+      ui: filterSegments(
+        ClapSegmentFilteringMode.START,
+        cameraShot,
+        clap.segments,
+        "interface"
+      ).at(0) as (ClapSegment | undefined)
+    })).filter(item => item.storyboard && item.ui) as {
+      camera: ClapSegment
+      storyboard: ClapSegment
+      ui: ClapSegment
+    }[]
+
+    shots.forEach(({ camera, storyboard, ui }, id) => {
+
+      panels.push(storyboard.prompt)
+
+      const renderedScene: RenderedScene = {
+        renderId: storyboard.id,
+        status: "pending",
+        assetUrl: "", 
+        alt: storyboard.prompt,
+        error: "",
+        maskUrl: "",
+        segments: []
+      }
+
+      if (storyboard.assetUrl) {
+        renderedScene.assetUrl = storyboard.assetUrl
+        renderedScene.status = "pregenerated" // <- special trick to indicate that it should not be re-generated
+      }
+
+      renderedScenes[id] = renderedScene
+
+      panelGenerationStatus[id] = false
+      
+      captions.push(ui.prompt)
+    })
+
+   return {
+      currentNbPanels: shots.length,
+      prompt,
+      storyPrompt,
+      stylePrompt,
+      panels,
+      renderedScenes,
+      captions,
+
+    }
+  },
+
+  loadClap: async (blob: Blob) => {
+    const { convertClapToComic, currentNbPanelsPerPage } = get()
+
+    const currentClap = await parseClap(blob)
+
+    const {
+      currentNbPanels,
+      prompt,
+      storyPrompt,
+      stylePrompt,
+      panels,
+      renderedScenes,
+      captions,
+    } = await convertClapToComic(currentClap)
+
+    // kids, don't do this in your projects: use state managers instead!
+    putTextInInput(document.getElementById("top-menu-input-style-prompt") as HTMLInputElement, stylePrompt)
+    putTextInInput(document.getElementById("top-menu-input-story-prompt") as HTMLInputElement, storyPrompt)
+
+    set({
+      currentClap,
+      currentNbPanels,
+      prompt,
+      panels,
+      renderedScenes,
+      captions,
+      currentNbPages: Math.round(currentNbPanels / currentNbPanelsPerPage),
+      upscaleQueue: {},
+      isGeneratingStory: false,
+      isGeneratingText: false,
+    })
+  },
+
   downloadClap: async () => {
-    const { convertComicToClap } = get()
+    const { convertComicToClap, prompt } = get()
 
     const currentClap = await convertComicToClap()
 
@@ -513,7 +636,13 @@ export const useStore = create<{
     const anchor = document.createElement("a")
     anchor.href = objectUrl
 
-    anchor.download = "my_ai_comic.clap"
+    const [stylePrompt, storyPrompt] = prompt.split("||").map(x => x.trim())
+
+    const cleanStylePrompt = stylePrompt.replace(/([a-z0-9_,]+)/gi, "_")
+    const cleanStoryPrompt = storyPrompt.replace(/([a-z0-9_,]+)/gi, "_")
+    const cleanName = `${cleanStoryPrompt.slice(0, 20)} (${cleanStylePrompt.slice(0, 20) || "default style"})`
+
+    anchor.download = `${cleanName}.clap`
 
     document.body.appendChild(anchor) // Append to the body (could be removed once clicked)
     anchor.click() // Trigger the download
@@ -521,5 +650,5 @@ export const useStore = create<{
     // Cleanup: revoke the object URL and remove the anchor element
     URL.revokeObjectURL(objectUrl)
     document.body.removeChild(anchor)
-  }
+  },
 }))

src/lib/fileToBase64.ts

@@ -0,0 +1,8 @@
+export function fileToBase64(file: File | Blob): Promise<string> {
+  return new Promise((resolve, reject) => {
+    const fileReader = new FileReader();
+    fileReader.readAsDataURL(file);
+    fileReader.onload = () => { resolve(`${fileReader.result}`); };
+    fileReader.onerror = (error) => { reject(error); };
+  });
+}

src/lib/putTextInInput.ts

@@ -0,0 +1,18 @@
+export function putTextInInput(input?: HTMLInputElement, text: string = "") {
+  if (!input) { return }
+
+  const nativeTextAreaValueSetter = Object.getOwnPropertyDescriptor(
+    window.HTMLInputElement.prototype,
+    "value"
+  )?.set;
+
+  // fallback
+  if (!nativeTextAreaValueSetter) {
+    input.value = text
+    return
+  }
+
+  nativeTextAreaValueSetter.call(input, text)
+  const event = new Event('input', { bubbles: true });
+  input.dispatchEvent(event)
+}

src/types.ts

@@ -61,6 +61,7 @@ export interface ImageSegment {
 }
 
 export type RenderedSceneStatus =
+  | "pregenerated"
   | "pending"
   | "completed"
   | "error"