Merge pull request #12793 from quarto-dev/fix/keep-ipynb

Handle keep-ipynb correctly now that fileInformationCache is responsible for the cleaning

Author: cderv

news/changelog-1.8.md

@@ -5,6 +5,7 @@ All changes included in 1.8:
 - ([#6607](https://github.com/quarto-dev/quarto-cli/issues/6607)): Add missing beamer template update for beamer theme options: `colorthemeoptions`, `fontthemeoptions`, `innerthemeoptions` and `outerthemeoptions`.
 - ([#12625](https://github.com/quarto-dev/quarto-cli/pull/12625)): Fire resize event on window when light/dark toggle is clicked, to tell widgets to resize.
 - ([#12657](https://github.com/quarto-dev/quarto-cli/pull/12657)): Load Giscus in generated script tag, to avoid wrong-theming in Chrome.
+- ([#12780](https://github.com/quarto-dev/quarto-cli/issues/12780)): `keep-ipynb: true` now works again correctly and intermediate `.quarto_ipynb` is not removed.
 
 ## Formats
 
@@ -63,4 +64,4 @@ All changes included in 1.8:
 ## Other fixes and improvements
 
 - ([#11321](https://github.com/quarto-dev/quarto-cli/issues/11321)): Follow [recommendation from LaTeX project](https://latex-project.org/news/latex2e-news/ltnews40.pdf) and use `lualatex` instead of `xelatex` as the default PDF engine.
-- ([#12782](https://github.com/quarto-dev/quarto-cli/pull/12782)): fix bug on `safeRemoveDirSync`'s detection of safe directory boundaries.
+- ([#12782](https://github.com/quarto-dev/quarto-cli/pull/12782)): fix bug on `safeRemoveDirSync`'s detection of safe directory boundaries.

src/command/render/render-contexts.ts

@@ -63,16 +63,13 @@ import { ExecutionEngine, ExecutionTarget } from "../../execute/types.ts";
 import {
   deleteProjectMetadata,
   directoryMetadataForInputFile,
-  projectTypeIsWebsite,
   toInputRelativePaths,
 } from "../../project/project-shared.ts";
 import {
   kProjectLibDir,
   kProjectType,
   ProjectContext,
 } from "../../project/types.ts";
-import { isHtmlDashboardOutput, isHtmlOutput } from "../../config/format.ts";
-import { formatHasBootstrap } from "../../format/html/format-html-info.ts";
 import { warnOnce } from "../../core/log.ts";
 import { dirAndStem } from "../../core/path.ts";
 import { fileExecutionEngineAndTarget } from "../../execute/engine.ts";
@@ -298,7 +295,7 @@ export async function renderContexts(
 
     // if this isn't for execute then cleanup context
     if (!forExecute && engine.executeTargetSkipped) {
-      engine.executeTargetSkipped(target, formats[formatKey].format);
+      engine.executeTargetSkipped(target, formats[formatKey].format, project);
     }
   }
   return contexts;

src/command/render/render-files.ts

@@ -200,7 +200,11 @@ export async function renderExecute(
 
         // notify engine that we skipped execute
         if (context.engine.executeTargetSkipped) {
-          context.engine.executeTargetSkipped(context.target, context.format);
+          context.engine.executeTargetSkipped(
+            context.target,
+            context.format,
+            context.project,
+          );
         }
 
         // return results

src/core/jupyter/jupyter-embed.ts

@@ -43,7 +43,13 @@ import {
   JupyterCellOutput,
 } from "../jupyter/types.ts";
 
-import { dirname, extname, join, basename, isAbsolute } from "../../deno_ral/path.ts";
+import {
+  basename,
+  dirname,
+  extname,
+  isAbsolute,
+  join,
+} from "../../deno_ral/path.ts";
 import { languages } from "../handlers/base.ts";
 import {
   extractJupyterWidgetDependencies,
@@ -596,6 +602,7 @@ async function getCachedNotebookInfo(
       quiet: flags.quiet,
       previewServer: context.options.previewServer,
       handledLanguages: languages(),
+      project: context.project,
     };
 
     const [dir, stem] = dirAndStem(nbAddress.path);

src/execute/jupyter/jupyter.ts

@@ -17,7 +17,7 @@ import { readYamlFromMarkdown } from "../../core/yaml.ts";
 import { isInteractiveSession } from "../../core/platform.ts";
 import { partitionMarkdown } from "../../core/pandoc/pandoc-partition.ts";
 
-import { dirAndStem, normalizePath, removeIfExists } from "../../core/path.ts";
+import { dirAndStem, normalizePath } from "../../core/path.ts";
 import { runningInCI } from "../../core/ci-info.ts";
 
 import {
@@ -109,7 +109,10 @@ import {
 import { jupyterCapabilities } from "../../core/jupyter/capabilities.ts";
 import { runExternalPreviewServer } from "../../preview/preview-server.ts";
 import { onCleanup } from "../../core/cleanup.ts";
-import { projectOutputDir } from "../../project/project-shared.ts";
+import {
+  ensureFileInformationCache,
+  projectOutputDir,
+} from "../../project/project-shared.ts";
 import { assert } from "testing/asserts";
 
 export const jupyterEngine: ExecutionEngine = {
@@ -436,7 +439,7 @@ export const jupyterEngine: ExecutionEngine = {
 
     // if it's a transient notebook then remove it
     // (unless keep-ipynb was specified)
-    cleanupNotebook(options.target, options.format);
+    cleanupNotebook(options.target, options.format, options.project);
 
     // Create markdown from the result
     const outputs = result.cellOutputs.map((output) => output.markdown);
@@ -713,12 +716,17 @@ async function disableDaemonForNotebook(target: ExecutionTarget) {
   return false;
 }
 
-function cleanupNotebook(target: ExecutionTarget, format: Format) {
-  // remove transient notebook if appropriate
+function cleanupNotebook(
+  target: ExecutionTarget,
+  format: Format,
+  project: ProjectContext,
+) {
+  // Make notebook non-transient when keep-ipynb is set
   const data = target.data as JupyterTargetData;
-  if (data.transient) {
-    if (!format.execute[kKeepIpynb]) {
-      removeIfExists(target.input);
+  const cached = ensureFileInformationCache(project, target.source);
+  if (data.transient && format.execute[kKeepIpynb]) {
+    if (cached.target && cached.target.data) {
+      (cached.target.data as JupyterTargetData).transient = false;
     }
   }
 }

src/execute/types.ts

@@ -49,7 +49,11 @@ export interface ExecutionEngine {
     format: Format,
   ) => Format;
   execute: (options: ExecuteOptions) => Promise<ExecuteResult>;
-  executeTargetSkipped?: (target: ExecutionTarget, format: Format) => void;
+  executeTargetSkipped?: (
+    target: ExecutionTarget,
+    format: Format,
+    project: ProjectContext,
+  ) => void;
   dependencies: (options: DependenciesOptions) => Promise<DependenciesResult>;
   postprocess: (options: PostProcessOptions) => Promise<void>;
   canFreeze: boolean;
@@ -89,7 +93,7 @@ export interface ExecuteOptions {
   quiet?: boolean;
   previewServer?: boolean;
   handledLanguages: string[]; // list of languages handled by cell language handlers, after the execution engine
-  project?: ProjectContext;
+  project: ProjectContext;
 }
 
 // result of execution

src/project/project-context.ts

@@ -70,6 +70,7 @@ import { projectResourceFiles } from "./project-resources.ts";
 
 import {
   cleanupFileInformationCache,
+  FileInformationCacheMap,
   ignoreFieldsForProjectType,
   normalizeFormatYaml,
   projectConfigFile,
@@ -272,7 +273,7 @@ export async function projectContext(
           dir: join(dir, ".quarto"),
           prefix: "quarto-session-temp",
         });
-        const fileInformationCache = new Map();
+        const fileInformationCache = new FileInformationCacheMap();
         const result: ProjectContext = {
           resolveBrand: async (fileName?: string) =>
             projectResolveBrand(result, fileName),
@@ -368,7 +369,7 @@ export async function projectContext(
           dir: join(dir, ".quarto"),
           prefix: "quarto-session-temp",
         });
-        const fileInformationCache = new Map();
+        const fileInformationCache = new FileInformationCacheMap();
         const result: ProjectContext = {
           resolveBrand: async (fileName?: string) =>
             projectResolveBrand(result, fileName),
@@ -443,7 +444,7 @@ export async function projectContext(
             dir: join(originalDir, ".quarto"),
             prefix: "quarto-session-temp",
           });
-          const fileInformationCache = new Map();
+          const fileInformationCache = new FileInformationCacheMap();
           const context: ProjectContext = {
             resolveBrand: async (fileName?: string) =>
               projectResolveBrand(context, fileName),

src/project/project-shared.ts

@@ -50,6 +50,7 @@ import { refSchema } from "../core/lib/yaml-schema/common.ts";
 import { Zod } from "../resources/types/zod/schema-types.ts";
 import { Brand } from "../core/brand/brand.ts";
 import { assert } from "testing/asserts";
+import { Cloneable } from "../core/safe-clone-deep.ts";
 
 export function projectExcludeDirs(context: ProjectContext): string[] {
   const outputDir = projectOutputDir(context);
@@ -633,6 +634,15 @@ export async function projectResolveBrand(
   }
 }
 
+// Create a class that extends Map and implements Cloneable
+export class FileInformationCacheMap extends Map<string, FileInformation>
+  implements Cloneable<Map<string, FileInformation>> {
+  clone(): Map<string, FileInformation> {
+    // Return the same instance (reference) instead of creating a clone
+    return this;
+  }
+}
+
 export function cleanupFileInformationCache(project: ProjectContext) {
   project.fileInformationCache.forEach((entry) => {
     if (entry?.target?.data) {

src/project/types/single-file/single-file.ts

@@ -21,6 +21,7 @@ import { MappedString } from "../../../core/mapped-text.ts";
 import { fileExecutionEngineAndTarget } from "../../../execute/engine.ts";
 import {
   cleanupFileInformationCache,
+  FileInformationCacheMap,
   projectFileMetadata,
   projectResolveBrand,
   projectResolveFullMarkdownForFile,
@@ -49,7 +50,7 @@ export async function singleFileProjectContext(
     notebookContext,
     environment: () => environmentMemoizer(result),
     renderFormats,
-    fileInformationCache: new Map(),
+    fileInformationCache: new FileInformationCacheMap(),
     fileExecutionEngineAndTarget: (
       file: string,
     ) => {

tests/docs/smoke-all/2025/05/21/keep_ipynb_project/.gitignore

@@ -0,0 +1 @@
+/.quarto/

tests/docs/smoke-all/2025/05/21/keep_ipynb_project/12780.qmd

@@ -0,0 +1,15 @@
+---
+format: html
+keep-ipynb: true
+_quarto:
+  tests:
+    html:
+      fileExists:
+        outputPath: 12780.quarto_ipynb
+      postRenderCleanup:
+        - ${input_stem}.quarto_ipynb
+---
+
+```{python}
+1 + 1
+```

tests/docs/smoke-all/2025/05/21/keep_ipynb_project/_quarto.yml

@@ -0,0 +1,2 @@
+project:
+  type: default

tests/docs/smoke-all/2025/05/21/keep_ipynb_single-file/12780.qmd

@@ -0,0 +1,15 @@
+---
+format: html
+keep-ipynb: true
+_quarto:
+  tests:
+    html:
+      fileExists:
+        outputPath: 12780.quarto_ipynb
+      postRenderCleanup:
+        - ${input_stem}.quarto_ipynb
+---
+
+```{python}
+1 + 1
+```