Aggregate PagesData routes into the module graph when enabled

Fix duplicate module idents by constructing PageData reference types with a query parameter

Author: lukesandberg

crates/next-api/benches/hmr.rs

@@ -349,7 +349,9 @@ impl HmrBenchmark {
                     data_endpoint,
                 } => {
                     let _ = endpoint_write_to_disk(**html_endpoint).await?;
-                    let _ = endpoint_write_to_disk(**data_endpoint).await?;
+                    if let Some(data_endpoint) = data_endpoint {
+                        let _ = endpoint_write_to_disk(**data_endpoint).await?;
+                    }
                 }
                 next_api::route::Route::PageApi { endpoint } => {
                     let _ = endpoint_write_to_disk(**endpoint).await?;

crates/next-api/src/operation.rs

@@ -166,7 +166,7 @@ async fn pick_endpoint(
         }
         EndpointSelector::RoutePageData(name) => {
             if let Some(Route::Page { data_endpoint, .. }) = endpoints.routes.get(&name) {
-                Some(*data_endpoint)
+                *data_endpoint
             } else {
                 None
             }

crates/next-api/src/pages.rs

@@ -199,18 +199,23 @@ impl PagesProject {
                         .to_resolved()
                         .await?,
                     ),
-                    data_endpoint: ResolvedVc::upcast(
-                        PageEndpoint::new(
-                            PageEndpointType::Data,
-                            self,
-                            pathname,
-                            original_name,
-                            page,
-                            pages_structure,
-                        )
-                        .to_resolved()
-                        .await?,
-                    ),
+                    // The data endpoint is only needed in development mode to support HMR
+                    data_endpoint: if self.project().next_mode().await?.is_development() {
+                        Some(ResolvedVc::upcast(
+                            PageEndpoint::new(
+                                PageEndpointType::Data,
+                                self,
+                                pathname,
+                                original_name,
+                                page,
+                                pages_structure,
+                            )
+                            .to_resolved()
+                            .await?,
+                        ))
+                    } else {
+                        None
+                    },
                 })
             })
         };
@@ -635,9 +640,9 @@ impl PageEndpoint {
             // When creating a data endpoint we also create an Html endpoint for the same source
             // So add a query parameter to differentiate between the two.
             if self.ty == PageEndpointType::Data {
-                RcStr::from("?server-data")
+                rcstr!("?server-data")
             } else {
-                RcStr::default()
+                rcstr!("")
             },
         )))
     }
@@ -703,8 +708,8 @@ impl PageEndpoint {
         let this = self.await?;
         let project = this.pages_project.project();
 
-        let should_trace = project.next_mode().await?.is_production();
         if *project.per_page_module_graph().await? {
+            let should_trace = project.next_mode().await?.is_production();
             let ssr_chunk_module = self.internal_ssr_chunk_module().await?;
             // Implements layout segment optimization to compute a graph "chain" for document, app,
             // page
@@ -996,7 +1001,12 @@ impl PageEndpoint {
                             ssr_module_graph,
                             current_availability_info,
                         )
-                        .await?;
+                        .await
+                        .context(format!(
+                            "failed to find chunk group for {} for {:?}",
+                            layout.ident().to_string().await?,
+                            this.ty,
+                        ))?;
 
                     current_chunks = current_chunks
                         .concatenate(*chunk_group.assets)

crates/next-api/src/project.rs

@@ -863,10 +863,13 @@ impl Project {
             match route {
                 Route::Page {
                     html_endpoint,
-                    data_endpoint: _,
+                    data_endpoint,
                 } => {
                     if !app_dir_only {
                         endpoints.push(*html_endpoint);
+                        if let Some(data_endpoint) = data_endpoint {
+                            endpoints.push(*data_endpoint);
+                        }
                     }
                 }
                 Route::PageApi { endpoint } => {

crates/next-api/src/route.rs

@@ -34,7 +34,7 @@ pub struct AppPageRoute {
 pub enum Route {
     Page {
         html_endpoint: ResolvedVc<Box<dyn Endpoint>>,
-        data_endpoint: ResolvedVc<Box<dyn Endpoint>>,
+        data_endpoint: Option<ResolvedVc<Box<dyn Endpoint>>>,
     },
     PageApi {
         endpoint: ResolvedVc<Box<dyn Endpoint>>,

crates/next-core/src/next_config.rs

@@ -1821,7 +1821,7 @@ impl NextConfig {
             NextMode::Development => self
                 .experimental
                 .turbopack_use_whole_app_module_graph_in_dev
-                .unwrap_or(false),
+                .unwrap_or(true),
             NextMode::Build => true,
         }))
     }

crates/next-core/src/next_pages/page_entry.rs

@@ -112,8 +112,10 @@ pub async fn create_page_ssr_entry_module(
 
         let file = File::from(result.build());
 
-        source = Vc::upcast(VirtualSource::new(
-            source.ident().path().owned().await?,
+        source = Vc::upcast(VirtualSource::new_with_ident(
+            // Reuse the ident of the original source since it may contain query parameters or
+            // modifiers that are used to make the source unique.
+            source.ident(),
             AssetContent::file(file.into()),
         ));
     }
@@ -197,8 +199,17 @@ async fn process_global_item(
     reference_type: ReferenceType,
     module_context: Vc<Box<dyn AssetContext>>,
 ) -> Result<Vc<Box<dyn Module>>> {
-    let source = Vc::upcast(FileSource::new(item.file_path().owned().await?));
-    Ok(module_context.process(source, reference_type).module())
+    let source = Vc::upcast(FileSource::new_with_query(
+        item.file_path().owned().await?,
+        if reference_type == ReferenceType::Entry(EntryReferenceSubType::PageData) {
+            rcstr!("?server-data")
+        } else {
+            rcstr!("")
+        },
+    ));
+    Ok(module_context
+        .process(source, reference_type.clone())
+        .module())
 }
 
 #[turbo_tasks::function]

packages/next/src/server/config-shared.ts

@@ -499,7 +499,8 @@ export interface ExperimentalConfig {
   turbopackUseBuiltinSass?: boolean
 
   /**
-   * Whether to use the whole app module graph in development mode.  This makes development mode chunking more like production but may come with an HMR performance regression.
+   * Whether to compute the whole app module graph in development mode. This may reduce memory usage
+   * by enabling more sharing of chunking decisions across routes.
    */
   turbopackUseWholeAppModuleGraphInDev?: boolean
 

turbopack/crates/turbopack-core/src/chunk/chunking/dev.rs

@@ -86,6 +86,8 @@ pub async fn app_vendors_split<'l>(
     let mut chunk_group_specific_chunk_items = Vec::new();
     let mut app_chunk_items = Vec::new();
     let mut vendors_chunk_items = Vec::new();
+    // TODO this needs to create new chunk groups for each async entry point to preserver `import()`
+    // semantics, even if it leads to more chunks.
     for item in chunk_items {
         let ChunkItemOrBatchWithInfo::ChunkItem {
             chunk_item: ChunkItemWithAsyncModuleInfo { module, .. },

turbopack/crates/turbopack-core/src/chunk/chunking_context.rs

@@ -136,6 +136,7 @@ pub struct ChunkingConfigs(FxHashMap<ResolvedVc<Box<dyn ChunkType>>, ChunkingCon
 pub trait ChunkingContext {
     #[turbo_tasks::function]
     fn name(self: Vc<Self>) -> Vc<RcStr>;
+    /// Whether to use file URIs in source maps.
     #[turbo_tasks::function]
     fn should_use_file_source_map_uris(self: Vc<Self>) -> Vc<bool>;
     /// The root path of the project