fix(cloudflare): added config for _routes.json generation (#8459)

* added config for _routes.json generation

* added changeset

* renamed test file

* updated comments

* Apply suggestions from code review

Co-authored-by: Sarah Rainsberger <sarah@rainsberger.ca>

* worked on tests

* worked on docs

* worked on docs

* worked on tests

* updated pnpm-lock.yaml

* worked on tests

* moved the _worker.js in cloudflareSpecialFiles statement

---------

Co-authored-by: Sarah Rainsberger <sarah@rainsberger.ca>
Co-authored-by: Alexander Niebuhr <alexander@nbhr.io>

.changeset/famous-seas-obey.md

@@ -0,0 +1,5 @@
+---
+'@astrojs/cloudflare': minor
+---
+
+Adds three new config options for `_routes.json` generation: `routes.strategy`, `routes.include`, and `routes.exclude`.

packages/integrations/cloudflare/README.md

@@ -75,6 +75,92 @@ export default defineConfig({
 
 Note that this adapter does not support using [Cloudflare Pages Middleware](https://developers.cloudflare.com/pages/platform/functions/middleware/). Astro will bundle the [Astro middleware](https://docs.astro.build/en/guides/middleware/) into each page.
 
+### routes.strategy
+
+`routes.strategy: "auto" | "include" | "exclude"`
+
+default `"auto"`
+
+Determines how `routes.json` will be generated if no [custom `_routes.json`](#custom-_routesjson) is provided.
+
+There are three options available:
+
+- **`"auto"` (default):** Will automatically select the strategy that generates the fewest entries. This should almost always be sufficient, so choose this option unless you have a specific reason not to.
+
+- **`include`:** Pages and endpoints that are not pre-rendered are listed as `include` entries, telling Cloudflare to invoke these routes as functions. `exclude` entries are only used to resolve conflicts. Usually the best strategy when your website has mostly static pages and only a few dynamic pages or endpoints.
+
+  Example: For `src/pages/index.astro` (static), `src/pages/company.astro` (static), `src/pages/users/faq.astro` (static) and `/src/pages/users/[id].astro` (SSR) this will produce the following `_routes.json`:
+
+  ```json
+  {
+    "version": 1,
+    "include": [
+      "/_image", // Astro's image endpoint
+      "/users/*" // Dynamic route
+    ],
+    "exclude": [
+      // Static routes that needs to be exempted from the dynamic wildcard route above
+      "/users/faq/",
+      "/users/faq/index.html"
+    ]
+  }
+  ```
+
+- **`exclude`:** Pre-rendered pages are listed as `exclude` entries (telling Cloudflare to handle these routes as static assets). Usually the best strategy when your website has mostly dynamic pages or endpoints and only a few static pages.
+
+  Example: For the same pages as in the previous example this will produce the following `_routes.json`:
+
+  ```json
+  {
+    "version": 1,
+    "include": [
+      "/*" // Handle everything as function except the routes below
+    ],
+    "exclude": [
+      // All static assets
+      "/",
+      "/company/",
+      "/index.html",
+      "/users/faq/",
+      "/favicon.png",
+      "/company/index.html",
+      "/users/faq/index.html"
+    ]
+  }
+  ```
+
+### routes.include
+
+`routes.include: string[]`
+
+default `[]`
+
+If you want to use the automatic `_routes.json` generation, but want to include additional routes (e.g. when having custom functions in the `functions` folder), you can use the `routes.include` option to add additional routes to the `include` array.
+
+### routes.exclude
+
+`routes.exclude: string[]`
+
+default `[]`
+
+If you want to use the automatic `_routes.json` generation, but want to exclude additional routes, you can use the `routes.exclude` option to add additional routes to the `exclude` array.
+
+The following example automatically generates `_routes.json` while including and excluding additional routes. Note that that is only necessary if you have custom functions in the `functions` folder that are not handled by Astro.
+
+```diff
+// astro.config.mjs
+export default defineConfig({
+    adapter: cloudflare({
+        mode: 'directory',
++       routes: {
++           strategy: 'include',
++           include: ['/users/*'], // handled by custom function: functions/users/[id].js
++           exclude: ['/users/faq'], // handled by static page: pages/users/faq.astro
++       },
+    }),
+});
+```
+
 ## Enabling Preview
 
 In order for preview to work you must install `wrangler`

packages/integrations/cloudflare/src/index.ts

@@ -21,6 +21,19 @@ export type { DirectoryRuntime } from './server.directory.js';
 type Options = {
 	mode?: 'directory' | 'advanced';
 	functionPerRoute?: boolean;
+	/** Configure automatic `routes.json` generation */
+	routes?: {
+		/** Strategy for generating `include` and `exclude` patterns
+		 * - `auto`: Will use the strategy that generates the least amount of entries.
+		 * - `include`: For each page or endpoint in your application that is not prerendered, an entry in the `include` array will be generated. For each page that is prerendered and whoose path is matched by an `include` entry, an entry in the `exclude` array will be generated.
+		 * - `exclude`: One `"/*"` entry in the `include` array will be generated. For each page that is prerendered, an entry in the `exclude` array will be generated.
+		 * */
+		strategy?: 'auto' | 'include' | 'exclude';
+		/** Additional `include` patterns */
+		include?: string[];
+		/** Additional `exclude` patterns */
+		exclude?: string[];
+	};
 	/**
 	 * 'off': current behaviour (wrangler is needed)
 	 * 'local': use a static req.cf object, and env vars defined in wrangler.toml & .dev.vars (astro dev is enough)
@@ -467,6 +480,7 @@ export default function createIntegration(args?: Options): AstroIntegration {
 
 				// move cloudflare specific files to the root
 				const cloudflareSpecialFiles = ['_headers', '_redirects', '_routes.json'];
+
 				if (_config.base !== '/') {
 					for (const file of cloudflareSpecialFiles) {
 						try {
@@ -480,6 +494,11 @@ export default function createIntegration(args?: Options): AstroIntegration {
 					}
 				}
 
+				// Add also the worker file so it's excluded from the _routes.json generation
+				if (!isModeDirectory) {
+					cloudflareSpecialFiles.push('_worker.js');
+				}
+
 				const routesExists = await fs.promises
 					.stat(new URL('./_routes.json', _config.outDir))
 					.then((stat) => stat.isFile())
@@ -587,39 +606,61 @@ export default function createIntegration(args?: Options): AstroIntegration {
 
 					staticPathList.push(...routes.filter((r) => r.type === 'redirect').map((r) => r.route));
 
-					// In order to product the shortest list of patterns, we first try to
-					// include all function endpoints, and then exclude all static paths
-					let include = deduplicatePatterns(
-						functionEndpoints.map((endpoint) => endpoint.includePattern)
-					);
-					let exclude = deduplicatePatterns(
-						staticPathList.filter((file: string) =>
-							functionEndpoints.some((endpoint) => endpoint.regexp.test(file))
-						)
-					);
+					const strategy = args?.routes?.strategy ?? 'auto';
+
+					// Strategy `include`: include all function endpoints, and then exclude static paths that would be matched by an include pattern
+					const includeStrategy =
+						strategy === 'exclude'
+							? undefined
+							: {
+									include: deduplicatePatterns(
+										functionEndpoints
+											.map((endpoint) => endpoint.includePattern)
+											.concat(args?.routes?.include ?? [])
+									),
+									exclude: deduplicatePatterns(
+										staticPathList
+											.filter((file: string) =>
+												functionEndpoints.some((endpoint) => endpoint.regexp.test(file))
+											)
+											.concat(args?.routes?.exclude ?? [])
+									),
+							  };
 
 					// Cloudflare requires at least one include pattern:
 					// https://developers.cloudflare.com/pages/platform/functions/routing/#limits
 					// So we add a pattern that we immediately exclude again
-					if (include.length === 0) {
-						include = ['/'];
-						exclude = ['/'];
+					if (includeStrategy?.include.length === 0) {
+						includeStrategy.include = ['/'];
+						includeStrategy.exclude = ['/'];
 					}
 
-					// If using only an exclude list would produce a shorter list of patterns,
-					// we use that instead
-					if (include.length + exclude.length > staticPathList.length) {
-						include = ['/*'];
-						exclude = deduplicatePatterns(staticPathList);
-					}
+					// Strategy `exclude`: include everything, and then exclude all static paths
+					const excludeStrategy =
+						strategy === 'include'
+							? undefined
+							: {
+									include: ['/*'],
+									exclude: deduplicatePatterns(staticPathList.concat(args?.routes?.exclude ?? [])),
+							  };
+
+					const includeStrategyLength = includeStrategy
+						? includeStrategy.include.length + includeStrategy.exclude.length
+						: Infinity;
+
+					const excludeStrategyLength = excludeStrategy
+						? excludeStrategy.include.length + excludeStrategy.exclude.length
+						: Infinity;
+
+					const winningStrategy =
+						includeStrategyLength <= excludeStrategyLength ? includeStrategy : excludeStrategy;
 
 					await fs.promises.writeFile(
 						new URL('./_routes.json', _config.outDir),
 						JSON.stringify(
 							{
 								version: 1,
-								include,
-								exclude,
+								...winningStrategy,
 							},
 							null,
 							2

packages/integrations/cloudflare/test/fixtures/routes-json/astro.config.mjs

@@ -1,11 +1,9 @@
 import { defineConfig } from 'astro/config';
-import cloudflare from '@astrojs/cloudflare';
 
 export default defineConfig({
-	adapter: cloudflare({ mode: 'directory' }),
+	// adapter will be set dynamically by the test
 	output: 'hybrid',
 	redirects: {
 		'/a/redirect': '/',
 	},
-	srcDir: process.env.SRC 
 });

packages/integrations/cloudflare/test/fixtures/routes-json/public/_redirects

@@ -0,0 +1 @@
+/redirectme / 302

packages/integrations/cloudflare/test/fixtures/routes-json/src/mixed/pages/a/[...rest].astro

@@ -0,0 +1,5 @@
+---
+export const prerender=false;
+---
+
+ok

packages/integrations/cloudflare/test/fixtures/routes-json/src/mixed/pages/a/[id].astro

@@ -0,0 +1,5 @@
+---
+export const prerender=false;
+---
+
+ok

packages/integrations/cloudflare/test/routes-json.test.js

@@ -0,0 +1,211 @@
+import { expect } from 'chai';
+import { loadFixture } from './test-utils.js';
+import cloudflare from '../dist/index.js';
+
+/** @type {import('./test-utils.js').Fixture} */
+describe('_routes.json generation', () => {
+	for (const mode of ['directory', 'advanced']) {
+		for (const functionPerRoute of [false, true]) {
+			describe(`with mode=${mode}, functionPerRoute=${functionPerRoute}`, () => {
+				describe('of both functions and static files', () => {
+					let fixture;
+
+					before(async () => {
+						fixture = await loadFixture({
+							root: './fixtures/routes-json/',
+							srcDir: './src/mixed',
+							adapter: cloudflare({
+								mode,
+								functionPerRoute,
+							}),
+						});
+						await fixture.build();
+					});
+
+					it('creates `include` for functions and `exclude` for static files where needed', async () => {
+						const _routesJson = await fixture.readFile('/_routes.json');
+						const routes = JSON.parse(_routesJson);
+
+						expect(routes).to.deep.equal({
+							version: 1,
+							include: ['/a/*', '/_image'],
+							exclude: ['/a/', '/a/redirect', '/a/index.html'],
+						});
+					});
+				});
+
+				describe('of only functions', () => {
+					let fixture;
+
+					before(async () => {
+						fixture = await loadFixture({
+							root: './fixtures/routes-json/',
+							srcDir: './src/dynamicOnly',
+							adapter: cloudflare({
+								mode,
+								functionPerRoute,
+							}),
+						});
+						await fixture.build();
+					});
+
+					it('creates a wildcard `include` and `exclude` only for static assets and redirects', async () => {
+						const _routesJson = await fixture.readFile('/_routes.json');
+						const routes = JSON.parse(_routesJson);
+
+						expect(routes).to.deep.equal({
+							version: 1,
+							include: ['/*'],
+							exclude: ['/public.txt', '/redirectme', '/a/redirect'],
+						});
+					});
+				});
+
+				describe('of only static files', () => {
+					let fixture;
+
+					before(async () => {
+						fixture = await loadFixture({
+							root: './fixtures/routes-json/',
+							srcDir: './src/staticOnly',
+							adapter: cloudflare({
+								mode,
+								functionPerRoute,
+							}),
+						});
+						await fixture.build();
+					});
+
+					it('create only one `include` and `exclude` that are supposed to match nothing', async () => {
+						const _routesJson = await fixture.readFile('/_routes.json');
+						const routes = JSON.parse(_routesJson);
+
+						expect(routes).to.deep.equal({
+							version: 1,
+							include: ['/_image'],
+							exclude: [],
+						});
+					});
+				});
+
+				describe('with strategy `"include"`', () => {
+					let fixture;
+
+					before(async () => {
+						fixture = await loadFixture({
+							root: './fixtures/routes-json/',
+							srcDir: './src/dynamicOnly',
+							adapter: cloudflare({
+								mode,
+								functionPerRoute,
+								routes: { strategy: 'include' },
+							}),
+						});
+						await fixture.build();
+					});
+
+					it('creates `include` entries even though the `"exclude"` strategy would have produced less entries.', async () => {
+						const _routesJson = await fixture.readFile('/_routes.json');
+						const routes = JSON.parse(_routesJson);
+
+						expect(routes).to.deep.equal({
+							version: 1,
+							include: ['/', '/_image', '/dynamic1', '/dynamic2', '/dynamic3'],
+							exclude: [],
+						});
+					});
+				});
+
+				describe('with strategy `"exclude"`', () => {
+					let fixture;
+
+					before(async () => {
+						fixture = await loadFixture({
+							root: './fixtures/routes-json/',
+							srcDir: './src/staticOnly',
+							adapter: cloudflare({
+								mode,
+								functionPerRoute,
+								routes: { strategy: 'exclude' },
+							}),
+						});
+						await fixture.build();
+					});
+
+					it('creates `exclude` entries even though the `"include"` strategy would have produced less entries.', async () => {
+						const _routesJson = await fixture.readFile('/_routes.json');
+						const routes = JSON.parse(_routesJson);
+
+						expect(routes).to.deep.equal({
+							version: 1,
+							include: ['/*'],
+							exclude: ['/', '/index.html', '/public.txt', '/redirectme', '/a/redirect'],
+						});
+					});
+				});
+
+				describe('with additional `include` entries', () => {
+					let fixture;
+
+					before(async () => {
+						fixture = await loadFixture({
+							root: './fixtures/routes-json/',
+							srcDir: './src/mixed',
+							adapter: cloudflare({
+								mode,
+								functionPerRoute,
+								routes: {
+									strategy: 'include',
+									include: ['/another', '/a/redundant'],
+								},
+							}),
+						});
+						await fixture.build();
+					});
+
+					it('creates `include` for functions and `exclude` for static files where needed', async () => {
+						const _routesJson = await fixture.readFile('/_routes.json');
+						const routes = JSON.parse(_routesJson);
+
+						expect(routes).to.deep.equal({
+							version: 1,
+							include: ['/a/*', '/_image', '/another'],
+							exclude: ['/a/', '/a/redirect', '/a/index.html'],
+						});
+					});
+				});
+
+				describe('with additional `exclude` entries', () => {
+					let fixture;
+
+					before(async () => {
+						fixture = await loadFixture({
+							root: './fixtures/routes-json/',
+							srcDir: './src/mixed',
+							adapter: cloudflare({
+								mode,
+								functionPerRoute,
+								routes: {
+									strategy: 'include',
+									exclude: ['/another', '/a/*', '/a/index.html'],
+								},
+							}),
+						});
+						await fixture.build();
+					});
+
+					it('creates `include` for functions and `exclude` for static files where needed', async () => {
+						const _routesJson = await fixture.readFile('/_routes.json');
+						const routes = JSON.parse(_routesJson);
+
+						expect(routes).to.deep.equal({
+							version: 1,
+							include: ['/a/*', '/_image'],
+							exclude: ['/a/', '/a/*', '/another'],
+						});
+					});
+				});
+			});
+		}
+	}
+});

packages/integrations/cloudflare/test/routesJson.js

@@ -1,78 +0,0 @@
-import { expect } from 'chai';
-import { loadFixture } from './test-utils.js';
-
-/** @type {import('./test-utils.js').Fixture} */
-describe('_routes.json generation', () => {
-	after(() => {
-		delete process.env.SRC;
-	});
-
-	describe('of both functions and static files', () => {
-		let fixture;
-
-		before(async () => {
-			process.env.SRC = './src/mixed';
-			fixture = await loadFixture({
-				root: './fixtures/routesJson/',
-			});
-			await fixture.build();
-		});
-
-		it('creates `include` for functions and `exclude` for static files where needed', async () => {
-			const _routesJson = await fixture.readFile('/_routes.json');
-			const routes = JSON.parse(_routesJson);
-
-			expect(routes).to.deep.equal({
-				version: 1,
-				include: ['/a/*', '/_image'],
-				exclude: ['/a/', '/a/redirect', '/a/index.html'],
-			});
-		});
-	});
-
-	describe('of only functions', () => {
-		let fixture;
-
-		before(async () => {
-			process.env.SRC = './src/dynamicOnly';
-			fixture = await loadFixture({
-				root: './fixtures/routesJson/',
-			});
-			await fixture.build();
-		});
-
-		it('creates a wildcard `include` and `exclude` only for the redirect', async () => {
-			const _routesJson = await fixture.readFile('/_routes.json');
-			const routes = JSON.parse(_routesJson);
-
-			expect(routes).to.deep.equal({
-				version: 1,
-				include: ['/*'],
-				exclude: ['/a/redirect'],
-			});
-		});
-	});
-
-	describe('of only static files', () => {
-		let fixture;
-
-		before(async () => {
-			process.env.SRC = './src/staticOnly';
-			fixture = await loadFixture({
-				root: './fixtures/routesJson/',
-			});
-			await fixture.build();
-		});
-
-		it('create only one `include` and `exclude` that are supposed to match nothing', async () => {
-			const _routesJson = await fixture.readFile('/_routes.json');
-			const routes = JSON.parse(_routesJson);
-
-			expect(routes).to.deep.equal({
-				version: 1,
-				include: ['/_image'],
-				exclude: [],
-			});
-		});
-	});
-});

pnpm-lock.yaml

@@ -3720,7 +3720,7 @@ importers:
         specifier: workspace:*
         version: link:../../../../../astro
 
-  packages/integrations/cloudflare/test/fixtures/routesJson:
+  packages/integrations/cloudflare/test/fixtures/routes-json:
     dependencies:
       '@astrojs/cloudflare':
         specifier: workspace:*