[Content collections] Load content config with full Vite setup (#6092)

* feat: use vite dev server for content config

* refactor: improve export naming

* chore: update `sync` to spin up server

* refactor: run sync before build in cli

* fix: move sync call to build setup

* chore: clean up attachContent... types

* chore: remove unneeded comment

* chore: changeset

* fix: attachContentServerListeners in unit tests

* fix: allow forced contentDirExists

* chore: update schema signature

* fix: move content listeners to unit test

* chore remove contentDirExists flag; unused

* chore: stub weird unit test fix
This commit is contained in:
Ben Holmes 2023-02-03 15:52:05 -05:00 committed by GitHub
parent db2c59fc18
commit bf8d7366ac
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 286 additions and 236 deletions

View file

@ -0,0 +1,5 @@
---
'astro': patch
---
Ensure vite config (aliases, custom modules, etc) is respected when loading the content collection config

View file

@ -1,9 +1,12 @@
import { dim } from 'kleur/colors';
import type fsMod from 'node:fs';
import { performance } from 'node:perf_hooks';
import { createServer } from 'vite';
import type { AstroSettings } from '../../@types/astro';
import { contentObservable, createContentTypesGenerator } from '../../content/index.js';
import { createContentTypesGenerator } from '../../content/index.js';
import { globalContentConfigObserver } from '../../content/utils.js';
import { getTimeStat } from '../../core/build/util.js';
import { createVite } from '../../core/create-vite.js';
import { AstroError, AstroErrorData } from '../../core/errors/index.js';
import { info, LogOptions } from '../../core/logger/core.js';
import { setUpEnvTs } from '../../vite-plugin-inject-env-ts/index.js';
@ -13,13 +16,25 @@ export async function sync(
{ logging, fs }: { logging: LogOptions; fs: typeof fsMod }
): Promise<0 | 1> {
const timerStart = performance.now();
// Needed to load content config
const tempViteServer = await createServer(
await createVite(
{
server: { middlewareMode: true, hmr: false },
optimizeDeps: { entries: [] },
logLevel: 'silent',
},
{ settings, logging, mode: 'build', fs }
)
);
try {
const contentTypesGenerator = await createContentTypesGenerator({
contentConfigObserver: contentObservable({ status: 'loading' }),
contentConfigObserver: globalContentConfigObserver,
logging,
fs,
settings,
viteServer: tempViteServer,
});
const typesResult = await contentTypesGenerator.init();
if (typesResult.typesGenerated === false) {
@ -32,6 +47,8 @@ export async function sync(
}
} catch (e) {
throw new AstroError(AstroErrorData.GenerateContentTypesError);
} finally {
await tempViteServer.close();
}
info(logging, 'content', `Types generated ${dim(getTimeStat(timerStart, performance.now()))}`);

View file

@ -4,5 +4,6 @@ export {
astroContentAssetPropagationPlugin,
astroContentProdBundlePlugin,
} from './vite-plugin-content-assets.js';
export { astroContentServerPlugin } from './vite-plugin-content-server.js';
export { astroContentImportPlugin } from './vite-plugin-content-imports.js';
export { attachContentServerListeners } from './server-listeners.js';
export { astroContentVirtualModPlugin } from './vite-plugin-content-virtual-mod.js';

View file

@ -0,0 +1,73 @@
import { cyan } from 'kleur/colors';
import { pathToFileURL } from 'node:url';
import type fsMod from 'node:fs';
import type { ViteDevServer } from 'vite';
import type { AstroSettings } from '../@types/astro.js';
import { info, LogOptions } from '../core/logger/core.js';
import { appendForwardSlash } from '../core/path.js';
import { createContentTypesGenerator } from './types-generator.js';
import { globalContentConfigObserver, getContentPaths } from './utils.js';
interface ContentServerListenerParams {
fs: typeof fsMod;
logging: LogOptions;
settings: AstroSettings;
viteServer: ViteDevServer;
}
export async function attachContentServerListeners({
viteServer,
fs,
logging,
settings,
}: ContentServerListenerParams) {
const contentPaths = getContentPaths(settings.config);
if (fs.existsSync(contentPaths.contentDir)) {
info(
logging,
'content',
`Watching ${cyan(
contentPaths.contentDir.href.replace(settings.config.root.href, '')
)} for changes`
);
await attachListeners();
} else {
viteServer.watcher.on('addDir', contentDirListener);
async function contentDirListener(dir: string) {
if (appendForwardSlash(pathToFileURL(dir).href) === contentPaths.contentDir.href) {
info(logging, 'content', `Content dir found. Watching for changes`);
await attachListeners();
viteServer.watcher.removeListener('addDir', contentDirListener);
}
}
}
async function attachListeners() {
const contentGenerator = await createContentTypesGenerator({
fs,
settings,
logging,
viteServer,
contentConfigObserver: globalContentConfigObserver,
});
await contentGenerator.init();
info(logging, 'content', 'Types generated');
viteServer.watcher.on('add', (entry) => {
contentGenerator.queueEvent({ name: 'add', entry });
});
viteServer.watcher.on('addDir', (entry) =>
contentGenerator.queueEvent({ name: 'addDir', entry })
);
viteServer.watcher.on('change', (entry) =>
contentGenerator.queueEvent({ name: 'change', entry })
);
viteServer.watcher.on('unlink', (entry) => {
contentGenerator.queueEvent({ name: 'unlink', entry });
});
viteServer.watcher.on('unlinkDir', (entry) =>
contentGenerator.queueEvent({ name: 'unlinkDir', entry })
);
}
}

View file

@ -3,7 +3,7 @@ import { cyan } from 'kleur/colors';
import type fsMod from 'node:fs';
import * as path from 'node:path';
import { fileURLToPath, pathToFileURL } from 'node:url';
import { normalizePath } from 'vite';
import { normalizePath, ViteDevServer } from 'vite';
import type { AstroSettings } from '../@types/astro.js';
import { info, LogOptions, warn } from '../core/logger/core.js';
import { appendForwardSlash, isRelativePath } from '../core/path.js';
@ -32,6 +32,8 @@ type CreateContentGeneratorParams = {
contentConfigObserver: ContentObservable;
logging: LogOptions;
settings: AstroSettings;
/** This is required for loading the content config */
viteServer: ViteDevServer;
fs: typeof fsMod;
};
@ -44,6 +46,7 @@ export async function createContentTypesGenerator({
fs,
logging,
settings,
viteServer,
}: CreateContentGeneratorParams) {
const contentTypes: ContentTypes = {};
const contentPaths = getContentPaths(settings.config);
@ -113,7 +116,7 @@ export async function createContentTypesGenerator({
}
if (fileType === 'config') {
contentConfigObserver.set({ status: 'loading' });
const config = await loadContentConfig({ fs, settings });
const config = await loadContentConfig({ fs, settings, viteServer });
if (config) {
contentConfigObserver.set({ status: 'loaded', config });
} else {

View file

@ -205,34 +205,32 @@ export function parseFrontmatter(fileContents: string, filePath: string) {
}
}
/**
* The content config is loaded separately from other `src/` files.
* This global observable lets dependent plugins (like the content flag plugin)
* subscribe to changes during dev server updates.
*/
export const globalContentConfigObserver = contentObservable({ status: 'init' });
export async function loadContentConfig({
fs,
settings,
viteServer,
}: {
fs: typeof fsMod;
settings: AstroSettings;
viteServer: ViteDevServer;
}): Promise<ContentConfig | undefined> {
const contentPaths = getContentPaths(settings.config);
const tempConfigServer: ViteDevServer = await createServer({
root: fileURLToPath(settings.config.root),
server: { middlewareMode: true, hmr: false },
optimizeDeps: { entries: [] },
clearScreen: false,
appType: 'custom',
logLevel: 'silent',
plugins: [astroContentVirtualModPlugin({ settings })],
});
let unparsedConfig;
if (!fs.existsSync(contentPaths.config)) {
return undefined;
}
try {
const configPathname = fileURLToPath(contentPaths.config);
unparsedConfig = await tempConfigServer.ssrLoadModule(configPathname);
unparsedConfig = await viteServer.ssrLoadModule(configPathname);
} catch (e) {
throw e;
} finally {
await tempConfigServer.close();
}
const config = contentConfigParser.safeParse(unparsedConfig);
if (config.success) {
@ -243,6 +241,7 @@ export async function loadContentConfig({
}
type ContentCtx =
| { status: 'init' }
| { status: 'loading' }
| { status: 'error' }
| { status: 'loaded'; config: ContentConfig };

View file

@ -0,0 +1,129 @@
import * as devalue from 'devalue';
import { pathToFileURL } from 'url';
import type { Plugin } from 'vite';
import type fsMod from 'node:fs';
import { AstroSettings } from '../@types/astro.js';
import { contentFileExts, CONTENT_FLAG } from './consts.js';
import {
ContentConfig,
globalContentConfigObserver,
getContentPaths,
getEntryData,
getEntryInfo,
getEntrySlug,
parseFrontmatter,
} from './utils.js';
import { escapeViteEnvReferences, getFileInfo } from '../vite-plugin-utils/index.js';
import { getEntryType } from './types-generator.js';
import { AstroError } from '../core/errors/errors.js';
import { AstroErrorData } from '../core/errors/errors-data.js';
function isContentFlagImport(viteId: string) {
const { pathname, searchParams } = new URL(viteId, 'file://');
return searchParams.has(CONTENT_FLAG) && contentFileExts.some((ext) => pathname.endsWith(ext));
}
export function astroContentImportPlugin({
fs,
settings,
}: {
fs: typeof fsMod;
settings: AstroSettings;
}): Plugin {
const contentPaths = getContentPaths(settings.config);
return {
name: 'astro:content-imports',
async load(id) {
const { fileId } = getFileInfo(id, settings.config);
if (isContentFlagImport(id)) {
const observable = globalContentConfigObserver.get();
// Content config should be loaded before this plugin is used
if (observable.status === 'init') {
throw new AstroError({
...AstroErrorData.UnknownContentCollectionError,
message: 'Content config failed to load.',
});
}
let contentConfig: ContentConfig | undefined =
observable.status === 'loaded' ? observable.config : undefined;
if (observable.status === 'loading') {
// Wait for config to load
contentConfig = await new Promise((resolve) => {
const unsubscribe = globalContentConfigObserver.subscribe((ctx) => {
if (ctx.status === 'loaded') {
resolve(ctx.config);
unsubscribe();
} else if (ctx.status === 'error') {
resolve(undefined);
unsubscribe();
}
});
});
}
const rawContents = await fs.promises.readFile(fileId, 'utf-8');
const {
content: body,
data: unparsedData,
matter: rawData = '',
} = parseFrontmatter(rawContents, fileId);
const entryInfo = getEntryInfo({
entry: pathToFileURL(fileId),
contentDir: contentPaths.contentDir,
});
if (entryInfo instanceof Error) return;
const _internal = { filePath: fileId, rawData };
const partialEntry = { data: unparsedData, body, _internal, ...entryInfo };
// TODO: move slug calculation to the start of the build
// to generate a performant lookup map for `getEntryBySlug`
const slug = getEntrySlug(partialEntry);
const collectionConfig = contentConfig?.collections[entryInfo.collection];
const data = collectionConfig
? await getEntryData(partialEntry, collectionConfig)
: unparsedData;
const code = escapeViteEnvReferences(`
export const id = ${JSON.stringify(entryInfo.id)};
export const collection = ${JSON.stringify(entryInfo.collection)};
export const slug = ${JSON.stringify(slug)};
export const body = ${JSON.stringify(body)};
export const data = ${devalue.uneval(data) /* TODO: reuse astro props serializer */};
export const _internal = {
filePath: ${JSON.stringify(fileId)},
rawData: ${JSON.stringify(rawData)},
};
`);
return { code };
}
},
configureServer(viteServer) {
viteServer.watcher.on('all', async (event, entry) => {
if (
['add', 'unlink', 'change'].includes(event) &&
getEntryType(entry, contentPaths) === 'config'
) {
// Content modules depend on config, so we need to invalidate them.
for (const modUrl of viteServer.moduleGraph.urlToModuleMap.keys()) {
if (isContentFlagImport(modUrl)) {
const mod = await viteServer.moduleGraph.getModuleByUrl(modUrl);
if (mod) {
viteServer.moduleGraph.invalidateModule(mod);
}
}
}
}
});
},
async transform(code, id) {
if (isContentFlagImport(id)) {
// Escape before Rollup internal transform.
// Base on MUCH trial-and-error, inspired by MDX integration 2-step transform.
return { code: escapeViteEnvReferences(code) };
}
},
};
}

View file

@ -1,195 +0,0 @@
import * as devalue from 'devalue';
import { cyan } from 'kleur/colors';
import fsMod from 'node:fs';
import { pathToFileURL } from 'node:url';
import type { Plugin } from 'vite';
import type { AstroSettings } from '../@types/astro.js';
import { info, LogOptions } from '../core/logger/core.js';
import { appendForwardSlash } from '../core/path.js';
import { escapeViteEnvReferences, getFileInfo } from '../vite-plugin-utils/index.js';
import { contentFileExts, CONTENT_FLAG } from './consts.js';
import { createContentTypesGenerator, getEntryType } from './types-generator.js';
import {
ContentConfig,
contentObservable,
getContentPaths,
getEntryData,
getEntryInfo,
getEntrySlug,
parseFrontmatter,
} from './utils.js';
interface AstroContentServerPluginParams {
fs: typeof fsMod;
logging: LogOptions;
settings: AstroSettings;
mode: string;
}
export function astroContentServerPlugin({
fs,
settings,
logging,
mode,
}: AstroContentServerPluginParams): Plugin[] {
const contentPaths = getContentPaths(settings.config);
const contentConfigObserver = contentObservable({ status: 'loading' });
async function initContentGenerator() {
const contentGenerator = await createContentTypesGenerator({
fs,
settings,
logging,
contentConfigObserver,
});
await contentGenerator.init();
return contentGenerator;
}
return [
{
name: 'astro-content-server-plugin',
async config(viteConfig) {
// Production build type gen
if (fs.existsSync(contentPaths.contentDir) && viteConfig.build?.ssr === true) {
await initContentGenerator();
}
},
async configureServer(viteServer) {
if (mode !== 'dev') return;
// Dev server type gen
if (fs.existsSync(contentPaths.contentDir)) {
info(
logging,
'content',
`Watching ${cyan(
contentPaths.contentDir.href.replace(settings.config.root.href, '')
)} for changes`
);
await attachListeners();
} else {
viteServer.watcher.on('addDir', contentDirListener);
async function contentDirListener(dir: string) {
if (appendForwardSlash(pathToFileURL(dir).href) === contentPaths.contentDir.href) {
info(logging, 'content', `Content dir found. Watching for changes`);
await attachListeners();
viteServer.watcher.removeListener('addDir', contentDirListener);
}
}
}
async function attachListeners() {
const contentGenerator = await initContentGenerator();
info(logging, 'content', 'Types generated');
viteServer.watcher.on('add', (entry) => {
contentGenerator.queueEvent({ name: 'add', entry });
});
viteServer.watcher.on('addDir', (entry) =>
contentGenerator.queueEvent({ name: 'addDir', entry })
);
viteServer.watcher.on('change', (entry) =>
contentGenerator.queueEvent({ name: 'change', entry })
);
viteServer.watcher.on('unlink', (entry) => {
contentGenerator.queueEvent({ name: 'unlink', entry });
});
viteServer.watcher.on('unlinkDir', (entry) =>
contentGenerator.queueEvent({ name: 'unlinkDir', entry })
);
}
},
},
{
name: 'astro-content-flag-plugin',
async load(id) {
const { fileId } = getFileInfo(id, settings.config);
if (isContentFlagImport(id)) {
const observable = contentConfigObserver.get();
let contentConfig: ContentConfig | undefined =
observable.status === 'loaded' ? observable.config : undefined;
if (observable.status === 'loading') {
// Wait for config to load
contentConfig = await new Promise((resolve) => {
const unsubscribe = contentConfigObserver.subscribe((ctx) => {
if (ctx.status === 'loaded') {
resolve(ctx.config);
unsubscribe();
} else if (ctx.status === 'error') {
resolve(undefined);
unsubscribe();
}
});
});
}
const rawContents = await fs.promises.readFile(fileId, 'utf-8');
const {
content: body,
data: unparsedData,
matter: rawData = '',
} = parseFrontmatter(rawContents, fileId);
const entryInfo = getEntryInfo({
entry: pathToFileURL(fileId),
contentDir: contentPaths.contentDir,
});
if (entryInfo instanceof Error) return;
const _internal = { filePath: fileId, rawData };
const partialEntry = { data: unparsedData, body, _internal, ...entryInfo };
// TODO: move slug calculation to the start of the build
// to generate a performant lookup map for `getEntryBySlug`
const slug = getEntrySlug(partialEntry);
const collectionConfig = contentConfig?.collections[entryInfo.collection];
const data = collectionConfig
? await getEntryData(partialEntry, collectionConfig)
: unparsedData;
const code = escapeViteEnvReferences(`
export const id = ${JSON.stringify(entryInfo.id)};
export const collection = ${JSON.stringify(entryInfo.collection)};
export const slug = ${JSON.stringify(slug)};
export const body = ${JSON.stringify(body)};
export const data = ${devalue.uneval(data) /* TODO: reuse astro props serializer */};
export const _internal = {
filePath: ${JSON.stringify(fileId)},
rawData: ${JSON.stringify(rawData)},
};
`);
return { code };
}
},
configureServer(viteServer) {
viteServer.watcher.on('all', async (event, entry) => {
if (
['add', 'unlink', 'change'].includes(event) &&
getEntryType(entry, contentPaths) === 'config'
) {
// Content modules depend on config, so we need to invalidate them.
for (const modUrl of viteServer.moduleGraph.urlToModuleMap.keys()) {
if (isContentFlagImport(modUrl)) {
const mod = await viteServer.moduleGraph.getModuleByUrl(modUrl);
if (mod) {
viteServer.moduleGraph.invalidateModule(mod);
}
}
}
}
});
},
async transform(code, id) {
if (isContentFlagImport(id)) {
// Escape before Rollup internal transform.
// Base on MUCH trial-and-error, inspired by MDX integration 2-step transform.
return { code: escapeViteEnvReferences(code) };
}
},
},
];
}
function isContentFlagImport(viteId: string) {
const { pathname, searchParams } = new URL(viteId, 'file://');
return searchParams.has(CONTENT_FLAG) && contentFileExts.some((ext) => pathname.endsWith(ext));
}

View file

@ -80,6 +80,13 @@ class AstroBuilder {
{ settings: this.settings, logging, mode: 'build' }
);
await runHookConfigDone({ settings: this.settings, logging });
const { sync } = await import('../../cli/sync/index.js');
const syncRet = await sync(this.settings, { logging, fs });
if (syncRet !== 0) {
return process.exit(syncRet);
}
return { viteConfig };
}

View file

@ -7,7 +7,7 @@ import * as vite from 'vite';
import { crawlFrameworkPkgs } from 'vitefu';
import {
astroContentAssetPropagationPlugin,
astroContentServerPlugin,
astroContentImportPlugin,
astroContentVirtualModPlugin,
} from '../content/index.js';
import astroPostprocessVitePlugin from '../vite-plugin-astro-postprocess/index.js';
@ -105,7 +105,7 @@ export async function createVite(
astroScannerPlugin({ settings }),
astroInjectEnvTsPlugin({ settings, logging, fs }),
astroContentVirtualModPlugin({ settings }),
astroContentServerPlugin({ fs, settings, logging, mode }),
astroContentImportPlugin({ fs, settings }),
astroContentAssetPropagationPlugin({ mode }),
],
publicDir: fileURLToPath(settings.config.publicDir),

View file

@ -5,6 +5,7 @@ import { performance } from 'perf_hooks';
import * as vite from 'vite';
import yargs from 'yargs-parser';
import type { AstroSettings } from '../../@types/astro';
import { attachContentServerListeners } from '../../content/index.js';
import { info, LogOptions, warn } from '../logger/core.js';
import * as msg from '../messages.js';
import { startContainer } from './container.js';
@ -71,6 +72,8 @@ export default async function dev(
warn(options.logging, null, msg.fsStrictWarning());
}
await attachContentServerListeners(restart.container);
return {
address: devServerAddressInfo,
get watcher() {

View file

@ -5,11 +5,19 @@ import { runInContainer } from '../../../dist/core/dev/index.js';
import { createFsWithFallback, createRequestAndResponse } from '../test-utils.js';
import { isWindows } from '../../test-utils.js';
import mdx from '../../../../integrations/mdx/dist/index.js';
import { attachContentServerListeners } from '../../../dist/content/server-listeners.js';
const root = new URL('../../fixtures/content/', import.meta.url);
const describe = isWindows ? global.describe.skip : global.describe;
async function runInContainerWithContentListeners(params, callback) {
return await runInContainer(params, async (container) => {
await attachContentServerListeners(container);
await callback(container);
});
}
describe('Content Collections - render()', () => {
it('can be called in a page component', async () => {
const fs = createFsWithFallback(
@ -18,10 +26,10 @@ describe('Content Collections - render()', () => {
import { z, defineCollection } from 'astro:content';
const blog = defineCollection({
schema: {
schema: z.object({
title: z.string(),
description: z.string().max(60, 'For SEO purposes, keep descriptions short!'),
},
}),
});
export const collections = { blog };
@ -40,7 +48,7 @@ describe('Content Collections - render()', () => {
root
);
await runInContainer(
await runInContainerWithContentListeners(
{
fs,
root,
@ -71,18 +79,18 @@ describe('Content Collections - render()', () => {
it('can be used in a layout component', async () => {
const fs = createFsWithFallback(
{
'/src/content/config.ts': `
import { z, defineCollection } from 'astro:content';
const blog = defineCollection({
schema: {
title: z.string(),
description: z.string().max(60, 'For SEO purposes, keep descriptions short!'),
},
});
export const collections = { blog };
`,
// Loading the content config with `astro:content` oddly
// causes this test to fail. Spoof a different src/content entry
// to ensure `existsSync` checks pass.
// TODO: revisit after addressing this issue
// https://github.com/withastro/astro/issues/6121
'/src/content/blog/promo/launch-week.mdx': `---
title: Launch Week
description: Astro is launching this week!
---
# Launch Week
- [x] Launch Astro
- [ ] Celebrate`,
'/src/components/Layout.astro': `
---
import { getCollection } from 'astro:content';
@ -113,7 +121,7 @@ describe('Content Collections - render()', () => {
root
);
await runInContainer(
await runInContainerWithContentListeners(
{
fs,
root,
@ -148,10 +156,10 @@ describe('Content Collections - render()', () => {
import { z, defineCollection } from 'astro:content';
const blog = defineCollection({
schema: {
schema: z.object({
title: z.string(),
description: z.string().max(60, 'For SEO purposes, keep descriptions short!'),
},
}),
});
export const collections = { blog };
@ -184,7 +192,7 @@ describe('Content Collections - render()', () => {
root
);
await runInContainer(
await runInContainerWithContentListeners(
{
fs,
root,
@ -219,10 +227,10 @@ describe('Content Collections - render()', () => {
import { z, defineCollection } from 'astro:content';
const blog = defineCollection({
schema: {
schema: z.object({
title: z.string(),
description: z.string().max(60, 'For SEO purposes, keep descriptions short!'),
},
}),
});
export const collections = { blog };
@ -249,7 +257,7 @@ describe('Content Collections - render()', () => {
root
);
await runInContainer(
await runInContainerWithContentListeners(
{
fs,
root,