diff --git a/.changeset/early-eyes-bow.md b/.changeset/early-eyes-bow.md new file mode 100644 index 000000000..a9a268eed --- /dev/null +++ b/.changeset/early-eyes-bow.md @@ -0,0 +1,6 @@ +--- +'astro': minor +'@astrojs/markdoc': minor +--- + +Content collections now support data formats including JSON and YAML. You can also create relationships, or references, between collections to pull information from one collection entry into another. Learn more on our [updated Content Collections docs](https://docs.astro.build/en/guides/content-collections/). diff --git a/package.json b/package.json index 2ce081cdc..f213c733d 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,8 @@ "format:imports": "organize-imports-cli ./packages/*/tsconfig.json ./packages/*/*/tsconfig.json", "test": "turbo run test --concurrency=1 --filter=astro --filter=create-astro --filter=\"@astrojs/*\"", "test:match": "cd packages/astro && pnpm run test:match", + "test:unit": "cd packages/astro && pnpm run test:unit", + "test:unit:match": "cd packages/astro && pnpm run test:unit:match", "test:smoke": "pnpm test:smoke:example && pnpm test:smoke:docs", "test:smoke:example": "turbo run build --concurrency=100% --filter=\"@example/*\"", "test:smoke:docs": "turbo run build --filter=docs", diff --git a/packages/astro/package.json b/packages/astro/package.json index 56d9ed102..1db0656c1 100644 --- a/packages/astro/package.json +++ b/packages/astro/package.json @@ -144,6 +144,7 @@ "github-slugger": "^2.0.0", "gray-matter": "^4.0.3", "html-escaper": "^3.0.3", + "js-yaml": "^4.1.0", "kleur": "^4.1.4", "magic-string": "^0.27.0", "mime": "^3.0.0", @@ -181,6 +182,7 @@ "@types/estree": "^0.0.51", "@types/hast": "^2.3.4", "@types/html-escaper": "^3.0.0", + "@types/js-yaml": "^4.0.5", "@types/mime": "^2.0.3", "@types/mocha": "^9.1.1", "@types/prettier": "^2.6.3", diff --git a/packages/astro/src/@types/astro.ts b/packages/astro/src/@types/astro.ts index 70ac9046c..e7ade1c6d 100644 --- a/packages/astro/src/@types/astro.ts +++ b/packages/astro/src/@types/astro.ts @@ -1250,12 +1250,22 @@ export type ContentEntryModule = { }; }; +export type DataEntryModule = { + id: string; + collection: string; + data: Record; + _internal: { + rawData: string; + filePath: string; + }; +}; + export interface ContentEntryType { extensions: string[]; getEntryInfo(params: { fileUrl: URL; contents: string; - }): GetEntryInfoReturnType | Promise; + }): GetContentEntryInfoReturnType | Promise; getRenderModule?( this: rollup.PluginContext, params: { @@ -1266,7 +1276,7 @@ export interface ContentEntryType { contentModuleTypes?: string; } -type GetEntryInfoReturnType = { +type GetContentEntryInfoReturnType = { data: Record; /** * Used for error hints to point to correct line and location @@ -1278,12 +1288,23 @@ type GetEntryInfoReturnType = { slug: string; }; +export interface DataEntryType { + extensions: string[]; + getEntryInfo(params: { + fileUrl: URL; + contents: string; + }): GetDataEntryInfoReturnType | Promise; +} + +export type GetDataEntryInfoReturnType = { data: Record; rawData?: string }; + export interface AstroSettings { config: AstroConfig; adapter: AstroAdapter | undefined; injectedRoutes: InjectedRoute[]; pageExtensions: string[]; contentEntryTypes: ContentEntryType[]; + dataEntryTypes: DataEntryType[]; renderers: AstroRenderer[]; scripts: { stage: InjectedScriptStage; diff --git a/packages/astro/src/content/consts.ts b/packages/astro/src/content/consts.ts index 1f0470d5a..bda154692 100644 --- a/packages/astro/src/content/consts.ts +++ b/packages/astro/src/content/consts.ts @@ -1,5 +1,8 @@ export const PROPAGATED_ASSET_FLAG = 'astroPropagatedAssets'; -export const CONTENT_FLAG = 'astroContent'; +export const CONTENT_FLAG = 'astroContentCollectionEntry'; +export const DATA_FLAG = 'astroDataCollectionEntry'; +export const CONTENT_FLAGS = [CONTENT_FLAG, DATA_FLAG, PROPAGATED_ASSET_FLAG] as const; + export const VIRTUAL_MODULE_ID = 'astro:content'; export const LINKS_PLACEHOLDER = '@@ASTRO-LINKS@@'; export const STYLES_PLACEHOLDER = '@@ASTRO-STYLES@@'; diff --git a/packages/astro/src/content/runtime-assets.ts b/packages/astro/src/content/runtime-assets.ts index a2421516a..b24dac8b4 100644 --- a/packages/astro/src/content/runtime-assets.ts +++ b/packages/astro/src/content/runtime-assets.ts @@ -4,7 +4,7 @@ import type { AstroSettings } from '../@types/astro.js'; import { emitESMImage } from '../assets/index.js'; export function createImage( - settings: AstroSettings, + settings: Pick, pluginContext: PluginContext, entryFilePath: string ) { diff --git a/packages/astro/src/content/runtime.ts b/packages/astro/src/content/runtime.ts index 35afc71e8..c3c39da87 100644 --- a/packages/astro/src/content/runtime.ts +++ b/packages/astro/src/content/runtime.ts @@ -1,6 +1,6 @@ import { AstroError, AstroErrorData } from '../core/errors/index.js'; import { prependForwardSlash } from '../core/path.js'; - +import { ZodIssueCode, string as zodString, type z } from 'zod'; import { createComponent, createHeadAndContent, @@ -9,7 +9,10 @@ import { renderTemplate, renderUniqueStylesheet, unescapeHTML, + type AstroComponentFactory, } from '../runtime/server/index.js'; +import type { ContentLookupMap } from './utils.js'; +import type { MarkdownHeading } from '@astrojs/markdown-remark'; type LazyImport = () => Promise; type GlobResult = Record; @@ -37,14 +40,31 @@ export function createCollectionToGlobResultMap({ const cacheEntriesByCollection = new Map(); export function createGetCollection({ - collectionToEntryMap, + contentCollectionToEntryMap, + dataCollectionToEntryMap, getRenderEntryImport, }: { - collectionToEntryMap: CollectionToEntryMap; + contentCollectionToEntryMap: CollectionToEntryMap; + dataCollectionToEntryMap: CollectionToEntryMap; getRenderEntryImport: GetEntryImport; }) { return async function getCollection(collection: string, filter?: (entry: any) => unknown) { - const lazyImports = Object.values(collectionToEntryMap[collection] ?? {}); + let type: 'content' | 'data'; + if (collection in contentCollectionToEntryMap) { + type = 'content'; + } else if (collection in dataCollectionToEntryMap) { + type = 'data'; + } else { + throw new AstroError({ + ...AstroErrorData.CollectionDoesNotExistError, + message: AstroErrorData.CollectionDoesNotExistError.message(collection), + }); + } + const lazyImports = Object.values( + type === 'content' + ? contentCollectionToEntryMap[collection] + : dataCollectionToEntryMap[collection] + ); let entries: any[] = []; // Cache `getCollection()` calls in production only // prevents stale cache in development @@ -54,20 +74,26 @@ export function createGetCollection({ entries = await Promise.all( lazyImports.map(async (lazyImport) => { const entry = await lazyImport(); - return { - id: entry.id, - slug: entry.slug, - body: entry.body, - collection: entry.collection, - data: entry.data, - async render() { - return render({ - collection: entry.collection, + return type === 'content' + ? { id: entry.id, - renderEntryImport: await getRenderEntryImport(collection, entry.slug), - }); - }, - }; + slug: entry.slug, + body: entry.body, + collection: entry.collection, + data: entry.data, + async render() { + return render({ + collection: entry.collection, + id: entry.id, + renderEntryImport: await getRenderEntryImport(collection, entry.slug), + }); + }, + } + : { + id: entry.id, + collection: entry.collection, + data: entry.data, + }; }) ); cacheEntriesByCollection.set(collection, entries); @@ -110,6 +136,121 @@ export function createGetEntryBySlug({ }; } +export function createGetDataEntryById({ + dataCollectionToEntryMap, +}: { + dataCollectionToEntryMap: CollectionToEntryMap; +}) { + return async function getDataEntryById(collection: string, id: string) { + const lazyImport = + dataCollectionToEntryMap[collection]?.[/*TODO: filePathToIdMap*/ id + '.json']; + + // TODO: AstroError + if (!lazyImport) throw new Error(`Entry ${collection} → ${id} was not found.`); + const entry = await lazyImport(); + + return { + id: entry.id, + collection: entry.collection, + data: entry.data, + }; + }; +} + +type ContentEntryResult = { + id: string; + slug: string; + body: string; + collection: string; + data: Record; + render(): Promise; +}; + +type DataEntryResult = { + id: string; + collection: string; + data: Record; +}; + +type EntryLookupObject = { collection: string; id: string } | { collection: string; slug: string }; + +export function createGetEntry({ + getEntryImport, + getRenderEntryImport, +}: { + getEntryImport: GetEntryImport; + getRenderEntryImport: GetEntryImport; +}) { + return async function getEntry( + // Can either pass collection and identifier as 2 positional args, + // Or pass a single object with the collection and identifier as properties. + // This means the first positional arg can have different shapes. + collectionOrLookupObject: string | EntryLookupObject, + _lookupId?: string + ): Promise { + let collection: string, lookupId: string; + if (typeof collectionOrLookupObject === 'string') { + collection = collectionOrLookupObject; + if (!_lookupId) + throw new AstroError({ + ...AstroErrorData.UnknownContentCollectionError, + message: '`getEntry()` requires an entry identifier as the second argument.', + }); + lookupId = _lookupId; + } else { + collection = collectionOrLookupObject.collection; + // Identifier could be `slug` for content entries, or `id` for data entries + lookupId = + 'id' in collectionOrLookupObject + ? collectionOrLookupObject.id + : collectionOrLookupObject.slug; + } + + const entryImport = await getEntryImport(collection, lookupId); + if (typeof entryImport !== 'function') return undefined; + + const entry = await entryImport(); + + if (entry._internal.type === 'content') { + return { + id: entry.id, + slug: entry.slug, + body: entry.body, + collection: entry.collection, + data: entry.data, + async render() { + return render({ + collection: entry.collection, + id: entry.id, + renderEntryImport: await getRenderEntryImport(collection, lookupId), + }); + }, + }; + } else if (entry._internal.type === 'data') { + return { + id: entry.id, + collection: entry.collection, + data: entry.data, + }; + } + return undefined; + }; +} + +export function createGetEntries(getEntry: ReturnType) { + return async function getEntries( + entries: { collection: string; id: string }[] | { collection: string; slug: string }[] + ) { + return Promise.all(entries.map((e) => getEntry(e))); + }; +} + +type RenderResult = { + Content: AstroComponentFactory; + headings: MarkdownHeading[]; + remarkPluginFrontmatter: Record; +}; + async function render({ collection, id, @@ -118,7 +259,7 @@ async function render({ collection: string; id: string; renderEntryImport?: LazyImport; -}) { +}): Promise { const UnexpectedRenderError = new AstroError({ ...AstroErrorData.UnknownContentCollectionError, message: `Unexpected error while rendering ${String(collection)} → ${String(id)}.`, @@ -186,3 +327,38 @@ async function render({ remarkPluginFrontmatter: mod.frontmatter ?? {}, }; } + +export function createReference({ lookupMap }: { lookupMap: ContentLookupMap }) { + return function reference(collection: string) { + return zodString().transform((lookupId: string, ctx) => { + const flattenedErrorPath = ctx.path.join('.'); + if (!lookupMap[collection]) { + ctx.addIssue({ + code: ZodIssueCode.custom, + message: `**${flattenedErrorPath}:** Reference to ${collection} invalid. Collection does not exist or is empty.`, + }); + return; + } + + const { type, entries } = lookupMap[collection]; + const entry = entries[lookupId]; + + if (!entry) { + ctx.addIssue({ + code: ZodIssueCode.custom, + message: `**${flattenedErrorPath}**: Reference to ${collection} invalid. Expected ${Object.keys( + entries + ) + .map((c) => JSON.stringify(c)) + .join(' | ')}. Received ${JSON.stringify(lookupId)}.`, + }); + return; + } + // Content is still identified by slugs, so map to a `slug` key for consistency. + if (type === 'content') { + return { slug: lookupId, collection }; + } + return { id: lookupId, collection }; + }); + }; +} diff --git a/packages/astro/src/content/template/types.d.ts b/packages/astro/src/content/template/types.d.ts index 7bb36c0f8..277d00acf 100644 --- a/packages/astro/src/content/template/types.d.ts +++ b/packages/astro/src/content/template/types.d.ts @@ -10,8 +10,7 @@ declare module 'astro:content' { declare module 'astro:content' { export { z } from 'astro/zod'; - export type CollectionEntry = - (typeof entryMap)[C][keyof (typeof entryMap)[C]]; + export type CollectionEntry = AnyEntryMap[C][keyof AnyEntryMap[C]]; // TODO: Remove this when having this fallback is no longer relevant. 2.3? 3.0? - erika, 2023-04-04 /** @@ -65,44 +64,138 @@ declare module 'astro:content' { export type SchemaContext = { image: ImageFunction }; - type BaseCollectionConfig = { + type DataCollectionConfig = { + type: 'data'; schema?: S | ((context: SchemaContext) => S); }; - export function defineCollection( - input: BaseCollectionConfig - ): BaseCollectionConfig; - type EntryMapKeys = keyof typeof entryMap; + type ContentCollectionConfig = { + type?: 'content'; + schema?: S | ((context: SchemaContext) => S); + }; + + type CollectionConfig = ContentCollectionConfig | DataCollectionConfig; + + export function defineCollection( + input: CollectionConfig + ): CollectionConfig; + type AllValuesOf = T extends any ? T[keyof T] : never; - type ValidEntrySlug = AllValuesOf<(typeof entryMap)[C]>['slug']; + type ValidContentEntrySlug = AllValuesOf< + ContentEntryMap[C] + >['slug']; export function getEntryBySlug< - C extends keyof typeof entryMap, - E extends ValidEntrySlug | (string & {}) + C extends keyof ContentEntryMap, + E extends ValidContentEntrySlug | (string & {}) >( collection: C, // Note that this has to accept a regular string too, for SSR entrySlug: E - ): E extends ValidEntrySlug + ): E extends ValidContentEntrySlug ? Promise> : Promise | undefined>; - export function getCollection>( + + export function getDataEntryById( + collection: C, + entryId: E + ): Promise>; + + export function getCollection>( collection: C, filter?: (entry: CollectionEntry) => entry is E ): Promise; - export function getCollection( + export function getCollection( collection: C, filter?: (entry: CollectionEntry) => unknown ): Promise[]>; + export function getEntry< + C extends keyof ContentEntryMap, + E extends ValidContentEntrySlug | (string & {}) + >(entry: { + collection: C; + slug: E; + }): E extends ValidContentEntrySlug + ? Promise> + : Promise | undefined>; + export function getEntry< + C extends keyof DataEntryMap, + E extends keyof DataEntryMap[C] | (string & {}) + >(entry: { + collection: C; + id: E; + }): E extends keyof DataEntryMap[C] + ? Promise + : Promise | undefined>; + export function getEntry< + C extends keyof ContentEntryMap, + E extends ValidContentEntrySlug | (string & {}) + >( + collection: C, + slug: E + ): E extends ValidContentEntrySlug + ? Promise> + : Promise | undefined>; + export function getEntry< + C extends keyof DataEntryMap, + E extends keyof DataEntryMap[C] | (string & {}) + >( + collection: C, + id: E + ): E extends keyof DataEntryMap[C] + ? Promise + : Promise | undefined>; + + /** Resolve an array of entry references from the same collection */ + export function getEntries( + entries: { + collection: C; + slug: ValidContentEntrySlug; + }[] + ): Promise[]>; + export function getEntries( + entries: { + collection: C; + id: keyof DataEntryMap[C]; + }[] + ): Promise[]>; + + export function reference( + collection: C + ): import('astro/zod').ZodEffects< + import('astro/zod').ZodString, + C extends keyof ContentEntryMap + ? { + collection: C; + slug: ValidContentEntrySlug; + } + : { + collection: C; + id: keyof DataEntryMap[C]; + } + >; + // Allow generic `string` to avoid excessive type errors in the config + // if `dev` is not running to update as you edit. + // Invalid collection names will be caught at build time. + export function reference( + collection: C + ): import('astro/zod').ZodEffects; + type ReturnTypeOrOriginal = T extends (...args: any[]) => infer R ? R : T; - type InferEntrySchema = import('astro/zod').infer< + type InferEntrySchema = import('astro/zod').infer< ReturnTypeOrOriginal['schema']> >; - const entryMap: { - // @@ENTRY_MAP@@ + type ContentEntryMap = { + // @@CONTENT_ENTRY_MAP@@ }; + type DataEntryMap = { + // @@DATA_ENTRY_MAP@@ + }; + + type AnyEntryMap = ContentEntryMap & DataEntryMap; + type ContentConfig = '@@CONTENT_CONFIG_TYPE@@'; } diff --git a/packages/astro/src/content/template/virtual-mod.mjs b/packages/astro/src/content/template/virtual-mod.mjs index a649804ce..aab50bf6b 100644 --- a/packages/astro/src/content/template/virtual-mod.mjs +++ b/packages/astro/src/content/template/virtual-mod.mjs @@ -3,28 +3,33 @@ import { createCollectionToGlobResultMap, createGetCollection, createGetEntryBySlug, + createGetEntry, + createGetEntries, + createGetDataEntryById, + createReference, } from 'astro/content/runtime'; export { z } from 'astro/zod'; -export function defineCollection(config) { - return config; -} - -// TODO: Remove this when having this fallback is no longer relevant. 2.3? 3.0? - erika, 2023-04-04 -export const image = () => { - throw new Error( - 'Importing `image()` from `astro:content` is no longer supported. See https://docs.astro.build/en/guides/assets/#update-content-collections-schemas for our new import instructions.' - ); -}; - const contentDir = '@@CONTENT_DIR@@'; -const entryGlob = import.meta.glob('@@ENTRY_GLOB_PATH@@', { - query: { astroContent: true }, +const contentEntryGlob = import.meta.glob('@@CONTENT_ENTRY_GLOB_PATH@@', { + query: { astroContentCollectionEntry: true }, +}); +const contentCollectionToEntryMap = createCollectionToGlobResultMap({ + globResult: contentEntryGlob, + contentDir, +}); + +const dataEntryGlob = import.meta.glob('@@DATA_ENTRY_GLOB_PATH@@', { + query: { astroDataCollectionEntry: true }, +}); +const dataCollectionToEntryMap = createCollectionToGlobResultMap({ + globResult: dataEntryGlob, + contentDir, }); const collectionToEntryMap = createCollectionToGlobResultMap({ - globResult: entryGlob, + globResult: { ...contentEntryGlob, ...dataEntryGlob }, contentDir, }); @@ -33,7 +38,7 @@ let lookupMap = {}; function createGlobLookup(glob) { return async (collection, lookupId) => { - const filePath = lookupMap[collection]?.[lookupId]; + const filePath = lookupMap[collection]?.entries[lookupId]; if (!filePath) return undefined; return glob[collection][filePath]; @@ -48,12 +53,31 @@ const collectionToRenderEntryMap = createCollectionToGlobResultMap({ contentDir, }); +export function defineCollection(config) { + if (!config.type) config.type = 'content'; + return config; +} + export const getCollection = createGetCollection({ - collectionToEntryMap, + contentCollectionToEntryMap, + dataCollectionToEntryMap, getRenderEntryImport: createGlobLookup(collectionToRenderEntryMap), }); export const getEntryBySlug = createGetEntryBySlug({ + getEntryImport: createGlobLookup(contentCollectionToEntryMap), + getRenderEntryImport: createGlobLookup(collectionToRenderEntryMap), +}); + +export const getDataEntryById = createGetDataEntryById({ + dataCollectionToEntryMap, +}); + +export const getEntry = createGetEntry({ getEntryImport: createGlobLookup(collectionToEntryMap), getRenderEntryImport: createGlobLookup(collectionToRenderEntryMap), }); + +export const getEntries = createGetEntries(getEntry); + +export const reference = createReference({ lookupMap }); diff --git a/packages/astro/src/content/types-generator.ts b/packages/astro/src/content/types-generator.ts index 86e47c1e3..0b93a4e93 100644 --- a/packages/astro/src/content/types-generator.ts +++ b/packages/astro/src/content/types-generator.ts @@ -5,29 +5,47 @@ import * as path from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; import { normalizePath, type ViteDevServer } from 'vite'; import type { AstroSettings, ContentEntryType } from '../@types/astro.js'; -import { AstroError, AstroErrorData } from '../core/errors/index.js'; import { info, warn, type LogOptions } from '../core/logger/core.js'; import { isRelativePath } from '../core/path.js'; import { CONTENT_TYPES_FILE, VIRTUAL_MODULE_ID } from './consts.js'; import { - getContentEntryConfigByExtMap, - getContentPaths, - getEntryInfo, - getEntrySlug, - getEntryType, - loadContentConfig, - NoCollectionError, type ContentConfig, type ContentObservable, type ContentPaths, + getContentEntryConfigByExtMap, + getContentPaths, + getEntryType, + getContentEntryIdAndSlug, + getEntrySlug, + getEntryCollectionName, + getDataEntryExts, + getDataEntryId, + reloadContentConfigObserver, } from './utils.js'; +import { AstroError } from '../core/errors/errors.js'; +import { AstroErrorData } from '../core/errors/errors-data.js'; type ChokidarEvent = 'add' | 'addDir' | 'change' | 'unlink' | 'unlinkDir'; type RawContentEvent = { name: ChokidarEvent; entry: string }; type ContentEvent = { name: ChokidarEvent; entry: URL }; -type ContentTypesEntryMetadata = { slug: string }; -type ContentTypes = Record>; +type DataEntryMetadata = Record; +type ContentEntryMetadata = { slug: string }; +type CollectionEntryMap = { + [collection: string]: + | { + type: 'unknown'; + entries: Record; + } + | { + type: 'content'; + entries: Record; + } + | { + type: 'data'; + entries: Record; + }; +}; type CreateContentGeneratorParams = { contentConfigObserver: ContentObservable; @@ -54,10 +72,11 @@ export async function createContentTypesGenerator({ settings, viteServer, }: CreateContentGeneratorParams) { - const contentTypes: ContentTypes = {}; + const collectionEntryMap: CollectionEntryMap = {}; const contentPaths = getContentPaths(settings.config, fs); const contentEntryConfigByExt = getContentEntryConfigByExtMap(settings); const contentEntryExts = [...contentEntryConfigByExt.keys()]; + const dataEntryExts = getDataEntryExts(settings); let events: EventWithOptions[] = []; let debounceTimeout: NodeJS.Timeout | undefined; @@ -106,18 +125,22 @@ export async function createContentTypesGenerator({ const collection = normalizePath( path.relative(fileURLToPath(contentPaths.contentDir), fileURLToPath(event.entry)) ); + const collectionKey = JSON.stringify(collection); // If directory is multiple levels deep, it is not a collection. Ignore event. const isCollectionEvent = collection.split('/').length === 1; if (!isCollectionEvent) return { shouldGenerateTypes: false }; + switch (event.name) { case 'addDir': - addCollection(contentTypes, JSON.stringify(collection)); + collectionEntryMap[JSON.stringify(collection)] = { type: 'unknown', entries: {} }; if (logLevel === 'info') { info(logging, 'content', `${cyan(collection)} collection added`); } break; case 'unlinkDir': - removeCollection(contentTypes, JSON.stringify(collection)); + if (collectionKey in collectionEntryMap) { + delete collectionEntryMap[JSON.stringify(collection)]; + } break; } return { shouldGenerateTypes: true }; @@ -126,28 +149,14 @@ export async function createContentTypesGenerator({ fileURLToPath(event.entry), contentPaths, contentEntryExts, + dataEntryExts, settings.config.experimental.assets ); if (fileType === 'ignored') { return { shouldGenerateTypes: false }; } if (fileType === 'config') { - contentConfigObserver.set({ status: 'loading' }); - try { - const config = await loadContentConfig({ fs, settings, viteServer }); - if (config) { - contentConfigObserver.set({ status: 'loaded', config }); - } else { - contentConfigObserver.set({ status: 'does-not-exist' }); - } - } catch (e) { - contentConfigObserver.set({ - status: 'error', - error: - e instanceof Error ? e : new AstroError(AstroErrorData.UnknownContentCollectionError), - }); - } - + await reloadContentConfigObserver({ fs, settings, viteServer }); return { shouldGenerateTypes: true }; } if (fileType === 'unsupported') { @@ -155,22 +164,22 @@ export async function createContentTypesGenerator({ if (event.name === 'unlink') { return { shouldGenerateTypes: false }; } - const entryInfo = getEntryInfo({ + const { id } = getContentEntryIdAndSlug({ entry: event.entry, contentDir: contentPaths.contentDir, - // Skip invalid file check. We already know it’s invalid. - allowFilesOutsideCollection: true, + collection: '', }); return { shouldGenerateTypes: false, - error: new UnsupportedFileTypeError(entryInfo.id), + error: new UnsupportedFileTypeError(id), }; } - const entryInfo = getEntryInfo({ - entry: event.entry, - contentDir: contentPaths.contentDir, - }); - if (entryInfo instanceof NoCollectionError) { + + const { entry } = event; + const { contentDir } = contentPaths; + + const collection = getEntryCollectionName({ entry, contentDir }); + if (collection === undefined) { if (['info', 'warn'].includes(logLevel)) { warn( logging, @@ -185,11 +194,68 @@ export async function createContentTypesGenerator({ return { shouldGenerateTypes: false }; } - const { id, collection, slug: generatedSlug } = entryInfo; + if (fileType === 'data') { + const id = getDataEntryId({ entry, contentDir, collection }); + const collectionKey = JSON.stringify(collection); + const entryKey = JSON.stringify(id); + + switch (event.name) { + case 'add': + if (!(collectionKey in collectionEntryMap)) { + collectionEntryMap[collectionKey] = { type: 'data', entries: {} }; + } + const collectionInfo = collectionEntryMap[collectionKey]; + if (collectionInfo.type === 'content') { + viteServer.ws.send({ + type: 'error', + err: new AstroError({ + ...AstroErrorData.MixedContentDataCollectionError, + message: AstroErrorData.MixedContentDataCollectionError.message(collectionKey), + location: { file: entry.pathname }, + }) as any, + }); + return { shouldGenerateTypes: false }; + } + if (!(entryKey in collectionEntryMap[collectionKey])) { + collectionEntryMap[collectionKey] = { + type: 'data', + entries: { ...collectionInfo.entries, [entryKey]: {} }, + }; + } + return { shouldGenerateTypes: true }; + case 'unlink': + if ( + collectionKey in collectionEntryMap && + entryKey in collectionEntryMap[collectionKey].entries + ) { + delete collectionEntryMap[collectionKey].entries[entryKey]; + } + return { shouldGenerateTypes: true }; + case 'change': + return { shouldGenerateTypes: false }; + } + } + const contentEntryType = contentEntryConfigByExt.get(path.extname(event.entry.pathname)); if (!contentEntryType) return { shouldGenerateTypes: false }; + const { id, slug: generatedSlug } = getContentEntryIdAndSlug({ entry, contentDir, collection }); const collectionKey = JSON.stringify(collection); + if (!(collectionKey in collectionEntryMap)) { + collectionEntryMap[collectionKey] = { type: 'content', entries: {} }; + } + const collectionInfo = collectionEntryMap[collectionKey]; + if (collectionInfo.type === 'data') { + viteServer.ws.send({ + type: 'error', + err: new AstroError({ + ...AstroErrorData.MixedContentDataCollectionError, + message: AstroErrorData.MixedContentDataCollectionError.message(collectionKey), + location: { file: entry.pathname }, + }) as any, + }); + return { shouldGenerateTypes: false }; + } const entryKey = JSON.stringify(id); switch (event.name) { @@ -202,16 +268,19 @@ export async function createContentTypesGenerator({ contentEntryType, fs, }); - if (!(collectionKey in contentTypes)) { - addCollection(contentTypes, collectionKey); - } - if (!(entryKey in contentTypes[collectionKey])) { - setEntry(contentTypes, collectionKey, entryKey, addedSlug); + if (!(entryKey in collectionEntryMap[collectionKey].entries)) { + collectionEntryMap[collectionKey] = { + type: 'content', + entries: { ...collectionInfo.entries, [entryKey]: { slug: addedSlug } }, + }; } return { shouldGenerateTypes: true }; case 'unlink': - if (collectionKey in contentTypes && entryKey in contentTypes[collectionKey]) { - removeEntry(contentTypes, collectionKey, entryKey); + if ( + collectionKey in collectionEntryMap && + entryKey in collectionEntryMap[collectionKey].entries + ) { + delete collectionEntryMap[collectionKey].entries[entryKey]; } return { shouldGenerateTypes: true }; case 'change': @@ -225,8 +294,9 @@ export async function createContentTypesGenerator({ contentEntryType, fs, }); - if (contentTypes[collectionKey]?.[entryKey]?.slug !== changedSlug) { - setEntry(contentTypes, collectionKey, entryKey, changedSlug); + const entryMetadata = collectionInfo.entries[entryKey]; + if (entryMetadata?.slug !== changedSlug) { + collectionInfo.entries[entryKey].slug = changedSlug; return { shouldGenerateTypes: true }; } return { shouldGenerateTypes: false }; @@ -287,18 +357,19 @@ export async function createContentTypesGenerator({ if (eventResponses.some((r) => r.shouldGenerateTypes)) { await writeContentFiles({ fs, - contentTypes, + collectionEntryMap, contentPaths, typeTemplateContent, contentConfig: observable.status === 'loaded' ? observable.config : undefined, contentEntryTypes: settings.contentEntryTypes, + viteServer, }); invalidateVirtualMod(viteServer); if (observable.status === 'loaded' && ['info', 'warn'].includes(logLevel)) { warnNonexistentCollections({ logging, contentConfig: observable.config, - contentTypes, + collectionEntryMap, }); } } @@ -315,58 +386,74 @@ function invalidateVirtualMod(viteServer: ViteDevServer) { viteServer.moduleGraph.invalidateModule(virtualMod); } -function addCollection(contentMap: ContentTypes, collectionKey: string) { - contentMap[collectionKey] = {}; -} - -function removeCollection(contentMap: ContentTypes, collectionKey: string) { - delete contentMap[collectionKey]; -} - -function setEntry( - contentTypes: ContentTypes, - collectionKey: string, - entryKey: string, - slug: string -) { - contentTypes[collectionKey][entryKey] = { slug }; -} - -function removeEntry(contentTypes: ContentTypes, collectionKey: string, entryKey: string) { - delete contentTypes[collectionKey][entryKey]; -} - async function writeContentFiles({ fs, contentPaths, - contentTypes, + collectionEntryMap, typeTemplateContent, contentEntryTypes, contentConfig, + viteServer, }: { fs: typeof fsMod; contentPaths: ContentPaths; - contentTypes: ContentTypes; + collectionEntryMap: CollectionEntryMap; typeTemplateContent: string; - contentEntryTypes: ContentEntryType[]; + contentEntryTypes: Pick[]; contentConfig?: ContentConfig; + viteServer: Pick; }) { let contentTypesStr = ''; - const collectionKeys = Object.keys(contentTypes).sort(); - for (const collectionKey of collectionKeys) { + let dataTypesStr = ''; + for (const collectionKey of Object.keys(collectionEntryMap).sort()) { const collectionConfig = contentConfig?.collections[JSON.parse(collectionKey)]; - contentTypesStr += `${collectionKey}: {\n`; - const entryKeys = Object.keys(contentTypes[collectionKey]).sort(); - for (const entryKey of entryKeys) { - const entryMetadata = contentTypes[collectionKey][entryKey]; - const dataType = collectionConfig?.schema ? `InferEntrySchema<${collectionKey}>` : 'any'; - const renderType = `{ render(): Render[${JSON.stringify( - path.extname(JSON.parse(entryKey)) - )}] }`; - const slugType = JSON.stringify(entryMetadata.slug); - contentTypesStr += `${entryKey}: {\n id: ${entryKey},\n slug: ${slugType},\n body: string,\n collection: ${collectionKey},\n data: ${dataType}\n} & ${renderType},\n`; + const collection = collectionEntryMap[collectionKey]; + if (collectionConfig?.type && collection.type !== collectionConfig.type) { + viteServer.ws.send({ + type: 'error', + err: new AstroError({ + ...AstroErrorData.ContentCollectionTypeMismatchError, + message: AstroErrorData.ContentCollectionTypeMismatchError.message( + collectionKey, + collection.type, + collectionConfig.type + ), + hint: + collection.type === 'data' + ? "Try adding `type: 'data'` to your collection config." + : undefined, + location: { file: '' /** required for error overlay `ws` messages */ }, + }) as any, + }); + return; + } + switch (collection.type) { + case 'content': + contentTypesStr += `${collectionKey}: {\n`; + for (const entryKey of Object.keys(collection.entries).sort()) { + const entryMetadata = collection.entries[entryKey]; + const dataType = collectionConfig?.schema ? `InferEntrySchema<${collectionKey}>` : 'any'; + const renderType = `{ render(): Render[${JSON.stringify( + path.extname(JSON.parse(entryKey)) + )}] }`; + + const slugType = JSON.stringify(entryMetadata.slug); + contentTypesStr += `${entryKey}: {\n id: ${entryKey};\n slug: ${slugType};\n body: string;\n collection: ${collectionKey};\n data: ${dataType}\n} & ${renderType};\n`; + } + contentTypesStr += `};\n`; + break; + case 'data': + // Add empty / unknown collections to the data type map by default + // This ensures `getCollection('empty-collection')` doesn't raise a type error + case 'unknown': + dataTypesStr += `${collectionKey}: {\n`; + for (const entryKey of Object.keys(collection.entries).sort()) { + const dataType = collectionConfig?.schema ? `InferEntrySchema<${collectionKey}>` : 'any'; + dataTypesStr += `${entryKey}: {\n id: ${entryKey};\n collection: ${collectionKey};\n data: ${dataType}\n};\n`; + } + dataTypesStr += `};\n`; + break; } - contentTypesStr += `},\n`; } if (!fs.existsSync(contentPaths.cacheDir)) { @@ -389,7 +476,8 @@ async function writeContentFiles({ typeTemplateContent = contentEntryType.contentModuleTypes + '\n' + typeTemplateContent; } } - typeTemplateContent = typeTemplateContent.replace('// @@ENTRY_MAP@@', contentTypesStr); + typeTemplateContent = typeTemplateContent.replace('// @@CONTENT_ENTRY_MAP@@', contentTypesStr); + typeTemplateContent = typeTemplateContent.replace('// @@DATA_ENTRY_MAP@@', dataTypesStr); typeTemplateContent = typeTemplateContent.replace( "'@@CONTENT_CONFIG_TYPE@@'", contentConfig ? `typeof import(${JSON.stringify(configPathRelativeToCacheDir)})` : 'never' @@ -403,15 +491,15 @@ async function writeContentFiles({ function warnNonexistentCollections({ contentConfig, - contentTypes, + collectionEntryMap, logging, }: { contentConfig: ContentConfig; - contentTypes: ContentTypes; + collectionEntryMap: CollectionEntryMap; logging: LogOptions; }) { for (const configuredCollection in contentConfig.collections) { - if (!contentTypes[JSON.stringify(configuredCollection)]) { + if (!collectionEntryMap[JSON.stringify(configuredCollection)]) { warn( logging, 'content', diff --git a/packages/astro/src/content/utils.ts b/packages/astro/src/content/utils.ts index d161b93de..7252ca931 100644 --- a/packages/astro/src/content/utils.ts +++ b/packages/astro/src/content/utils.ts @@ -14,13 +14,30 @@ import type { } from '../@types/astro.js'; import { VALID_INPUT_FORMATS } from '../assets/consts.js'; import { AstroError, AstroErrorData } from '../core/errors/index.js'; -import { CONTENT_TYPES_FILE } from './consts.js'; +import { CONTENT_TYPES_FILE, CONTENT_FLAGS } from './consts.js'; import { errorMap } from './error-map.js'; import { createImage } from './runtime-assets.js'; +import { formatYAMLException, isYAMLException } from '../core/errors/utils.js'; -export const collectionConfigParser = z.object({ - schema: z.any().optional(), -}); +/** + * Amap from a collection + slug to the local file path. + * This is used internally to resolve entry imports when using `getEntry()`. + * @see `src/content/virtual-mod.mjs` + */ +export type ContentLookupMap = { + [collectionName: string]: { type: 'content' | 'data'; entries: { [lookupId: string]: string } }; +}; + +export const collectionConfigParser = z.union([ + z.object({ + type: z.literal('content').optional().default('content'), + schema: z.any().optional(), + }), + z.object({ + type: z.literal('data'), + schema: z.any().optional(), + }), +]); export function getDotAstroTypeReference({ root, srcDir }: { root: URL; srcDir: URL }) { const { cacheDir } = getContentPaths({ root, srcDir }); @@ -39,11 +56,6 @@ export type CollectionConfig = z.infer; export type ContentConfig = z.infer; type EntryInternal = { rawData: string | undefined; filePath: string }; -export type EntryInfo = { - id: string; - slug: string; - collection: string; -}; export const msg = { collectionConfigMissing: (collection: string) => @@ -72,31 +84,46 @@ export function parseEntrySlug({ } export async function getEntryData( - entry: EntryInfo & { unvalidatedData: Record; _internal: EntryInternal }, + entry: { + id: string; + collection: string; + unvalidatedData: Record; + _internal: EntryInternal; + }, collectionConfig: CollectionConfig, pluginContext: PluginContext, - settings: AstroSettings + config: AstroConfig ) { - // Remove reserved `slug` field before parsing data - let { slug, ...data } = entry.unvalidatedData; + let data; + if (collectionConfig.type === 'data') { + data = entry.unvalidatedData; + } else { + const { slug, ...unvalidatedData } = entry.unvalidatedData; + data = unvalidatedData; + } let schema = collectionConfig.schema; if (typeof schema === 'function') { - if (!settings.config.experimental.assets) { + if (!config.experimental.assets) { throw new Error( 'The function shape for schema can only be used when `experimental.assets` is enabled.' ); } schema = schema({ - image: createImage(settings, pluginContext, entry._internal.filePath), + image: createImage({ config }, pluginContext, entry._internal.filePath), }); } if (schema) { - // Catch reserved `slug` field inside schema + // Catch reserved `slug` field inside content schemas // Note: will not warn for `z.union` or `z.intersection` schemas - if (typeof schema === 'object' && 'shape' in schema && schema.shape.slug) { + if ( + collectionConfig.type === 'content' && + typeof schema === 'object' && + 'shape' in schema && + schema.shape.slug + ) { throw new AstroError({ ...AstroErrorData.ContentSchemaContainsSlugError, message: AstroErrorData.ContentSchemaContainsSlugError.message(entry.collection), @@ -104,28 +131,33 @@ export async function getEntryData( } // Use `safeParseAsync` to allow async transforms - const parsed = await schema.safeParseAsync(entry.unvalidatedData, { - errorMap, + let formattedError; + const parsed = await (schema as z.ZodSchema).safeParseAsync(entry.unvalidatedData, { + errorMap(error, ctx) { + if (error.code === 'custom' && error.params?.isHoistedAstroError) { + formattedError = error.params?.astroError; + } + return errorMap(error, ctx); + }, }); if (parsed.success) { data = parsed.data; } else { - const formattedError = new AstroError({ - ...AstroErrorData.InvalidContentEntryFrontmatterError, - message: AstroErrorData.InvalidContentEntryFrontmatterError.message( - entry.collection, - entry.id, - parsed.error - ), - location: { - file: entry._internal.filePath, - line: getFrontmatterErrorLine( - entry._internal.rawData, - String(parsed.error.errors[0].path[0]) + if (!formattedError) { + formattedError = new AstroError({ + ...AstroErrorData.InvalidContentEntryFrontmatterError, + message: AstroErrorData.InvalidContentEntryFrontmatterError.message( + entry.collection, + entry.id, + parsed.error ), - column: 0, - }, - }); + location: { + file: entry._internal.filePath, + line: getYAMLErrorLine(entry._internal.rawData, String(parsed.error.errors[0].path[0])), + column: 0, + }, + }); + } throw formattedError; } } @@ -136,6 +168,10 @@ export function getContentEntryExts(settings: Pick t.extensions).flat(); } +export function getDataEntryExts(settings: Pick) { + return settings.dataEntryTypes.map((t) => t.extensions).flat(); +} + export function getContentEntryConfigByExtMap(settings: Pick) { const map: Map = new Map(); for (const entryType of settings.contentEntryTypes) { @@ -146,35 +182,45 @@ export function getContentEntryConfigByExtMap(settings: Pick & { entry: string | URL }) { + const entryPath = typeof entry === 'string' ? entry : fileURLToPath(entry); + const rawRelativePath = path.relative(fileURLToPath(contentDir), entryPath); + const collectionName = path.dirname(rawRelativePath).split(path.sep)[0]; + const isOutsideCollection = + !collectionName || collectionName === '' || collectionName === '..' || collectionName === '.'; -export function getEntryInfo( - params: Pick & { - entry: string | URL; - allowFilesOutsideCollection?: true; + if (isOutsideCollection) { + return undefined; } -): EntryInfo; -export function getEntryInfo({ + + return collectionName; +} + +export function getDataEntryId({ entry, contentDir, - allowFilesOutsideCollection = false, -}: Pick & { - entry: string | URL; - allowFilesOutsideCollection?: boolean; -}): EntryInfo | NoCollectionError { - const rawRelativePath = path.relative( - fileURLToPath(contentDir), - typeof entry === 'string' ? entry : fileURLToPath(entry) - ); - const rawCollection = path.dirname(rawRelativePath).split(path.sep).shift(); - const isOutsideCollection = rawCollection === '..' || rawCollection === '.'; + collection, +}: Pick & { entry: URL; collection: string }): string { + const relativePath = getRelativeEntryPath(entry, collection, contentDir); + const withoutFileExt = relativePath.replace(new RegExp(path.extname(relativePath) + '$'), ''); - if (!rawCollection || (!allowFilesOutsideCollection && isOutsideCollection)) - return new NoCollectionError(); + return withoutFileExt; +} - const rawId = path.relative(rawCollection, rawRelativePath); - const rawIdWithoutFileExt = rawId.replace(new RegExp(path.extname(rawId) + '$'), ''); - const rawSlugSegments = rawIdWithoutFileExt.split(path.sep); +export function getContentEntryIdAndSlug({ + entry, + contentDir, + collection, +}: Pick & { entry: URL; collection: string }): { + id: string; + slug: string; +} { + const relativePath = getRelativeEntryPath(entry, collection, contentDir); + const withoutFileExt = relativePath.replace(new RegExp(path.extname(relativePath) + '$'), ''); + const rawSlugSegments = withoutFileExt.split(path.sep); const slug = rawSlugSegments // Slugify each route segment to handle capitalization and spaces. @@ -184,20 +230,26 @@ export function getEntryInfo({ .replace(/\/index$/, ''); const res = { - id: normalizePath(rawId), + id: normalizePath(relativePath), slug, - collection: normalizePath(rawCollection), }; return res; } +function getRelativeEntryPath(entry: URL, collection: string, contentDir: URL) { + const relativeToContent = path.relative(fileURLToPath(contentDir), fileURLToPath(entry)); + const relativeToCollection = path.relative(collection, relativeToContent); + return relativeToCollection; +} + export function getEntryType( entryPath: string, paths: Pick, contentFileExts: string[], + dataFileExts: string[], // TODO: Unflag this when we're ready to release assets - erika, 2023-04-12 - experimentalAssets: boolean -): 'content' | 'config' | 'ignored' | 'unsupported' { + experimentalAssets = false +): 'content' | 'data' | 'config' | 'ignored' | 'unsupported' { const { ext, base } = path.parse(entryPath); const fileUrl = pathToFileURL(entryPath); @@ -209,6 +261,8 @@ export function getEntryType( return 'ignored'; } else if (contentFileExts.includes(ext)) { return 'content'; + } else if (dataFileExts.includes(ext)) { + return 'data'; } else if (fileUrl.href === paths.config.url.href) { return 'config'; } else { @@ -238,33 +292,29 @@ export function hasUnderscoreBelowContentDirectoryPath( return false; } -function getFrontmatterErrorLine(rawFrontmatter: string | undefined, frontmatterKey: string) { - if (!rawFrontmatter) return 0; - const indexOfFrontmatterKey = rawFrontmatter.indexOf(`\n${frontmatterKey}`); - if (indexOfFrontmatterKey === -1) return 0; +function getYAMLErrorLine(rawData: string | undefined, objectKey: string) { + if (!rawData) return 0; + const indexOfObjectKey = rawData.search( + // Match key either at the top of the file or after a newline + // Ensures matching on top-level object keys only + new RegExp(`(\n|^)${objectKey}`) + ); + if (indexOfObjectKey === -1) return 0; - const frontmatterBeforeKey = rawFrontmatter.substring(0, indexOfFrontmatterKey + 1); - const numNewlinesBeforeKey = frontmatterBeforeKey.split('\n').length; + const dataBeforeKey = rawData.substring(0, indexOfObjectKey + 1); + const numNewlinesBeforeKey = dataBeforeKey.split('\n').length; return numNewlinesBeforeKey; } -/** - * Match YAML exception handling from Astro core errors - * @see 'astro/src/core/errors.ts' - */ export function parseFrontmatter(fileContents: string, filePath: string) { try { // `matter` is empty string on cache results // clear cache to prevent this (matter as any).clearCache(); return matter(fileContents); - } catch (e: any) { - if (e.name === 'YAMLException') { - const err: Error & ViteErrorPayload['err'] = e; - err.id = filePath; - err.loc = { file: e.id, line: e.mark.line + 1, column: e.mark.column }; - err.message = e.reason; - throw err; + } catch (e) { + if (isYAMLException(e)) { + throw formatYAMLException(e); } else { throw e; } @@ -278,6 +328,11 @@ export function parseFrontmatter(fileContents: string, filePath: string) { */ export const globalContentConfigObserver = contentObservable({ status: 'init' }); +export function hasContentFlag(viteId: string, flag: (typeof CONTENT_FLAGS)[number]) { + const flags = new URLSearchParams(viteId.split('?')[1] ?? ''); + return flags.has(flag); +} + export async function loadContentConfig({ fs, settings, @@ -292,12 +347,9 @@ export async function loadContentConfig({ if (!contentPaths.config.exists) { return undefined; } - try { - const configPathname = fileURLToPath(contentPaths.config.url); - unparsedConfig = await viteServer.ssrLoadModule(configPathname); - } catch (e) { - throw e; - } + const configPathname = fileURLToPath(contentPaths.config.url); + unparsedConfig = await viteServer.ssrLoadModule(configPathname); + const config = contentConfigParser.safeParse(unparsedConfig); if (config.success) { return config.data; @@ -306,6 +358,31 @@ export async function loadContentConfig({ } } +export async function reloadContentConfigObserver({ + observer = globalContentConfigObserver, + ...loadContentConfigOpts +}: { + fs: typeof fsMod; + settings: AstroSettings; + viteServer: ViteDevServer; + observer?: ContentObservable; +}) { + observer.set({ status: 'loading' }); + try { + const config = await loadContentConfig(loadContentConfigOpts); + if (config) { + observer.set({ status: 'loaded', config }); + } else { + observer.set({ status: 'does-not-exist' }); + } + } catch (e) { + observer.set({ + status: 'error', + error: e instanceof Error ? e : new AstroError(AstroErrorData.UnknownContentCollectionError), + }); + } +} + type ContentCtx = | { status: 'init' } | { status: 'loading' } @@ -414,7 +491,7 @@ export async function getEntrySlug({ } const { slug: frontmatterSlug } = await contentEntryType.getEntryInfo({ fileUrl, - contents: await fs.promises.readFile(fileUrl, 'utf-8'), + contents, }); return parseEntrySlug({ generatedSlug, frontmatterSlug, id, collection }); } diff --git a/packages/astro/src/content/vite-plugin-content-imports.ts b/packages/astro/src/content/vite-plugin-content-imports.ts index c25eff37f..466f4bd00 100644 --- a/packages/astro/src/content/vite-plugin-content-imports.ts +++ b/packages/astro/src/content/vite-plugin-content-imports.ts @@ -4,29 +4,35 @@ import { extname } from 'node:path'; import type { PluginContext } from 'rollup'; import { pathToFileURL } from 'url'; import type { Plugin } from 'vite'; -import type { AstroSettings, ContentEntryModule, ContentEntryType } from '../@types/astro.js'; +import type { + AstroSettings, + ContentEntryModule, + ContentEntryType, + DataEntryModule, + DataEntryType, +} from '../@types/astro.js'; import { AstroErrorData } from '../core/errors/errors-data.js'; import { AstroError } from '../core/errors/errors.js'; import { escapeViteEnvReferences, getFileInfo } from '../vite-plugin-utils/index.js'; -import { CONTENT_FLAG } from './consts.js'; +import { CONTENT_FLAG, DATA_FLAG } from './consts.js'; import { - getContentEntryConfigByExtMap, + type ContentConfig, + type ContentPaths, getContentEntryExts, getContentPaths, getEntryData, - getEntryInfo, + parseEntrySlug, + getContentEntryIdAndSlug, getEntryType, globalContentConfigObserver, - NoCollectionError, - parseEntrySlug, - type ContentConfig, + getContentEntryConfigByExtMap, + getEntryCollectionName, + getDataEntryExts, + hasContentFlag, + getDataEntryId, + reloadContentConfigObserver, } from './utils.js'; -function isContentFlagImport(viteId: string) { - const flags = new URLSearchParams(viteId.split('?')[1]); - return flags.has(CONTENT_FLAG); -} - function getContentRendererByViteId( viteId: string, settings: Pick @@ -45,6 +51,13 @@ function getContentRendererByViteId( } const CHOKIDAR_MODIFIED_EVENTS = ['add', 'unlink', 'change']; +/** + * If collection entries change, import modules need to be invalidated. + * Reasons why: + * - 'config' - content imports depend on the config file for parsing schemas + * - 'data' | 'content' - the config may depend on collection entries via `reference()` + */ +const COLLECTION_TYPES_TO_INVALIDATE_ON = ['data', 'content', 'config']; export function astroContentImportPlugin({ fs, @@ -55,14 +68,46 @@ export function astroContentImportPlugin({ }): Plugin[] { const contentPaths = getContentPaths(settings.config, fs); const contentEntryExts = getContentEntryExts(settings); + const dataEntryExts = getDataEntryExts(settings); const contentEntryConfigByExt = getContentEntryConfigByExtMap(settings); + const dataEntryExtToParser: Map = new Map(); + for (const entryType of settings.dataEntryTypes) { + for (const ext of entryType.extensions) { + dataEntryExtToParser.set(ext, entryType.getEntryInfo); + } + } + const plugins: Plugin[] = [ { name: 'astro:content-imports', async transform(_, viteId) { - if (isContentFlagImport(viteId)) { + if (hasContentFlag(viteId, DATA_FLAG)) { + const fileId = viteId.split('?')[0] ?? viteId; + // Data collections don't need to rely on the module cache. + // This cache only exists for the `render()` function specific to content. + const { id, data, collection, _internal } = await getDataEntryModule({ + fileId, + dataEntryExtToParser, + contentPaths, + settings, + fs, + pluginContext: this, + }); + + const code = escapeViteEnvReferences(` +export const id = ${JSON.stringify(id)}; +export const collection = ${JSON.stringify(collection)}; +export const data = ${devalue.uneval(data) /* TODO: reuse astro props serializer */}; +export const _internal = { + type: 'data', + filePath: ${JSON.stringify(_internal.filePath)}, + rawData: ${JSON.stringify(_internal.rawData)}, +}; +`); + return code; + } else if (hasContentFlag(viteId, CONTENT_FLAG)) { const fileId = viteId.split('?')[0]; const { id, slug, collection, body, data, _internal } = await setContentEntryModuleCache({ fileId, @@ -76,6 +121,7 @@ export function astroContentImportPlugin({ export const body = ${JSON.stringify(body)}; export const data = ${devalue.uneval(data) /* TODO: reuse astro props serializer */}; export const _internal = { + type: 'content', filePath: ${JSON.stringify(_internal.filePath)}, rawData: ${JSON.stringify(_internal.rawData)}, };`); @@ -85,19 +131,28 @@ export function astroContentImportPlugin({ }, configureServer(viteServer) { viteServer.watcher.on('all', async (event, entry) => { - if ( - CHOKIDAR_MODIFIED_EVENTS.includes(event) && - getEntryType( + if (CHOKIDAR_MODIFIED_EVENTS.includes(event)) { + const entryType = getEntryType( entry, contentPaths, contentEntryExts, + dataEntryExts, settings.config.experimental.assets - ) === 'config' - ) { - // Content modules depend on config, so we need to invalidate them. + ); + if (!COLLECTION_TYPES_TO_INVALIDATE_ON.includes(entryType)) return; + + // The content config could depend on collection entries via `reference()`. + // Reload the config in case of changes. + if (entryType === 'content' || entryType === 'data') { + await reloadContentConfigObserver({ fs, settings, viteServer }); + } + + // Invalidate all content imports and `render()` modules. + // TODO: trace `reference()` calls for fine-grained invalidation. for (const modUrl of viteServer.moduleGraph.urlToModuleMap.keys()) { if ( - isContentFlagImport(modUrl) || + hasContentFlag(modUrl, CONTENT_FLAG) || + hasContentFlag(modUrl, DATA_FLAG) || Boolean(getContentRendererByViteId(modUrl, settings)) ) { const mod = await viteServer.moduleGraph.getModuleByUrl(modUrl); @@ -210,13 +265,13 @@ export function astroContentImportPlugin({ fileUrl: pathToFileURL(fileId), contents: rawContents, }); - const entryInfoResult = getEntryInfo({ - entry: pathToFileURL(fileId), - contentDir: contentPaths.contentDir, - }); - if (entryInfoResult instanceof NoCollectionError) throw entryInfoResult; + const entry = pathToFileURL(fileId); + const { contentDir } = contentPaths; + const collection = getEntryCollectionName({ entry, contentDir }); + if (collection === undefined) + throw new AstroError(AstroErrorData.UnknownContentCollectionError); - const { id, slug: generatedSlug, collection } = entryInfoResult; + const { id, slug: generatedSlug } = getContentEntryIdAndSlug({ entry, contentDir, collection }); const _internal = { filePath: fileId, rawData: rawData }; // TODO: move slug calculation to the start of the build @@ -231,10 +286,10 @@ export function astroContentImportPlugin({ const collectionConfig = contentConfig?.collections[collection]; let data = collectionConfig ? await getEntryData( - { id, collection, slug, _internal, unvalidatedData }, + { id, collection, _internal, unvalidatedData }, collectionConfig, pluginContext, - settings + settings.config ) : unvalidatedData; @@ -297,3 +352,68 @@ async function getContentConfigFromGlobal() { return contentConfig; } + +type GetDataEntryModuleParams = { + fs: typeof fsMod; + fileId: string; + contentPaths: Pick; + pluginContext: PluginContext; + dataEntryExtToParser: Map; + settings: Pick; +}; + +async function getDataEntryModule({ + fileId, + dataEntryExtToParser, + contentPaths, + fs, + pluginContext, + settings, +}: GetDataEntryModuleParams): Promise { + const contentConfig = await getContentConfigFromGlobal(); + let rawContents; + try { + rawContents = await fs.promises.readFile(fileId, 'utf-8'); + } catch (e) { + throw new AstroError({ + ...AstroErrorData.UnknownContentCollectionError, + message: `Unexpected error reading entry ${JSON.stringify(fileId)}.`, + stack: e instanceof Error ? e.stack : undefined, + }); + } + const fileExt = extname(fileId); + const dataEntryParser = dataEntryExtToParser.get(fileExt); + + if (!dataEntryParser) { + throw new AstroError({ + ...AstroErrorData.UnknownContentCollectionError, + message: `No parser found for data entry ${JSON.stringify( + fileId + )}. Did you apply an integration for this file type?`, + }); + } + const { data: unvalidatedData, rawData = '' } = await dataEntryParser({ + fileUrl: pathToFileURL(fileId), + contents: rawContents, + }); + const entry = pathToFileURL(fileId); + const { contentDir } = contentPaths; + const collection = getEntryCollectionName({ entry, contentDir }); + if (collection === undefined) throw new AstroError(AstroErrorData.UnknownContentCollectionError); + + const id = getDataEntryId({ entry, contentDir, collection }); + + const _internal = { filePath: fileId, rawData }; + + const collectionConfig = contentConfig?.collections[collection]; + const data = collectionConfig + ? await getEntryData( + { id, collection, _internal, unvalidatedData }, + collectionConfig, + pluginContext, + settings.config + ) + : unvalidatedData; + + return { id, collection, data, _internal }; +} diff --git a/packages/astro/src/content/vite-plugin-content-virtual-mod.ts b/packages/astro/src/content/vite-plugin-content-virtual-mod.ts index a880510b0..3f5b463ab 100644 --- a/packages/astro/src/content/vite-plugin-content-virtual-mod.ts +++ b/packages/astro/src/content/vite-plugin-content-virtual-mod.ts @@ -1,20 +1,24 @@ -import glob, { type Options as FastGlobOptions } from 'fast-glob'; +import glob from 'fast-glob'; import fsMod from 'node:fs'; import { extname } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; import type { Plugin } from 'vite'; import type { AstroSettings } from '../@types/astro.js'; import { rootRelativePath } from '../core/util.js'; +import { AstroError, AstroErrorData } from '../core/errors/index.js'; import { VIRTUAL_MODULE_ID } from './consts.js'; import { getContentEntryConfigByExtMap, + getDataEntryExts, getContentPaths, - getEntryInfo, - getEntrySlug, getExtGlob, - hasUnderscoreBelowContentDirectoryPath, - NoCollectionError, + getEntryCollectionName, + getContentEntryIdAndSlug, + getEntrySlug, + getDataEntryId, + getEntryType, type ContentPaths, + type ContentLookupMap, } from './utils.js'; interface AstroContentVirtualModPluginParams { @@ -29,34 +33,41 @@ export function astroContentVirtualModPlugin({ const contentEntryConfigByExt = getContentEntryConfigByExtMap(settings); const contentEntryExts = [...contentEntryConfigByExt.keys()]; + const dataEntryExts = getDataEntryExts(settings); - const extGlob = getExtGlob(contentEntryExts); - const entryGlob = `${relContentDir}**/*${extGlob}`; const virtualModContents = fsMod .readFileSync(contentPaths.virtualModTemplate, 'utf-8') + .replace( + '@@COLLECTION_NAME_BY_REFERENCE_KEY@@', + new URL('reference-map.json', contentPaths.cacheDir).pathname + ) .replace('@@CONTENT_DIR@@', relContentDir) - .replace('@@ENTRY_GLOB_PATH@@', entryGlob) - .replace('@@RENDER_ENTRY_GLOB_PATH@@', entryGlob); + .replace('@@CONTENT_ENTRY_GLOB_PATH@@', `${relContentDir}**/*${getExtGlob(contentEntryExts)}`) + .replace('@@DATA_ENTRY_GLOB_PATH@@', `${relContentDir}**/*${getExtGlob(dataEntryExts)}`) + .replace( + '@@RENDER_ENTRY_GLOB_PATH@@', + `${relContentDir}**/*${getExtGlob(/** Note: data collections excluded */ contentEntryExts)}` + ); const astroContentVirtualModuleId = '\0' + VIRTUAL_MODULE_ID; return { name: 'astro-content-virtual-mod-plugin', - enforce: 'pre', resolveId(id) { if (id === VIRTUAL_MODULE_ID) { return astroContentVirtualModuleId; } }, async load(id) { - const stringifiedLookupMap = await getStringifiedLookupMap({ - fs: fsMod, - contentPaths, - contentEntryConfigByExt, - root: settings.config.root, - }); - if (id === astroContentVirtualModuleId) { + const stringifiedLookupMap = await getStringifiedLookupMap({ + fs: fsMod, + contentPaths, + contentEntryConfigByExt, + dataEntryExts, + root: settings.config.root, + }); + return { code: virtualModContents.replace( '/* @@LOOKUP_MAP_ASSIGNMENT@@ */', @@ -70,52 +81,65 @@ export function astroContentVirtualModPlugin({ /** * Generate a map from a collection + slug to the local file path. - * This is used internally to resolve entry imports when using `getEntryBySlug()`. + * This is used internally to resolve entry imports when using `getEntry()`. * @see `src/content/virtual-mod.mjs` */ export async function getStringifiedLookupMap({ contentPaths, contentEntryConfigByExt, + dataEntryExts, root, fs, }: { contentEntryConfigByExt: ReturnType; - contentPaths: Pick; + dataEntryExts: string[]; + contentPaths: Pick; root: URL; fs: typeof fsMod; }) { const { contentDir } = contentPaths; - const globOpts: FastGlobOptions = { - absolute: true, - cwd: fileURLToPath(root), - fs: { - readdir: fs.readdir.bind(fs), - readdirSync: fs.readdirSync.bind(fs), - }, - }; - const relContentDir = rootRelativePath(root, contentDir, false); + const contentEntryExts = [...contentEntryConfigByExt.keys()]; + + let lookupMap: ContentLookupMap = {}; const contentGlob = await glob( - `${relContentDir}**/*${getExtGlob([...contentEntryConfigByExt.keys()])}`, - globOpts + `${relContentDir}**/*${getExtGlob([...dataEntryExts, ...contentEntryExts])}`, + { + absolute: true, + cwd: fileURLToPath(root), + fs: { + readdir: fs.readdir.bind(fs), + readdirSync: fs.readdirSync.bind(fs), + }, + } ); - let filePathByLookupId: { - [collection: string]: Record; - } = {}; await Promise.all( - contentGlob - // Ignore underscore files in lookup map - .filter((e) => !hasUnderscoreBelowContentDirectoryPath(pathToFileURL(e), contentDir)) - .map(async (filePath) => { - const info = getEntryInfo({ contentDir, entry: filePath }); - // Globbed entry outside a collection directory - // Log warning during type generation, safe to ignore in lookup map - if (info instanceof NoCollectionError) return; - const contentEntryType = contentEntryConfigByExt.get(extname(filePath)); - if (!contentEntryType) return; + contentGlob.map(async (filePath) => { + const entryType = getEntryType(filePath, contentPaths, contentEntryExts, dataEntryExts); + // Globbed ignored or unsupported entry. + // Logs warning during type generation, should ignore in lookup map. + if (entryType !== 'content' && entryType !== 'data') return; - const { id, collection, slug: generatedSlug } = info; + const collection = getEntryCollectionName({ contentDir, entry: pathToFileURL(filePath) }); + if (!collection) throw UnexpectedLookupMapError; + + if (lookupMap[collection]?.type && lookupMap[collection].type !== entryType) { + throw new AstroError({ + ...AstroErrorData.MixedContentDataCollectionError, + message: AstroErrorData.MixedContentDataCollectionError.message(collection), + }); + } + + if (entryType === 'content') { + const contentEntryType = contentEntryConfigByExt.get(extname(filePath)); + if (!contentEntryType) throw UnexpectedLookupMapError; + + const { id, slug: generatedSlug } = await getContentEntryIdAndSlug({ + entry: pathToFileURL(filePath), + contentDir, + collection, + }); const slug = await getEntrySlug({ id, collection, @@ -124,12 +148,30 @@ export async function getStringifiedLookupMap({ fileUrl: pathToFileURL(filePath), contentEntryType, }); - filePathByLookupId[collection] = { - ...filePathByLookupId[collection], - [slug]: rootRelativePath(root, filePath), + lookupMap[collection] = { + type: 'content', + entries: { + ...lookupMap[collection]?.entries, + [slug]: rootRelativePath(root, filePath), + }, }; - }) + } else { + const id = getDataEntryId({ entry: pathToFileURL(filePath), contentDir, collection }); + lookupMap[collection] = { + type: 'data', + entries: { + ...lookupMap[collection]?.entries, + [id]: rootRelativePath(root, filePath), + }, + }; + } + }) ); - return JSON.stringify(filePathByLookupId); + return JSON.stringify(lookupMap); } + +const UnexpectedLookupMapError = new AstroError({ + ...AstroErrorData.UnknownContentCollectionError, + message: `Unexpected error while parsing content entry IDs and slugs.`, +}); diff --git a/packages/astro/src/core/config/settings.ts b/packages/astro/src/core/config/settings.ts index 7939516da..b4565b1c4 100644 --- a/packages/astro/src/core/config/settings.ts +++ b/packages/astro/src/core/config/settings.ts @@ -1,6 +1,6 @@ import type { AstroConfig, AstroSettings, AstroUserConfig } from '../../@types/astro'; import { SUPPORTED_MARKDOWN_FILE_EXTENSIONS } from './../constants.js'; - +import path from 'node:path'; import { fileURLToPath, pathToFileURL } from 'url'; import jsxRenderer from '../../jsx/renderer.js'; import { isHybridOutput } from '../../prerender/utils.js'; @@ -9,8 +9,13 @@ import { getDefaultClientDirectives } from '../client-directive/index.js'; import { createDefaultDevConfig } from './config.js'; import { AstroTimer } from './timer.js'; import { loadTSConfig } from './tsconfig.js'; +import yaml from 'js-yaml'; +import { formatYAMLException, isYAMLException } from '../errors/utils.js'; +import { AstroError, AstroErrorData } from '../errors/index.js'; +import { getContentPaths } from '../../content/index.js'; export function createBaseSettings(config: AstroConfig): AstroSettings { + const { contentDir } = getContentPaths(config); return { config, tsConfig: undefined, @@ -23,6 +28,78 @@ export function createBaseSettings(config: AstroConfig): AstroSettings { : [], pageExtensions: ['.astro', '.html', ...SUPPORTED_MARKDOWN_FILE_EXTENSIONS], contentEntryTypes: [markdownContentEntryType], + dataEntryTypes: [ + { + extensions: ['.json'], + getEntryInfo({ contents, fileUrl }) { + if (contents === undefined || contents === '') return { data: {} }; + + const pathRelToContentDir = path.relative( + fileURLToPath(contentDir), + fileURLToPath(fileUrl) + ); + let data; + try { + data = JSON.parse(contents); + } catch (e) { + throw new AstroError({ + ...AstroErrorData.DataCollectionEntryParseError, + message: AstroErrorData.DataCollectionEntryParseError.message( + pathRelToContentDir, + e instanceof Error ? e.message : 'contains invalid JSON.' + ), + location: { file: fileUrl.pathname }, + stack: e instanceof Error ? e.stack : undefined, + }); + } + + if (data == null || typeof data !== 'object') { + throw new AstroError({ + ...AstroErrorData.DataCollectionEntryParseError, + message: AstroErrorData.DataCollectionEntryParseError.message( + pathRelToContentDir, + 'data is not an object.' + ), + location: { file: fileUrl.pathname }, + }); + } + + return { data }; + }, + }, + { + extensions: ['.yaml', '.yml'], + getEntryInfo({ contents, fileUrl }) { + try { + const data = yaml.load(contents, { filename: fileURLToPath(fileUrl) }); + const rawData = contents; + + return { data, rawData }; + } catch (e) { + const pathRelToContentDir = path.relative( + fileURLToPath(contentDir), + fileURLToPath(fileUrl) + ); + const formattedError = isYAMLException(e) + ? formatYAMLException(e) + : new Error('contains invalid YAML.'); + + throw new AstroError({ + ...AstroErrorData.DataCollectionEntryParseError, + message: AstroErrorData.DataCollectionEntryParseError.message( + pathRelToContentDir, + formattedError.message + ), + stack: formattedError.stack, + location: + 'loc' in formattedError + ? { file: fileUrl.pathname, ...formattedError.loc } + : { file: fileUrl.pathname }, + }); + } + }, + }, + ], renderers: [jsxRenderer], scripts: [], clientDirectives: getDefaultClientDirectives(), diff --git a/packages/astro/src/core/errors/errors-data.ts b/packages/astro/src/core/errors/errors-data.ts index 9bde01ca9..15d0f7056 100644 --- a/packages/astro/src/core/errors/errors-data.ts +++ b/packages/astro/src/core/errors/errors-data.ts @@ -1002,6 +1002,66 @@ See https://docs.astro.build/en/guides/server-side-rendering/ for more informati hint: 'See https://docs.astro.build/en/guides/content-collections/ for more on the `slug` field.', }, + /** + * @docs + * @message A collection queried via `getCollection()` does not exist. + * @description + * When querying a collection, ensure a collection directory with the requested name exists under `src/content/`. + */ + CollectionDoesNotExistError: { + title: 'Collection does not exist', + code: 9004, + message: (collection: string) => { + return `The collection **${collection}** does not exist. Ensure a collection directory with this name exists.`; + }, + hint: 'See https://docs.astro.build/en/guides/content-collections/ for more on creating collections.', + }, + /** + * @docs + * @message `COLLECTION_NAME` contains a mix of content and data entries. All entries must be of the same type. + * @see + * - [Defining content collections](https://docs.astro.build/en/guides/content-collections/#defining-collections) + * @description + * A content collection cannot contain a mix of content and data entries. You must store entries in separate collections by type. + */ + MixedContentDataCollectionError: { + title: 'Content and data cannot be in same collection.', + code: 9005, + message: (collection: string) => { + return `**${collection}** contains a mix of content and data entries. All entries must be of the same type.`; + }, + hint: 'Store data entries in a new collection separate from your content collection.', + }, + /** + * @docs + * @message `COLLECTION_NAME` contains entries of type `ACTUAL_TYPE`, but is configured as a `EXPECTED_TYPE` collection. + * @see + * - [Defining content collections](https://docs.astro.build/en/guides/content-collections/#defining-collections) + * @description + * Content collections must contain entries of the type configured. Collections are `type: 'content'` by default. Try adding `type: 'data'` to your collection config for data collections. + */ + ContentCollectionTypeMismatchError: { + title: 'Collection contains entries of a different type.', + code: 9006, + message: (collection: string, expectedType: string, actualType: string) => { + return `${collection} contains ${expectedType} entries, but is configured as a ${actualType} collection.`; + }, + }, + /** + * @docs + * @message `COLLECTION_ENTRY_NAME` failed to parse. + * @description + * Collection entries of `type: 'data'` must return an object with valid JSON (for `.json` entries) or YAML (for `.yaml` entries). + */ + DataCollectionEntryParseError: { + title: 'Data collection entry failed to parse.', + code: 9007, + message: (entryId: string, errorMessage: string) => { + return `**${entryId}** failed to parse: ${errorMessage}`; + }, + hint: 'Ensure your data entry is an object with valid JSON (for `.json` entries) or YAML (for `.yaml` entries).', + }, + // Generic catch-all - Only use this in extreme cases, like if there was a cosmic ray bit flip UnknownError: { title: 'Unknown Error.', diff --git a/packages/astro/src/core/errors/errors.ts b/packages/astro/src/core/errors/errors.ts index 8ff43c60e..995f401ef 100644 --- a/packages/astro/src/core/errors/errors.ts +++ b/packages/astro/src/core/errors/errors.ts @@ -28,6 +28,10 @@ type ErrorTypes = | 'InternalError' | 'AggregateError'; +export function isAstroError(e: unknown): e is AstroError { + return e instanceof Error && (e as AstroError).type === 'AstroError'; +} + export class AstroError extends Error { // NOTE: If this property is named `code`, Rollup will use it to fill the `pluginCode` property downstream // This cause issues since we expect `pluginCode` to be a string containing code diff --git a/packages/astro/src/core/errors/index.ts b/packages/astro/src/core/errors/index.ts index 24ba6b94e..438a77b39 100644 --- a/packages/astro/src/core/errors/index.ts +++ b/packages/astro/src/core/errors/index.ts @@ -1,5 +1,12 @@ export type { ErrorLocation, ErrorWithMetadata } from './errors'; export { AstroErrorData, type AstroErrorCodes } from './errors-data.js'; -export { AggregateError, AstroError, CompilerError, CSSError, MarkdownError } from './errors.js'; +export { + AggregateError, + AstroError, + CompilerError, + CSSError, + MarkdownError, + isAstroError, +} from './errors.js'; export { codeFrame } from './printer.js'; export { createSafeError, positionAt } from './utils.js'; diff --git a/packages/astro/src/core/errors/utils.ts b/packages/astro/src/core/errors/utils.ts index fed7dc459..e7f6f2643 100644 --- a/packages/astro/src/core/errors/utils.ts +++ b/packages/astro/src/core/errors/utils.ts @@ -1,6 +1,8 @@ import type { DiagnosticCode } from '@astrojs/compiler/shared/diagnostics.js'; import type { SSRError } from '../../@types/astro.js'; import { AstroErrorData, type AstroErrorCodes, type ErrorData } from './errors-data.js'; +import type { YAMLException } from 'js-yaml'; +import type { ErrorPayload as ViteErrorPayload } from 'vite'; /** * Get the line and character based on the offset @@ -71,6 +73,21 @@ function getLineOffsets(text: string) { return lineOffsets; } +export function isYAMLException(err: unknown): err is YAMLException { + return err instanceof Error && err.name === 'YAMLException'; +} + +/** Format YAML exceptions as Vite errors */ +export function formatYAMLException(e: YAMLException): ViteErrorPayload['err'] { + return { + name: e.name, + id: e.mark.name, + loc: { file: e.mark.name, line: e.mark.line + 1, column: e.mark.column }, + message: e.reason, + stack: e.stack ?? '', + }; +} + /** Coalesce any throw variable to an Error instance. */ export function createSafeError(err: any): Error { if (err instanceof Error || (err && err.name && err.message)) { diff --git a/packages/astro/src/core/sync/index.ts b/packages/astro/src/core/sync/index.ts index 0017225d0..144792fb4 100644 --- a/packages/astro/src/core/sync/index.ts +++ b/packages/astro/src/core/sync/index.ts @@ -1,7 +1,7 @@ import { dim } from 'kleur/colors'; import type fsMod from 'node:fs'; import { performance } from 'node:perf_hooks'; -import { createServer } from 'vite'; +import { createServer, type HMRPayload } from 'vite'; import type { Arguments } from 'yargs-parser'; import type { AstroSettings } from '../../@types/astro'; import { createContentTypesGenerator } from '../../content/index.js'; @@ -13,6 +13,7 @@ import { createVite } from '../create-vite.js'; import { AstroError, AstroErrorData, createSafeError } from '../errors/index.js'; import { info, type LogOptions } from '../logger/core.js'; import { printHelp } from '../messages.js'; +import { isAstroError } from '../errors/index.js'; export type ProcessExit = 0 | 1; @@ -74,6 +75,16 @@ export async function sync( ) ); + // Patch `ws.send` to bubble up error events + // `ws.on('error')` does not fire for some reason + const wsSend = tempViteServer.ws.send; + tempViteServer.ws.send = (payload: HMRPayload) => { + if (payload.type === 'error') { + throw payload.err; + } + return wsSend(payload); + }; + try { const contentTypesGenerator = await createContentTypesGenerator({ contentConfigObserver: globalContentConfigObserver, @@ -99,6 +110,9 @@ export async function sync( } } catch (e) { const safeError = createSafeError(e); + if (isAstroError(e)) { + throw e; + } throw new AstroError( { ...AstroErrorData.GenerateContentTypesError, diff --git a/packages/astro/src/integrations/index.ts b/packages/astro/src/integrations/index.ts index e53e37e42..5d88a1196 100644 --- a/packages/astro/src/integrations/index.ts +++ b/packages/astro/src/integrations/index.ts @@ -9,6 +9,7 @@ import type { AstroSettings, BuildConfig, ContentEntryType, + DataEntryType, HookParameters, RouteData, } from '../@types/astro.js'; @@ -127,6 +128,9 @@ export async function runHookConfigSetup({ function addContentEntryType(contentEntryType: ContentEntryType) { updatedSettings.contentEntryTypes.push(contentEntryType); } + function addDataEntryType(dataEntryType: DataEntryType) { + updatedSettings.dataEntryTypes.push(dataEntryType); + } Object.defineProperty(hooks, 'addPageExtension', { value: addPageExtension, @@ -138,6 +142,11 @@ export async function runHookConfigSetup({ writable: false, enumerable: false, }); + Object.defineProperty(hooks, 'addDataEntryType', { + value: addDataEntryType, + writable: false, + enumerable: false, + }); // --- await withTakingALongTimeMsg({ diff --git a/packages/astro/test/content-collection-references.test.js b/packages/astro/test/content-collection-references.test.js new file mode 100644 index 000000000..da1548621 --- /dev/null +++ b/packages/astro/test/content-collection-references.test.js @@ -0,0 +1,158 @@ +import { expect } from 'chai'; +import * as cheerio from 'cheerio'; +import { fixLineEndings, loadFixture } from './test-utils.js'; + +describe('Content Collections - references', () => { + let fixture; + let devServer; + before(async () => { + fixture = await loadFixture({ root: './fixtures/content-collection-references/' }); + }); + + const modes = ['dev', 'prod']; + + for (const mode of modes) { + describe(mode, () => { + before(async () => { + if (mode === 'prod') { + await fixture.build(); + } else if (mode === 'dev') { + devServer = await fixture.startDevServer(); + } + }); + + after(async () => { + if (mode === 'dev') devServer?.stop(); + }); + + describe(`JSON result`, () => { + let json; + before(async () => { + if (mode === 'prod') { + const rawJson = await fixture.readFile('/welcome-data.json'); + json = JSON.parse(rawJson); + } else if (mode === 'dev') { + const rawJsonResponse = await fixture.fetch('/welcome-data.json'); + const rawJson = await rawJsonResponse.text(); + json = JSON.parse(rawJson); + } + }); + + it('Returns expected keys', () => { + expect(json).to.haveOwnProperty('welcomePost'); + expect(json).to.haveOwnProperty('banner'); + expect(json).to.haveOwnProperty('author'); + expect(json).to.haveOwnProperty('relatedPosts'); + }); + + it('Returns `banner` data', () => { + const { banner } = json; + expect(banner).to.haveOwnProperty('data'); + expect(banner.id).to.equal('welcome'); + expect(banner.collection).to.equal('banners'); + expect(banner.data.alt).to.equal( + 'Futuristic landscape with chrome buildings and blue skies' + ); + + expect(banner.data.src.width).to.equal(400); + expect(banner.data.src.height).to.equal(225); + expect(banner.data.src.format).to.equal('jpg'); + expect(banner.data.src.src.includes('the-future')).to.be.true; + }); + + it('Returns `author` data', () => { + const { author } = json; + expect(author).to.haveOwnProperty('data'); + expect(author).to.deep.equal({ + id: 'nate-moore', + collection: 'authors', + data: { + name: 'Nate Something Moore', + twitter: 'https://twitter.com/n_moore', + }, + }); + }); + + it('Returns `relatedPosts` data', () => { + const { relatedPosts } = json; + expect(Array.isArray(relatedPosts)).to.be.true; + const topLevelInfo = relatedPosts.map(({ data, body, ...meta }) => ({ + ...meta, + body: fixLineEndings(body).trim(), + })); + expect(topLevelInfo).to.deep.equal([ + { + id: 'related-1.md', + slug: 'related-1', + body: '# Related post 1\n\nThis is related to the welcome post.', + collection: 'blog', + }, + { + id: 'related-2.md', + slug: 'related-2', + body: '# Related post 2\n\nThis is related to the welcome post.', + collection: 'blog', + }, + ]); + const postData = relatedPosts.map(({ data }) => data); + expect(postData).to.deep.equal([ + { + title: 'Related post 1', + banner: { id: 'welcome', collection: 'banners' }, + author: { id: 'fred-schott', collection: 'authors' }, + }, + { + title: 'Related post 2', + banner: { id: 'welcome', collection: 'banners' }, + author: { id: 'ben-holmes', collection: 'authors' }, + }, + ]); + }); + }); + + describe(`Render result`, () => { + let $; + before(async () => { + if (mode === 'prod') { + const html = await fixture.readFile('/welcome/index.html'); + $ = cheerio.load(html); + } else if (mode === 'dev') { + const htmlResponse = await fixture.fetch('/welcome'); + const html = await htmlResponse.text(); + $ = cheerio.load(html); + } + }); + + it('Renders `banner` data', () => { + const banner = $('img[data-banner]'); + expect(banner.length).to.equal(1); + expect(banner.attr('src')).to.include('the-future'); + expect(banner.attr('alt')).to.equal( + 'Futuristic landscape with chrome buildings and blue skies' + ); + expect(banner.attr('width')).to.equal('400'); + expect(banner.attr('height')).to.equal('225'); + }); + + it('Renders `author` data', () => { + const author = $('a[data-author-name]'); + expect(author.length).to.equal(1); + expect(author.attr('href')).to.equal('https://twitter.com/n_moore'); + expect(author.text()).to.equal('Nate Something Moore'); + }); + + it('Renders `relatedPosts` data', () => { + const relatedPosts = $('ul[data-related-posts]'); + expect(relatedPosts.length).to.equal(1); + const relatedPost1 = relatedPosts.find('li').eq(0); + + expect(relatedPost1.find('a').attr('href')).to.equal('/blog/related-1'); + expect(relatedPost1.find('a').text()).to.equal('Related post 1'); + const relatedPost2 = relatedPosts.find('li').eq(1); + expect(relatedPost2.find('a').attr('href')).to.equal('/blog/related-2'); + expect(relatedPost2.find('a').text()).to.equal('Related post 2'); + }); + }); + }); + } +}); diff --git a/packages/astro/test/data-collections.test.js b/packages/astro/test/data-collections.test.js new file mode 100644 index 000000000..104284031 --- /dev/null +++ b/packages/astro/test/data-collections.test.js @@ -0,0 +1,151 @@ +import { expect } from 'chai'; +import { loadFixture } from './test-utils.js'; + +const authorIds = ['Ben Holmes', 'Fred K Schott', 'Nate Moore']; +const translationIds = ['en', 'es', 'fr']; + +describe('Content Collections - data collections', () => { + let fixture; + before(async () => { + fixture = await loadFixture({ root: './fixtures/data-collections/' }); + await fixture.build(); + }); + + describe('Authors Collection', () => { + let json; + before(async () => { + const rawJson = await fixture.readFile('/authors/all.json'); + json = JSON.parse(rawJson); + }); + + it('Returns', async () => { + expect(Array.isArray(json)).to.be.true; + expect(json.length).to.equal(3); + }); + + it('Generates correct ids', async () => { + const ids = json.map((item) => item.id).sort(); + expect(ids).to.deep.equal(['Ben Holmes', 'Fred K Schott', 'Nate Moore']); + }); + + it('Generates correct data', async () => { + const names = json.map((item) => item.data.name); + expect(names).to.deep.equal(['Ben J Holmes', 'Fred K Schott', 'Nate Something Moore']); + + const twitterUrls = json.map((item) => item.data.twitter); + expect(twitterUrls).to.deep.equal([ + 'https://twitter.com/bholmesdev', + 'https://twitter.com/FredKSchott', + 'https://twitter.com/n_moore', + ]); + }); + }); + + describe('Authors Entry', () => { + for (const authorId of authorIds) { + let json; + before(async () => { + const rawJson = await fixture.readFile(`/authors/${authorId}.json`); + json = JSON.parse(rawJson); + }); + + it(`Returns ${authorId}`, async () => { + expect(json).to.haveOwnProperty('id'); + expect(json.id).to.equal(authorId); + }); + + it(`Generates correct data for ${authorId}`, async () => { + expect(json).to.haveOwnProperty('data'); + expect(json.data).to.haveOwnProperty('name'); + expect(json.data).to.haveOwnProperty('twitter'); + + switch (authorId) { + case 'Ben Holmes': + expect(json.data.name).to.equal('Ben J Holmes'); + expect(json.data.twitter).to.equal('https://twitter.com/bholmesdev'); + break; + case 'Fred K Schott': + expect(json.data.name).to.equal('Fred K Schott'); + expect(json.data.twitter).to.equal('https://twitter.com/FredKSchott'); + break; + case 'Nate Moore': + expect(json.data.name).to.equal('Nate Something Moore'); + expect(json.data.twitter).to.equal('https://twitter.com/n_moore'); + break; + } + }); + } + }); + + describe('Translations Collection', () => { + let json; + before(async () => { + const rawJson = await fixture.readFile('/translations/all.json'); + json = JSON.parse(rawJson); + }); + + it('Returns', async () => { + expect(Array.isArray(json)).to.be.true; + expect(json.length).to.equal(3); + }); + + it('Generates correct ids', async () => { + const ids = json.map((item) => item.id).sort(); + expect(ids).to.deep.equal(translationIds); + }); + + it('Generates correct data', async () => { + const sorted = json.sort((a, b) => a.id.localeCompare(b.id)); + const homepageGreetings = sorted.map((item) => item.data.homepage?.greeting); + expect(homepageGreetings).to.deep.equal([ + 'Hello World!', + '¡Hola Mundo!', + 'Bonjour le monde!', + ]); + + const homepagePreambles = sorted.map((item) => item.data.homepage?.preamble); + expect(homepagePreambles).to.deep.equal([ + 'Welcome to the future of content.', + 'Bienvenido al futuro del contenido.', + 'Bienvenue dans le futur du contenu.', + ]); + }); + }); + + describe('Translations Entry', () => { + for (const translationId of translationIds) { + let json; + before(async () => { + const rawJson = await fixture.readFile(`/translations/${translationId}.json`); + json = JSON.parse(rawJson); + }); + + it(`Returns ${translationId}`, async () => { + expect(json).to.haveOwnProperty('id'); + expect(json.id).to.equal(translationId); + }); + + it(`Generates correct data for ${translationId}`, async () => { + expect(json).to.haveOwnProperty('data'); + expect(json.data).to.haveOwnProperty('homepage'); + expect(json.data.homepage).to.haveOwnProperty('greeting'); + expect(json.data.homepage).to.haveOwnProperty('preamble'); + + switch (translationId) { + case 'en': + expect(json.data.homepage.greeting).to.equal('Hello World!'); + expect(json.data.homepage.preamble).to.equal('Welcome to the future of content.'); + break; + case 'es': + expect(json.data.homepage.greeting).to.equal('¡Hola Mundo!'); + expect(json.data.homepage.preamble).to.equal('Bienvenido al futuro del contenido.'); + break; + case 'fr': + expect(json.data.homepage.greeting).to.equal('Bonjour le monde!'); + expect(json.data.homepage.preamble).to.equal('Bienvenue dans le futur du contenu.'); + break; + } + }); + } + }); +}); diff --git a/packages/astro/test/fixtures/content-collection-references/astro.config.mjs b/packages/astro/test/fixtures/content-collection-references/astro.config.mjs new file mode 100644 index 000000000..913ddc876 --- /dev/null +++ b/packages/astro/test/fixtures/content-collection-references/astro.config.mjs @@ -0,0 +1,8 @@ +import { defineConfig } from 'astro/config'; + +// https://astro.build/config +export default defineConfig({ + experimental: { + assets: true + }, +}); diff --git a/packages/astro/test/fixtures/content-collection-references/package.json b/packages/astro/test/fixtures/content-collection-references/package.json new file mode 100644 index 000000000..3476c8bdf --- /dev/null +++ b/packages/astro/test/fixtures/content-collection-references/package.json @@ -0,0 +1,16 @@ +{ + "name": "@example/content-collection-references", + "type": "module", + "version": "0.0.1", + "private": true, + "scripts": { + "dev": "astro dev", + "start": "astro dev", + "build": "astro build", + "preview": "astro preview", + "astro": "astro" + }, + "dependencies": { + "astro": "workspace:*" + } +} diff --git a/packages/astro/test/fixtures/content-collection-references/src/assets/the-future.jpg b/packages/astro/test/fixtures/content-collection-references/src/assets/the-future.jpg new file mode 100644 index 000000000..c47e4ff88 Binary files /dev/null and b/packages/astro/test/fixtures/content-collection-references/src/assets/the-future.jpg differ diff --git a/packages/astro/test/fixtures/content-collection-references/src/content/authors/ben-holmes.yml b/packages/astro/test/fixtures/content-collection-references/src/content/authors/ben-holmes.yml new file mode 100644 index 000000000..54e6743d9 --- /dev/null +++ b/packages/astro/test/fixtures/content-collection-references/src/content/authors/ben-holmes.yml @@ -0,0 +1,2 @@ +name: Ben J Holmes +twitter: https://twitter.com/bholmesdev diff --git a/packages/astro/test/fixtures/content-collection-references/src/content/authors/fred-schott.yml b/packages/astro/test/fixtures/content-collection-references/src/content/authors/fred-schott.yml new file mode 100644 index 000000000..0b51067d9 --- /dev/null +++ b/packages/astro/test/fixtures/content-collection-references/src/content/authors/fred-schott.yml @@ -0,0 +1,2 @@ +name: Fred K Schott +twitter: https://twitter.com/FredKSchott diff --git a/packages/astro/test/fixtures/content-collection-references/src/content/authors/nate-moore.yml b/packages/astro/test/fixtures/content-collection-references/src/content/authors/nate-moore.yml new file mode 100644 index 000000000..953f348a0 --- /dev/null +++ b/packages/astro/test/fixtures/content-collection-references/src/content/authors/nate-moore.yml @@ -0,0 +1,2 @@ +name: Nate Something Moore +twitter: https://twitter.com/n_moore diff --git a/packages/astro/test/fixtures/content-collection-references/src/content/banners/welcome.json b/packages/astro/test/fixtures/content-collection-references/src/content/banners/welcome.json new file mode 100644 index 000000000..c62d06aab --- /dev/null +++ b/packages/astro/test/fixtures/content-collection-references/src/content/banners/welcome.json @@ -0,0 +1,4 @@ +{ + "alt": "Futuristic landscape with chrome buildings and blue skies", + "src": "~/assets/the-future.jpg" +} diff --git a/packages/astro/test/fixtures/content-collection-references/src/content/blog/related-1.md b/packages/astro/test/fixtures/content-collection-references/src/content/blog/related-1.md new file mode 100644 index 000000000..48a738777 --- /dev/null +++ b/packages/astro/test/fixtures/content-collection-references/src/content/blog/related-1.md @@ -0,0 +1,9 @@ +--- +title: Related post 1 +banner: welcome +author: fred-schott +--- + +# Related post 1 + +This is related to the welcome post. diff --git a/packages/astro/test/fixtures/content-collection-references/src/content/blog/related-2.md b/packages/astro/test/fixtures/content-collection-references/src/content/blog/related-2.md new file mode 100644 index 000000000..3299ca1b4 --- /dev/null +++ b/packages/astro/test/fixtures/content-collection-references/src/content/blog/related-2.md @@ -0,0 +1,9 @@ +--- +title: Related post 2 +banner: welcome +author: ben-holmes +--- + +# Related post 2 + +This is related to the welcome post. diff --git a/packages/astro/test/fixtures/content-collection-references/src/content/blog/welcome.md b/packages/astro/test/fixtures/content-collection-references/src/content/blog/welcome.md new file mode 100644 index 000000000..15f834ad1 --- /dev/null +++ b/packages/astro/test/fixtures/content-collection-references/src/content/blog/welcome.md @@ -0,0 +1,12 @@ +--- +title: Welcome to the future of content! +banner: welcome +author: nate-moore +relatedPosts: +- related-1 +- related-2 +--- + +# Welcome to the future! + +This is how content was _always_ meant to be. diff --git a/packages/astro/test/fixtures/content-collection-references/src/content/config.ts b/packages/astro/test/fixtures/content-collection-references/src/content/config.ts new file mode 100644 index 000000000..a314748c5 --- /dev/null +++ b/packages/astro/test/fixtures/content-collection-references/src/content/config.ts @@ -0,0 +1,29 @@ +import { defineCollection, z, reference } from 'astro:content'; + +const banners = defineCollection({ + type: 'data', + schema: ({ image }) => + z.object({ + alt: z.string(), + src: image(), + }), +}); + +const authors = defineCollection({ + type: 'data', + schema: z.object({ + name: z.string(), + twitter: z.string().url(), + }), +}); + +const blog = defineCollection({ + schema: z.object({ + title: z.string(), + banner: reference('banners'), + author: reference('authors'), + relatedPosts: z.array(reference('blog')).optional(), + }), +}); + +export const collections = { blog, authors, banners }; diff --git a/packages/astro/test/fixtures/content-collection-references/src/pages/welcome-data.json.js b/packages/astro/test/fixtures/content-collection-references/src/pages/welcome-data.json.js new file mode 100644 index 000000000..4f529dac6 --- /dev/null +++ b/packages/astro/test/fixtures/content-collection-references/src/pages/welcome-data.json.js @@ -0,0 +1,25 @@ +import { getEntry, getEntries } from 'astro:content'; + +export async function get() { + const welcomePost = await getEntry('blog', 'welcome'); + + if (!welcomePost?.data) { + return { + body: { error: 'blog/welcome did not return `data`.' }, + } + } + + const banner = await getEntry(welcomePost.data.banner); + const author = await getEntry(welcomePost.data.author); + const rawRelatedPosts = await getEntries(welcomePost.data.relatedPosts ?? []); + const relatedPosts = rawRelatedPosts.map(({ render /** filter out render() function */, ...p }) => p); + + return { + body: JSON.stringify({ + welcomePost, + banner, + author, + relatedPosts, + }) + } +} diff --git a/packages/astro/test/fixtures/content-collection-references/src/pages/welcome.astro b/packages/astro/test/fixtures/content-collection-references/src/pages/welcome.astro new file mode 100644 index 000000000..3645d37bb --- /dev/null +++ b/packages/astro/test/fixtures/content-collection-references/src/pages/welcome.astro @@ -0,0 +1,39 @@ +--- +import { Image } from 'astro:assets'; +import { getEntry, getEntries } from 'astro:content'; + +const welcomePost = await getEntry('blog', 'welcome'); + +if (!welcomePost?.data) { + throw new Error('Render - blog/welcome did not return `data`.'); +} + +const author = await getEntry(welcomePost.data.author); +const banner = await getEntry(welcomePost.data.banner); +const relatedPosts = await getEntries(welcomePost.data.relatedPosts ?? []); +--- + + + + + + + + Astro + + + + {author.data.name} + +

Related posts

+ + + diff --git a/packages/astro/test/fixtures/content-mixed-errors/astro.config.mjs b/packages/astro/test/fixtures/content-mixed-errors/astro.config.mjs new file mode 100644 index 000000000..882e6515a --- /dev/null +++ b/packages/astro/test/fixtures/content-mixed-errors/astro.config.mjs @@ -0,0 +1,4 @@ +import { defineConfig } from 'astro/config'; + +// https://astro.build/config +export default defineConfig({}); diff --git a/packages/astro/test/fixtures/content-mixed-errors/package.json b/packages/astro/test/fixtures/content-mixed-errors/package.json new file mode 100644 index 000000000..d90bfabda --- /dev/null +++ b/packages/astro/test/fixtures/content-mixed-errors/package.json @@ -0,0 +1,16 @@ +{ + "name": "@test/content-mixed-errors", + "type": "module", + "version": "0.0.1", + "private": true, + "scripts": { + "dev": "astro dev", + "start": "astro dev", + "build": "astro build", + "preview": "astro preview", + "astro": "astro" + }, + "dependencies": { + "astro": "workspace:*" + } +} diff --git a/packages/astro/test/fixtures/content-mixed-errors/src/content/authors/placeholder.json b/packages/astro/test/fixtures/content-mixed-errors/src/content/authors/placeholder.json new file mode 100644 index 000000000..64ae1c04c --- /dev/null +++ b/packages/astro/test/fixtures/content-mixed-errors/src/content/authors/placeholder.json @@ -0,0 +1,3 @@ +{ + "name": "Placeholder" +} diff --git a/packages/astro/test/fixtures/content-mixed-errors/src/content/blog/placeholder.md b/packages/astro/test/fixtures/content-mixed-errors/src/content/blog/placeholder.md new file mode 100644 index 000000000..f7f65691b --- /dev/null +++ b/packages/astro/test/fixtures/content-mixed-errors/src/content/blog/placeholder.md @@ -0,0 +1,3 @@ +--- +title: Placeholder post +--- diff --git a/packages/astro/test/fixtures/content-mixed-errors/src/pages/authors.astro b/packages/astro/test/fixtures/content-mixed-errors/src/pages/authors.astro new file mode 100644 index 000000000..8352a3d27 --- /dev/null +++ b/packages/astro/test/fixtures/content-mixed-errors/src/pages/authors.astro @@ -0,0 +1,10 @@ +--- +import { getCollection } from 'astro:content'; +try { + await getCollection('authors') +} catch (e) { + return e +} +--- + +

Worked

diff --git a/packages/astro/test/fixtures/content-mixed-errors/src/pages/blog.astro b/packages/astro/test/fixtures/content-mixed-errors/src/pages/blog.astro new file mode 100644 index 000000000..0d5d2836e --- /dev/null +++ b/packages/astro/test/fixtures/content-mixed-errors/src/pages/blog.astro @@ -0,0 +1,7 @@ +--- +import { getCollection } from 'astro:content'; + +await getCollection('blog') +--- + +

Worked

diff --git a/packages/astro/test/fixtures/data-collections/astro.config.mjs b/packages/astro/test/fixtures/data-collections/astro.config.mjs new file mode 100644 index 000000000..882e6515a --- /dev/null +++ b/packages/astro/test/fixtures/data-collections/astro.config.mjs @@ -0,0 +1,4 @@ +import { defineConfig } from 'astro/config'; + +// https://astro.build/config +export default defineConfig({}); diff --git a/packages/astro/test/fixtures/data-collections/package.json b/packages/astro/test/fixtures/data-collections/package.json new file mode 100644 index 000000000..711eb4956 --- /dev/null +++ b/packages/astro/test/fixtures/data-collections/package.json @@ -0,0 +1,16 @@ +{ + "name": "@test/data-collections", + "type": "module", + "version": "0.0.1", + "private": true, + "scripts": { + "dev": "astro dev", + "start": "astro dev", + "build": "astro build", + "preview": "astro preview", + "astro": "astro" + }, + "dependencies": { + "astro": "workspace:*" + } +} diff --git a/packages/astro/test/fixtures/data-collections/src/content/authors-without-config/Ben Holmes.yml b/packages/astro/test/fixtures/data-collections/src/content/authors-without-config/Ben Holmes.yml new file mode 100644 index 000000000..54e6743d9 --- /dev/null +++ b/packages/astro/test/fixtures/data-collections/src/content/authors-without-config/Ben Holmes.yml @@ -0,0 +1,2 @@ +name: Ben J Holmes +twitter: https://twitter.com/bholmesdev diff --git a/packages/astro/test/fixtures/data-collections/src/content/authors-without-config/Fred K Schott.yml b/packages/astro/test/fixtures/data-collections/src/content/authors-without-config/Fred K Schott.yml new file mode 100644 index 000000000..0b51067d9 --- /dev/null +++ b/packages/astro/test/fixtures/data-collections/src/content/authors-without-config/Fred K Schott.yml @@ -0,0 +1,2 @@ +name: Fred K Schott +twitter: https://twitter.com/FredKSchott diff --git a/packages/astro/test/fixtures/data-collections/src/content/authors-without-config/Nate Moore.yml b/packages/astro/test/fixtures/data-collections/src/content/authors-without-config/Nate Moore.yml new file mode 100644 index 000000000..953f348a0 --- /dev/null +++ b/packages/astro/test/fixtures/data-collections/src/content/authors-without-config/Nate Moore.yml @@ -0,0 +1,2 @@ +name: Nate Something Moore +twitter: https://twitter.com/n_moore diff --git a/packages/astro/test/fixtures/data-collections/src/content/config.ts b/packages/astro/test/fixtures/data-collections/src/content/config.ts new file mode 100644 index 000000000..5f3de9423 --- /dev/null +++ b/packages/astro/test/fixtures/data-collections/src/content/config.ts @@ -0,0 +1,20 @@ +import { defineCollection, z } from 'astro:content'; + +const docs = defineCollection({ + type: 'content', + schema: z.object({ + title: z.string(), + }) +}); + +const i18n = defineCollection({ + type: 'data', + schema: z.object({ + homepage: z.object({ + greeting: z.string(), + preamble: z.string(), + }) + }), +}); + +export const collections = { docs, i18n }; diff --git a/packages/astro/test/fixtures/data-collections/src/content/docs/example.md b/packages/astro/test/fixtures/data-collections/src/content/docs/example.md new file mode 100644 index 000000000..356e65f64 --- /dev/null +++ b/packages/astro/test/fixtures/data-collections/src/content/docs/example.md @@ -0,0 +1,3 @@ +--- +title: The future of content +--- diff --git a/packages/astro/test/fixtures/data-collections/src/content/i18n/en.json b/packages/astro/test/fixtures/data-collections/src/content/i18n/en.json new file mode 100644 index 000000000..51d127f4a --- /dev/null +++ b/packages/astro/test/fixtures/data-collections/src/content/i18n/en.json @@ -0,0 +1,6 @@ +{ + "homepage": { + "greeting": "Hello World!", + "preamble": "Welcome to the future of content." + } +} diff --git a/packages/astro/test/fixtures/data-collections/src/content/i18n/es.json b/packages/astro/test/fixtures/data-collections/src/content/i18n/es.json new file mode 100644 index 000000000..bf4c7af0f --- /dev/null +++ b/packages/astro/test/fixtures/data-collections/src/content/i18n/es.json @@ -0,0 +1,6 @@ +{ + "homepage": { + "greeting": "¡Hola Mundo!", + "preamble": "Bienvenido al futuro del contenido." + } +} diff --git a/packages/astro/test/fixtures/data-collections/src/content/i18n/fr.yaml b/packages/astro/test/fixtures/data-collections/src/content/i18n/fr.yaml new file mode 100644 index 000000000..90a86d411 --- /dev/null +++ b/packages/astro/test/fixtures/data-collections/src/content/i18n/fr.yaml @@ -0,0 +1,3 @@ +homepage: + greeting: "Bonjour le monde!" + preamble: "Bienvenue dans le futur du contenu." diff --git a/packages/astro/test/fixtures/data-collections/src/pages/authors/[id].json.js b/packages/astro/test/fixtures/data-collections/src/pages/authors/[id].json.js new file mode 100644 index 000000000..1cc26fb73 --- /dev/null +++ b/packages/astro/test/fixtures/data-collections/src/pages/authors/[id].json.js @@ -0,0 +1,22 @@ +import { getEntry } from 'astro:content'; + +const ids = ['Ben Holmes', 'Fred K Schott', 'Nate Moore'] + +export function getStaticPaths() { + return ids.map(id => ({ params: { id } })) +} + +/** @param {import('astro').APIContext} params */ +export async function get({ params }) { + const { id } = params; + const author = await getEntry('authors-without-config', id); + if (!author) { + return { + body: JSON.stringify({ error: `Author ${id} Not found` }), + } + } else { + return { + body: JSON.stringify(author), + } + } +} diff --git a/packages/astro/test/fixtures/data-collections/src/pages/authors/all.json.js b/packages/astro/test/fixtures/data-collections/src/pages/authors/all.json.js new file mode 100644 index 000000000..e4c804064 --- /dev/null +++ b/packages/astro/test/fixtures/data-collections/src/pages/authors/all.json.js @@ -0,0 +1,9 @@ +import { getCollection } from 'astro:content'; + +export async function get() { + const authors = await getCollection('authors-without-config'); + + return { + body: JSON.stringify(authors), + } +} diff --git a/packages/astro/test/fixtures/data-collections/src/pages/translations/[lang].json.js b/packages/astro/test/fixtures/data-collections/src/pages/translations/[lang].json.js new file mode 100644 index 000000000..73c90354d --- /dev/null +++ b/packages/astro/test/fixtures/data-collections/src/pages/translations/[lang].json.js @@ -0,0 +1,22 @@ +import { getEntry } from 'astro:content'; + +const langs = ['en', 'es', 'fr'] + +export function getStaticPaths() { + return langs.map(lang => ({ params: { lang } })) +} + +/** @param {import('astro').APIContext} params */ +export async function get({ params }) { + const { lang } = params; + const translations = await getEntry('i18n', lang); + if (!translations) { + return { + body: JSON.stringify({ error: `Translation ${lang} Not found` }), + } + } else { + return { + body: JSON.stringify(translations), + } + } +} diff --git a/packages/astro/test/fixtures/data-collections/src/pages/translations/all.json.js b/packages/astro/test/fixtures/data-collections/src/pages/translations/all.json.js new file mode 100644 index 000000000..7d953838f --- /dev/null +++ b/packages/astro/test/fixtures/data-collections/src/pages/translations/all.json.js @@ -0,0 +1,9 @@ +import { getCollection } from 'astro:content'; + +export async function get() { + const translations = await getCollection('i18n'); + + return { + body: JSON.stringify(translations), + } +} diff --git a/packages/astro/test/units/content-collections/get-entry-info.test.js b/packages/astro/test/units/content-collections/get-entry-info.test.js index 9f413bbee..385a91578 100644 --- a/packages/astro/test/units/content-collections/get-entry-info.test.js +++ b/packages/astro/test/units/content-collections/get-entry-info.test.js @@ -1,44 +1,46 @@ -import { getEntryInfo } from '../../../dist/content/utils.js'; +import { getContentEntryIdAndSlug, getEntryCollectionName } from '../../../dist/content/utils.js'; import { expect } from 'chai'; -describe('Content Collections - getEntryInfo', () => { +describe('Content Collections - entry info', () => { const contentDir = new URL('src/content/', import.meta.url); - it('Returns correct entry info', () => { + it('Returns correct collection name', () => { const entry = new URL('blog/first-post.md', contentDir); - const info = getEntryInfo({ entry, contentDir }); + const collection = getEntryCollectionName({ entry, contentDir }); + expect(collection).to.equal('blog'); + }); + + it('Detects when entry is outside of a collection', () => { + const entry = new URL('base-post.md', contentDir); + const collection = getEntryCollectionName({ entry, contentDir }); + expect(collection).to.be.undefined; + }); + + it('Returns correct collection when nested directories used', () => { + const entry = new URL('docs/2021/01/01/index.md', contentDir); + const collection = getEntryCollectionName({ entry, contentDir }); + expect(collection).to.equal('docs'); + }); + + it('Returns correct entry info', () => { + const collection = 'blog'; + const entry = new URL(`${collection}/first-post.md`, contentDir); + const info = getContentEntryIdAndSlug({ entry, contentDir, collection }); expect(info.id).to.equal('first-post.md'); expect(info.slug).to.equal('first-post'); - expect(info.collection).to.equal('blog'); }); it('Returns correct slug when spaces used', () => { - const entry = new URL('blog/first post.mdx', contentDir); - const info = getEntryInfo({ entry, contentDir }); + const collection = 'blog'; + const entry = new URL(`${collection}/first post.mdx`, contentDir); + const info = getContentEntryIdAndSlug({ entry, contentDir, collection }); expect(info.slug).to.equal('first-post'); }); it('Returns correct slug when nested directories used', () => { - const entry = new URL('blog/2021/01/01/index.md', contentDir); - const info = getEntryInfo({ entry, contentDir }); + const collection = 'blog'; + const entry = new URL(`${collection}/2021/01/01/index.md`, contentDir); + const info = getContentEntryIdAndSlug({ entry, contentDir, collection }); expect(info.slug).to.equal('2021/01/01'); }); - - it('Returns correct collection when nested directories used', () => { - const entry = new URL('blog/2021/01/01/index.md', contentDir); - const info = getEntryInfo({ entry, contentDir }); - expect(info.collection).to.equal('blog'); - }); - - it('Returns error when outside collection directory', () => { - const entry = new URL('blog.md', contentDir); - expect(getEntryInfo({ entry, contentDir }) instanceof Error).to.equal(true); - }); - - it('Silences error on `allowFilesOutsideCollection`', () => { - const entry = new URL('blog.md', contentDir); - const entryInfo = getEntryInfo({ entry, contentDir, allowFilesOutsideCollection: true }); - expect(entryInfo instanceof Error).to.equal(false); - expect(entryInfo.id).to.equal('blog.md'); - }); }); diff --git a/packages/astro/test/units/content-collections/get-entry-type.test.js b/packages/astro/test/units/content-collections/get-entry-type.test.js index 2aa9580e8..e7effa52f 100644 --- a/packages/astro/test/units/content-collections/get-entry-type.test.js +++ b/packages/astro/test/units/content-collections/get-entry-type.test.js @@ -26,6 +26,7 @@ const fixtures = [ ]; const contentFileExts = ['.md', '.mdx']; +const dataFileExts = ['.yaml', '.yml', '.json']; // TODO: Remove `getEntryType` last parameter once `experimental.assets` is no longer experimental describe('Content Collections - getEntryType', () => { @@ -34,58 +35,70 @@ describe('Content Collections - getEntryType', () => { it('Returns "content" for Markdown files', () => { for (const entryPath of ['blog/first-post.md', 'blog/first-post.mdx']) { const entry = fileURLToPath(new URL(entryPath, contentPaths.contentDir)); - const type = getEntryType(entry, contentPaths, contentFileExts, false); + const type = getEntryType(entry, contentPaths, contentFileExts, dataFileExts, false); expect(type).to.equal('content'); } }); + it('Returns "data" for JSON and YAML files', () => { + for (const entryPath of [ + 'banners/welcome.json', + 'banners/welcome.yaml', + 'banners/welcome.yml', + ]) { + const entry = fileURLToPath(new URL(entryPath, contentPaths.contentDir)); + const type = getEntryType(entry, contentPaths, contentFileExts, dataFileExts, false); + expect(type).to.equal('data'); + } + }); + it('Returns "content" for Markdown files in nested directories', () => { for (const entryPath of ['blog/2021/01/01/index.md', 'blog/2021/01/01/index.mdx']) { const entry = fileURLToPath(new URL(entryPath, contentPaths.contentDir)); - const type = getEntryType(entry, contentPaths, contentFileExts, false); + const type = getEntryType(entry, contentPaths, contentFileExts, dataFileExts, false); expect(type).to.equal('content'); } }); it('Returns "config" for config files', () => { const entry = fileURLToPath(contentPaths.config.url); - const type = getEntryType(entry, contentPaths, contentFileExts, false); + const type = getEntryType(entry, contentPaths, contentFileExts, dataFileExts, false); expect(type).to.equal('config'); }); it('Returns "unsupported" for non-Markdown files', () => { const entry = fileURLToPath(new URL('blog/robots.txt', contentPaths.contentDir)); - const type = getEntryType(entry, contentPaths, contentFileExts, false); + const type = getEntryType(entry, contentPaths, contentFileExts, dataFileExts, false); expect(type).to.equal('unsupported'); }); it('Returns "ignored" for .DS_Store', () => { const entry = fileURLToPath(new URL('blog/.DS_Store', contentPaths.contentDir)); - const type = getEntryType(entry, contentPaths, contentFileExts, false); + const type = getEntryType(entry, contentPaths, contentFileExts, dataFileExts, false); expect(type).to.equal('ignored'); }); it('Returns "ignored" for unsupported files using an underscore', () => { const entry = fileURLToPath(new URL('blog/_draft-robots.txt', contentPaths.contentDir)); - const type = getEntryType(entry, contentPaths, contentFileExts, false); + const type = getEntryType(entry, contentPaths, contentFileExts, dataFileExts, false); expect(type).to.equal('ignored'); }); it('Returns "ignored" when using underscore on file name', () => { const entry = fileURLToPath(new URL('blog/_first-post.md', contentPaths.contentDir)); - const type = getEntryType(entry, contentPaths, contentFileExts, false); + const type = getEntryType(entry, contentPaths, contentFileExts, dataFileExts, false); expect(type).to.equal('ignored'); }); it('Returns "ignored" when using underscore on directory name', () => { const entry = fileURLToPath(new URL('blog/_draft/first-post.md', contentPaths.contentDir)); - const type = getEntryType(entry, contentPaths, contentFileExts, false); + const type = getEntryType(entry, contentPaths, contentFileExts, dataFileExts, false); expect(type).to.equal('ignored'); }); it('Returns "ignored" for images', () => { const entry = fileURLToPath(new URL('blog/first-post.png', contentPaths.contentDir)); - const type = getEntryType(entry, contentPaths, contentFileExts, true); + const type = getEntryType(entry, contentPaths, contentFileExts, dataFileExts, true); expect(type).to.equal('ignored'); }); }); diff --git a/packages/astro/test/units/dev/collections-mixed-content-errors.test.js b/packages/astro/test/units/dev/collections-mixed-content-errors.test.js new file mode 100644 index 000000000..355f54a0f --- /dev/null +++ b/packages/astro/test/units/dev/collections-mixed-content-errors.test.js @@ -0,0 +1,122 @@ +import { expect } from 'chai'; +import { createFsWithFallback } from '../test-utils.js'; +import { defaultLogging } from '../../test-utils.js'; +import { validateConfig } from '../../../dist/core/config/config.js'; +import { createSettings } from '../../../dist/core/config/index.js'; +import { fileURLToPath } from 'url'; +import { sync as _sync } from '../../../dist/core/sync/index.js'; + +const root = new URL('../../fixtures/content-mixed-errors/', import.meta.url); +const logging = defaultLogging; + +async function sync({ fs, config = {} }) { + const astroConfig = await validateConfig(config, fileURLToPath(root), 'prod'); + const settings = createSettings(astroConfig, fileURLToPath(root)); + + return _sync(settings, { logging, fs }); +} + +describe('Content Collections - mixed content errors', () => { + it('raises "mixed content" error when content in data collection', async () => { + const fs = createFsWithFallback( + { + '/src/content/authors/ben.md': `--- +name: Ben +--- + +# Ben`, + '/src/content/authors/tony.json': `{ "name": "Tony" }`, + '/src/content/config.ts': ` + + import { z, defineCollection } from 'astro:content'; + + const authors = defineCollection({ + type: 'data', + schema: z.object({ + name: z.string(), + }), + }); + + export const collections = { authors };`, + }, + root + ); + + try { + await sync({ fs }); + expect.fail(0, 1, 'Expected sync to throw'); + } catch (e) { + expect(e).to.be.instanceOf(Error); + expect(e.type).to.equal('AstroError'); + expect(e.errorCode).to.equal(9005); + expect(e.message).to.include('authors'); + } + }); + + it('raises "mixed content" error when data in content collection', async () => { + const fs = createFsWithFallback( + { + '/src/content/blog/post.md': `--- +title: Post +--- + +# Post`, + '/src/content/blog/post.yaml': `title: YAML Post`, + '/src/content/config.ts': ` + + import { z, defineCollection } from 'astro:content'; + + const blog = defineCollection({ + type: 'content', + schema: z.object({ + title: z.string(), + }), + }); + + export const collections = { blog };`, + }, + root + ); + + try { + await sync({ fs }); + expect.fail(0, 1, 'Expected sync to throw'); + } catch (e) { + expect(e).to.be.instanceOf(Error); + expect(e.type).to.equal('AstroError'); + expect(e.errorCode).to.equal(9005); + expect(e.message).to.include('blog'); + } + }); + + it('raises error when data collection configured as content collection', async () => { + const fs = createFsWithFallback( + { + '/src/content/banners/welcome.json': `{ "src": "/example", "alt": "Welcome" }`, + '/src/content/config.ts': ` + + import { z, defineCollection } from 'astro:content'; + + const banners = defineCollection({ + schema: z.object({ + src: z.string(), + alt: z.string(), + }), + }); + + export const collections = { banners };`, + }, + root + ); + + try { + await sync({ fs }); + expect.fail(0, 1, 'Expected sync to throw'); + } catch (e) { + expect(e).to.be.instanceOf(Error); + expect(e.type).to.equal('AstroError'); + expect(e.errorCode).to.equal(9006); + expect(e.hint).to.include("Try adding `type: 'data'`"); + } + }); +}); diff --git a/packages/integrations/markdoc/src/index.ts b/packages/integrations/markdoc/src/index.ts index 0ae1f1fad..4360800a0 100644 --- a/packages/integrations/markdoc/src/index.ts +++ b/packages/integrations/markdoc/src/index.ts @@ -91,7 +91,7 @@ export default function markdocIntegration(legacyConfig?: any): AstroIntegration const res = `import { jsx as h } from 'astro/jsx-runtime'; import { Renderer } from '@astrojs/markdoc/components'; import { collectHeadings, applyDefaultConfig, Markdoc, headingSlugger } from '@astrojs/markdoc/runtime'; -import * as entry from ${JSON.stringify(viteId + '?astroContent')}; +import * as entry from ${JSON.stringify(viteId + '?astroContentCollectionEntry')}; ${ markdocConfigResult ? `import _userConfig from ${JSON.stringify( diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6d12a7553..224a78ee6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -624,6 +624,9 @@ importers: html-escaper: specifier: ^3.0.3 version: 3.0.3 + js-yaml: + specifier: ^4.1.0 + version: 4.1.0 kleur: specifier: ^4.1.4 version: 4.1.5 @@ -730,6 +733,9 @@ importers: '@types/html-escaper': specifier: ^3.0.0 version: 3.0.0 + '@types/js-yaml': + specifier: ^4.0.5 + version: 4.0.5 '@types/mime': specifier: ^2.0.3 version: 2.0.3 @@ -2352,6 +2358,12 @@ importers: specifier: workspace:* version: link:../../.. + packages/astro/test/fixtures/content-collection-references: + dependencies: + astro: + specifier: workspace:* + version: link:../../.. + packages/astro/test/fixtures/content-collections: dependencies: '@astrojs/mdx': @@ -2382,6 +2394,12 @@ importers: specifier: workspace:* version: link:../../.. + packages/astro/test/fixtures/content-mixed-errors: + dependencies: + astro: + specifier: workspace:* + version: link:../../.. + packages/astro/test/fixtures/content-ssr-integration: dependencies: '@astrojs/mdx': @@ -2548,6 +2566,12 @@ importers: packages/astro/test/fixtures/custom-elements/my-component-lib: {} + packages/astro/test/fixtures/data-collections: + dependencies: + astro: + specifier: workspace:* + version: link:../../.. + packages/astro/test/fixtures/debug-component: dependencies: astro: @@ -8850,6 +8874,10 @@ packages: ci-info: 3.3.1 dev: true + /@types/js-yaml@4.0.5: + resolution: {integrity: sha512-FhpRzf927MNQdRZP0J5DLIdTXhjLYzeUTmLAu69mnVksLH9CJY3IuSeEgbKUki7GQZm0WqDkGzyxju2EZGD2wA==} + dev: true + /@types/json-schema@7.0.11: resolution: {integrity: sha512-wOuvG1SN4Us4rez+tylwwwCV1psiNVOkJeM3AUWUNWg/jDQY2+HE/444y5gc+jBmRqASOm2Oeh5c1axHobwRKQ==} dev: true