From 65846f415169a43b6bb42ce56e02ab0435e61b30 Mon Sep 17 00:00:00 2001 From: bholmesdev Date: Tue, 7 Feb 2023 14:00:23 -0500 Subject: [PATCH] wip: get dream API for file loader working --- .../with-markdoc/src/content/blog/test.mdoc | 5 ++ examples/with-markdoc/src/pages/index.astro | 4 ++ packages/astro/src/content/consts.ts | 3 +- packages/astro/src/content/types-generator.ts | 11 +++-- packages/astro/src/content/utils.ts | 32 ++++++------- .../src/content/vite-plugin-content-assets.ts | 4 +- .../content/vite-plugin-content-imports.ts | 48 +++++++++++++------ .../vite-plugin-content-virtual-mod.ts | 8 +++- packages/astro/src/content/~dream.ts | 44 +++++++++++++++++ 9 files changed, 122 insertions(+), 37 deletions(-) create mode 100644 examples/with-markdoc/src/content/blog/test.mdoc create mode 100644 packages/astro/src/content/~dream.ts diff --git a/examples/with-markdoc/src/content/blog/test.mdoc b/examples/with-markdoc/src/content/blog/test.mdoc new file mode 100644 index 000000000..2b9868202 --- /dev/null +++ b/examples/with-markdoc/src/content/blog/test.mdoc @@ -0,0 +1,5 @@ +--- +title: Example! +--- + +# Hey there diff --git a/examples/with-markdoc/src/pages/index.astro b/examples/with-markdoc/src/pages/index.astro index 6b423317c..10fbfedb5 100644 --- a/examples/with-markdoc/src/pages/index.astro +++ b/examples/with-markdoc/src/pages/index.astro @@ -5,6 +5,10 @@ import RenderMarkdoc from '../renderer/RenderMarkdoc.astro'; import { getTransformed } from '../components/test.mdoc'; import { Code } from 'astro/components'; import Marquee from '../components/Marquee.astro'; +import { getEntryBySlug } from 'astro:content'; + +const mdocEntry = await getEntryBySlug('blog', 'test'); +console.log(mdocEntry); --- diff --git a/packages/astro/src/content/consts.ts b/packages/astro/src/content/consts.ts index 9966e7121..9b052ff8a 100644 --- a/packages/astro/src/content/consts.ts +++ b/packages/astro/src/content/consts.ts @@ -1,4 +1,5 @@ -export const contentFileExts = ['.md', '.mdx']; +/** TODO as const*/ +export const defaultContentFileExts = ['.md', '.mdx']; export const PROPAGATED_ASSET_FLAG = 'astroPropagatedAssets'; export const CONTENT_FLAG = 'astroContent'; export const VIRTUAL_MODULE_ID = 'astro:content'; diff --git a/packages/astro/src/content/types-generator.ts b/packages/astro/src/content/types-generator.ts index 8d990d586..1d43df77a 100644 --- a/packages/astro/src/content/types-generator.ts +++ b/packages/astro/src/content/types-generator.ts @@ -8,7 +8,7 @@ import type { AstroSettings } from '../@types/astro.js'; import { AstroError, AstroErrorData } from '../core/errors/index.js'; import { info, LogOptions, warn } from '../core/logger/core.js'; import { isRelativePath } from '../core/path.js'; -import { CONTENT_TYPES_FILE } from './consts.js'; +import { CONTENT_TYPES_FILE, defaultContentFileExts } from './consts.js'; import { ContentConfig, ContentObservable, @@ -22,6 +22,7 @@ import { NoCollectionError, parseFrontmatter, } from './utils.js'; +import { contentEntryTypes } from './~dream.js'; type ChokidarEvent = 'add' | 'addDir' | 'change' | 'unlink' | 'unlinkDir'; type RawContentEvent = { name: ChokidarEvent; entry: string }; @@ -57,6 +58,10 @@ export async function createContentTypesGenerator({ }: CreateContentGeneratorParams) { const contentTypes: ContentTypes = {}; const contentPaths = getContentPaths(settings.config, fs); + const contentFileExts = [ + ...defaultContentFileExts, + ...contentEntryTypes.map((t) => t.extensions).flat(), + ]; let events: EventWithOptions[] = []; let debounceTimeout: NodeJS.Timeout | undefined; @@ -121,7 +126,7 @@ export async function createContentTypesGenerator({ } return { shouldGenerateTypes: true }; } - const fileType = getEntryType(fileURLToPath(event.entry), contentPaths); + const fileType = getEntryType(fileURLToPath(event.entry), contentPaths, contentFileExts); if (fileType === 'ignored') { return { shouldGenerateTypes: false }; } @@ -300,7 +305,7 @@ async function parseSlug({ // on dev server startup or production build init. const rawContents = await fs.promises.readFile(event.entry, 'utf-8'); const { data: frontmatter } = parseFrontmatter(rawContents, fileURLToPath(event.entry)); - return getEntrySlug({ ...entryInfo, data: frontmatter }); + return getEntrySlug({ ...entryInfo, unvalidatedSlug: frontmatter.slug }); } function setEntry( diff --git a/packages/astro/src/content/utils.ts b/packages/astro/src/content/utils.ts index afe15eeb0..8798ec5b9 100644 --- a/packages/astro/src/content/utils.ts +++ b/packages/astro/src/content/utils.ts @@ -7,6 +7,7 @@ import { ErrorPayload as ViteErrorPayload, normalizePath, ViteDevServer } from ' import { z } from 'zod'; import { AstroConfig, AstroSettings } from '../@types/astro.js'; import { AstroError, AstroErrorData } from '../core/errors/index.js'; +import { appendForwardSlash } from '../core/path.js'; import { contentFileExts, CONTENT_TYPES_FILE } from './consts.js'; export const collectionConfigParser = z.object({ @@ -29,14 +30,7 @@ export const contentConfigParser = z.object({ export type CollectionConfig = z.infer; export type ContentConfig = z.infer; -type Entry = { - id: string; - collection: string; - slug: string; - data: any; - body: string; - _internal: { rawData: string; filePath: string }; -}; +type EntryInternal = { rawData: string; filePath: string }; export type EntryInfo = { id: string; @@ -53,10 +47,10 @@ export function getEntrySlug({ id, collection, slug, - data: unparsedData, -}: Pick) { + unvalidatedSlug, +}: EntryInfo & { unvalidatedSlug?: unknown }) { try { - return z.string().default(slug).parse(unparsedData.slug); + return z.string().default(slug).parse(unvalidatedSlug); } catch { throw new AstroError({ ...AstroErrorData.InvalidContentEntrySlugError, @@ -65,9 +59,12 @@ export function getEntrySlug({ } } -export async function getEntryData(entry: Entry, collectionConfig: CollectionConfig) { +export async function getEntryData( + entry: EntryInfo & { unvalidatedData: Record; _internal: EntryInternal }, + collectionConfig: CollectionConfig +) { // Remove reserved `slug` field before parsing data - let { slug, ...data } = entry.data; + let { slug, ...data } = entry.unvalidatedData; if (collectionConfig.schema) { // TODO: remove for 2.0 stable release if ( @@ -94,7 +91,9 @@ export async function getEntryData(entry: Entry, collectionConfig: CollectionCon }); } // Use `safeParseAsync` to allow async transforms - const parsed = await collectionConfig.schema.safeParseAsync(entry.data, { errorMap }); + const parsed = await collectionConfig.schema.safeParseAsync(entry.unvalidatedData, { + errorMap, + }); if (parsed.success) { data = parsed.data; } else { @@ -160,14 +159,15 @@ export function getEntryInfo({ export function getEntryType( entryPath: string, - paths: Pick + paths: Pick, + contentFileExts: string[] ): 'content' | 'config' | 'ignored' | 'unsupported' { const { ext, base } = path.parse(entryPath); const fileUrl = pathToFileURL(entryPath); if (hasUnderscoreBelowContentDirectoryPath(fileUrl, paths.contentDir) || isOnIgnoreList(base)) { return 'ignored'; - } else if ((contentFileExts as readonly string[]).includes(ext)) { + } else if (contentFileExts.includes(ext)) { return 'content'; } else if (fileUrl.href === paths.config.url.href) { return 'config'; diff --git a/packages/astro/src/content/vite-plugin-content-assets.ts b/packages/astro/src/content/vite-plugin-content-assets.ts index 3a48edfc1..8021258b9 100644 --- a/packages/astro/src/content/vite-plugin-content-assets.ts +++ b/packages/astro/src/content/vite-plugin-content-assets.ts @@ -11,7 +11,7 @@ import { prependForwardSlash } from '../core/path.js'; import { getStylesForURL } from '../core/render/dev/css.js'; import { getScriptsForURL } from '../core/render/dev/scripts.js'; import { - contentFileExts, + defaultContentFileExts, LINKS_PLACEHOLDER, PROPAGATED_ASSET_FLAG, SCRIPTS_PLACEHOLDER, @@ -22,7 +22,7 @@ function isPropagatedAsset(viteId: string): boolean { const url = new URL(viteId, 'file://'); return ( url.searchParams.has(PROPAGATED_ASSET_FLAG) && - contentFileExts.some((ext) => url.pathname.endsWith(ext)) + defaultContentFileExts.some((ext) => url.pathname.endsWith(ext)) ); } diff --git a/packages/astro/src/content/vite-plugin-content-imports.ts b/packages/astro/src/content/vite-plugin-content-imports.ts index d8075a1a1..ddbfecffd 100644 --- a/packages/astro/src/content/vite-plugin-content-imports.ts +++ b/packages/astro/src/content/vite-plugin-content-imports.ts @@ -1,3 +1,4 @@ +import { contentEntryTypes } from './~dream.js'; import * as devalue from 'devalue'; import type fsMod from 'node:fs'; import { pathToFileURL } from 'url'; @@ -6,7 +7,7 @@ import { AstroSettings } from '../@types/astro.js'; import { AstroErrorData } from '../core/errors/errors-data.js'; import { AstroError } from '../core/errors/errors.js'; import { escapeViteEnvReferences, getFileInfo } from '../vite-plugin-utils/index.js'; -import { contentFileExts, CONTENT_FLAG } from './consts.js'; +import { defaultContentFileExts, CONTENT_FLAG } from './consts.js'; import { ContentConfig, getContentPaths, @@ -19,8 +20,8 @@ import { } from './utils.js'; function isContentFlagImport(viteId: string) { - const { pathname, searchParams } = new URL(viteId, 'file://'); - return searchParams.has(CONTENT_FLAG) && contentFileExts.some((ext) => pathname.endsWith(ext)); + const { searchParams } = new URL(viteId, 'file://'); + return searchParams.has(CONTENT_FLAG); } export function astroContentImportPlugin({ @@ -31,6 +32,10 @@ export function astroContentImportPlugin({ settings: AstroSettings; }): Plugin { const contentPaths = getContentPaths(settings.config, fs); + const contentFileExts = [ + ...defaultContentFileExts, + ...contentEntryTypes.map((t) => t.extensions).flat(), + ]; return { name: 'astro:content-imports', @@ -69,11 +74,27 @@ export function astroContentImportPlugin({ }); } const rawContents = await fs.promises.readFile(fileId, 'utf-8'); - const { - content: body, - data: unparsedData, - matter: rawData = '', - } = parseFrontmatter(rawContents, fileId); + const contentEntryType = contentEntryTypes.find((entryType) => + entryType.extensions.some((ext) => fileId.endsWith(ext)) + ); + let body: string, + unvalidatedData: Record, + unvalidatedSlug: string, + rawData: string; + if (contentEntryType) { + const info = await contentEntryType.getEntryInfo({ fileUrl: pathToFileURL(fileId) }); + body = info.body; + unvalidatedData = info.data; + unvalidatedSlug = info.slug; + rawData = info.rawData; + } else { + const parsed = parseFrontmatter(rawContents, fileId); + body = parsed.content; + unvalidatedData = parsed.data; + unvalidatedSlug = parsed.data.slug; + rawData = parsed.matter; + } + const entryInfo = getEntryInfo({ entry: pathToFileURL(fileId), contentDir: contentPaths.contentDir, @@ -81,15 +102,14 @@ export function astroContentImportPlugin({ if (entryInfo instanceof Error) return; const _internal = { filePath: fileId, rawData }; - const partialEntry = { data: unparsedData, body, _internal, ...entryInfo }; // TODO: move slug calculation to the start of the build // to generate a performant lookup map for `getEntryBySlug` - const slug = getEntrySlug(partialEntry); + const slug = getEntrySlug({ ...entryInfo, unvalidatedSlug }); const collectionConfig = contentConfig?.collections[entryInfo.collection]; const data = collectionConfig - ? await getEntryData(partialEntry, collectionConfig) - : unparsedData; + ? await getEntryData({ ...entryInfo, _internal, unvalidatedData }, collectionConfig) + : unvalidatedData; const code = escapeViteEnvReferences(` export const id = ${JSON.stringify(entryInfo.id)}; @@ -99,7 +119,7 @@ export const body = ${JSON.stringify(body)}; export const data = ${devalue.uneval(data) /* TODO: reuse astro props serializer */}; export const _internal = { filePath: ${JSON.stringify(fileId)}, - rawData: ${JSON.stringify(rawData)}, + rawData: ${JSON.stringify(unvalidatedData)}, }; `); return { code }; @@ -109,7 +129,7 @@ export const _internal = { viteServer.watcher.on('all', async (event, entry) => { if ( ['add', 'unlink', 'change'].includes(event) && - getEntryType(entry, contentPaths) === 'config' + getEntryType(entry, contentPaths, contentFileExts) === 'config' ) { // Content modules depend on config, so we need to invalidate them. for (const modUrl of viteServer.moduleGraph.urlToModuleMap.keys()) { diff --git a/packages/astro/src/content/vite-plugin-content-virtual-mod.ts b/packages/astro/src/content/vite-plugin-content-virtual-mod.ts index 120083ebf..f09821449 100644 --- a/packages/astro/src/content/vite-plugin-content-virtual-mod.ts +++ b/packages/astro/src/content/vite-plugin-content-virtual-mod.ts @@ -4,8 +4,9 @@ import type { Plugin } from 'vite'; import { normalizePath } from 'vite'; import type { AstroSettings } from '../@types/astro.js'; import { appendForwardSlash, prependForwardSlash } from '../core/path.js'; -import { contentFileExts, VIRTUAL_MODULE_ID } from './consts.js'; +import { defaultContentFileExts, VIRTUAL_MODULE_ID } from './consts.js'; import { getContentPaths } from './utils.js'; +import { contentEntryTypes } from './~dream.js'; interface AstroContentVirtualModPluginParams { settings: AstroSettings; @@ -22,6 +23,11 @@ export function astroContentVirtualModPlugin({ ) ) ); + const contentFileExts = [ + ...defaultContentFileExts, + ...contentEntryTypes.map((t) => t.extensions).flat(), + ]; + const entryGlob = `${relContentDir}**/*{${contentFileExts.join(',')}}`; const virtualModContents = fsMod .readFileSync(contentPaths.virtualModTemplate, 'utf-8') diff --git a/packages/astro/src/content/~dream.ts b/packages/astro/src/content/~dream.ts new file mode 100644 index 000000000..0223d54e3 --- /dev/null +++ b/packages/astro/src/content/~dream.ts @@ -0,0 +1,44 @@ +import fs from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { parseFrontmatter } from './utils.js'; + +// Register things for typesafety +declare module 'astro:content' { + interface FancyRender { + '.mdoc': { + getParsed(): string; + getTransformed(): Promise; + }; + } +} + +type ContentEntryType = { + extensions: string[]; + getEntryInfo(params: { fileUrl: URL }): Promise<{ + data: Record; + /** + * Used for error hints to point to correct line and location + * Should be the untouched data as read from the file, + * including newlines + */ + rawData: string; + body: string; + slug: string; + }>; +}; + +export const contentEntryTypes: ContentEntryType[] = [ + { + extensions: ['.mdoc'], + async getEntryInfo({ fileUrl }) { + const rawContents = await fs.promises.readFile(fileUrl, 'utf-8'); + const parsed = parseFrontmatter(rawContents, fileURLToPath(fileUrl)); + return { + data: parsed.data, + body: parsed.content, + slug: parsed.data.slug, + rawData: parsed.matter, + }; + }, + }, +];