michael/astro
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

wip: get dream API for file loader working

Browse source
This commit is contained in:
bholmesdev 2023-02-07 14:00:23 -05:00
parent 926c82d8a5
commit 65846f4151
9 changed files with 122 additions and 37 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
examples/with-markdoc/src/content/blog/test.mdoc Normal file
View file

@ -0,0 +1,5 @@
---
title: Example!
---
# Hey there

4
examples/with-markdoc/src/pages/index.astro
View file

@ -5,6 +5,10 @@ import RenderMarkdoc from '../renderer/RenderMarkdoc.astro';
import { getTransformed } from '../components/test.mdoc'; import { getTransformed } from '../components/test.mdoc';
import { Code } from 'astro/components'; import { Code } from 'astro/components';
import Marquee from '../components/Marquee.astro'; import Marquee from '../components/Marquee.astro';
import { getEntryBySlug } from 'astro:content';
const mdocEntry = await getEntryBySlug('blog', 'test');
console.log(mdocEntry);
--- ---
<html lang="en"> <html lang="en">

3
packages/astro/src/content/consts.ts
View file

@ -1,4 +1,5 @@
export const contentFileExts = ['.md', '.mdx']; /** TODO as const*/
export const defaultContentFileExts = ['.md', '.mdx'];
export const PROPAGATED_ASSET_FLAG = 'astroPropagatedAssets'; export const PROPAGATED_ASSET_FLAG = 'astroPropagatedAssets';
export const CONTENT_FLAG = 'astroContent'; export const CONTENT_FLAG = 'astroContent';
export const VIRTUAL_MODULE_ID = 'astro:content'; export const VIRTUAL_MODULE_ID = 'astro:content';

11
packages/astro/src/content/types-generator.ts
View file

@ -8,7 +8,7 @@ import type { AstroSettings } from '../@types/astro.js';
import { AstroError, AstroErrorData } from '../core/errors/index.js'; import { AstroError, AstroErrorData } from '../core/errors/index.js';
import { info, LogOptions, warn } from '../core/logger/core.js'; import { info, LogOptions, warn } from '../core/logger/core.js';
import { isRelativePath } from '../core/path.js'; import { isRelativePath } from '../core/path.js';
import { CONTENT_TYPES_FILE } from './consts.js'; import { CONTENT_TYPES_FILE, defaultContentFileExts } from './consts.js';
import { import {
ContentConfig, ContentConfig,
ContentObservable, ContentObservable,
@ -22,6 +22,7 @@ import {
NoCollectionError, NoCollectionError,
parseFrontmatter, parseFrontmatter,
} from './utils.js'; } from './utils.js';
import { contentEntryTypes } from './~dream.js';
type ChokidarEvent = 'add' | 'addDir' | 'change' | 'unlink' | 'unlinkDir'; type ChokidarEvent = 'add' | 'addDir' | 'change' | 'unlink' | 'unlinkDir';
type RawContentEvent = { name: ChokidarEvent; entry: string }; type RawContentEvent = { name: ChokidarEvent; entry: string };
@ -57,6 +58,10 @@ export async function createContentTypesGenerator({
}: CreateContentGeneratorParams) { }: CreateContentGeneratorParams) {
const contentTypes: ContentTypes = {}; const contentTypes: ContentTypes = {};
const contentPaths = getContentPaths(settings.config, fs); const contentPaths = getContentPaths(settings.config, fs);
const contentFileExts = [
...defaultContentFileExts,
...contentEntryTypes.map((t) => t.extensions).flat(),
];
let events: EventWithOptions[] = []; let events: EventWithOptions[] = [];
let debounceTimeout: NodeJS.Timeout | undefined; let debounceTimeout: NodeJS.Timeout | undefined;
@ -121,7 +126,7 @@ export async function createContentTypesGenerator({
} }
return { shouldGenerateTypes: true }; return { shouldGenerateTypes: true };
} }
const fileType = getEntryType(fileURLToPath(event.entry), contentPaths); const fileType = getEntryType(fileURLToPath(event.entry), contentPaths, contentFileExts);
if (fileType === 'ignored') { if (fileType === 'ignored') {
return { shouldGenerateTypes: false }; return { shouldGenerateTypes: false };
} }
@ -300,7 +305,7 @@ async function parseSlug({
// on dev server startup or production build init. // on dev server startup or production build init.
const rawContents = await fs.promises.readFile(event.entry, 'utf-8'); const rawContents = await fs.promises.readFile(event.entry, 'utf-8');
const { data: frontmatter } = parseFrontmatter(rawContents, fileURLToPath(event.entry)); const { data: frontmatter } = parseFrontmatter(rawContents, fileURLToPath(event.entry));
return getEntrySlug({ ...entryInfo, data: frontmatter }); return getEntrySlug({ ...entryInfo, unvalidatedSlug: frontmatter.slug });
} }
function setEntry( function setEntry(

32
packages/astro/src/content/utils.ts
View file

@ -7,6 +7,7 @@ import { ErrorPayload as ViteErrorPayload, normalizePath, ViteDevServer } from '
import { z } from 'zod'; import { z } from 'zod';
import { AstroConfig, AstroSettings } from '../@types/astro.js'; import { AstroConfig, AstroSettings } from '../@types/astro.js';
import { AstroError, AstroErrorData } from '../core/errors/index.js'; import { AstroError, AstroErrorData } from '../core/errors/index.js';
import { appendForwardSlash } from '../core/path.js';
import { contentFileExts, CONTENT_TYPES_FILE } from './consts.js'; import { contentFileExts, CONTENT_TYPES_FILE } from './consts.js';
export const collectionConfigParser = z.object({ export const collectionConfigParser = z.object({
@ -29,14 +30,7 @@ export const contentConfigParser = z.object({
export type CollectionConfig = z.infer<typeof collectionConfigParser>; export type CollectionConfig = z.infer<typeof collectionConfigParser>;
export type ContentConfig = z.infer<typeof contentConfigParser>; export type ContentConfig = z.infer<typeof contentConfigParser>;
type Entry = { type EntryInternal = { rawData: string; filePath: string };
id: string;
collection: string;
slug: string;
data: any;
body: string;
_internal: { rawData: string; filePath: string };
};
export type EntryInfo = { export type EntryInfo = {
id: string; id: string;
@ -53,10 +47,10 @@ export function getEntrySlug({
id, id,
collection, collection,
slug, slug,
data: unparsedData, unvalidatedSlug,
}: Pick<Entry, 'id' | 'collection' | 'slug' | 'data'>) { }: EntryInfo & { unvalidatedSlug?: unknown }) {
try { try {
return z.string().default(slug).parse(unparsedData.slug); return z.string().default(slug).parse(unvalidatedSlug);
} catch { } catch {
throw new AstroError({ throw new AstroError({
...AstroErrorData.InvalidContentEntrySlugError, ...AstroErrorData.InvalidContentEntrySlugError,
@ -65,9 +59,12 @@ export function getEntrySlug({
} }
} }
export async function getEntryData(entry: Entry, collectionConfig: CollectionConfig) { export async function getEntryData(
entry: EntryInfo & { unvalidatedData: Record<string, unknown>; _internal: EntryInternal },
collectionConfig: CollectionConfig
) {
// Remove reserved `slug` field before parsing data // Remove reserved `slug` field before parsing data
let { slug, ...data } = entry.data; let { slug, ...data } = entry.unvalidatedData;
if (collectionConfig.schema) { if (collectionConfig.schema) {
// TODO: remove for 2.0 stable release // TODO: remove for 2.0 stable release
if ( if (
@ -94,7 +91,9 @@ export async function getEntryData(entry: Entry, collectionConfig: CollectionCon
}); });
} }
// Use `safeParseAsync` to allow async transforms // Use `safeParseAsync` to allow async transforms
const parsed = await collectionConfig.schema.safeParseAsync(entry.data, { errorMap }); const parsed = await collectionConfig.schema.safeParseAsync(entry.unvalidatedData, {
errorMap,
});
if (parsed.success) { if (parsed.success) {
data = parsed.data; data = parsed.data;
} else { } else {
@ -160,14 +159,15 @@ export function getEntryInfo({
export function getEntryType( export function getEntryType(
entryPath: string, entryPath: string,
paths: Pick<ContentPaths, 'config' | 'contentDir'> paths: Pick<ContentPaths, 'config' | 'contentDir'>,
contentFileExts: string[]
): 'content' | 'config' | 'ignored' | 'unsupported' { ): 'content' | 'config' | 'ignored' | 'unsupported' {
const { ext, base } = path.parse(entryPath); const { ext, base } = path.parse(entryPath);
const fileUrl = pathToFileURL(entryPath); const fileUrl = pathToFileURL(entryPath);
if (hasUnderscoreBelowContentDirectoryPath(fileUrl, paths.contentDir) || isOnIgnoreList(base)) { if (hasUnderscoreBelowContentDirectoryPath(fileUrl, paths.contentDir) || isOnIgnoreList(base)) {
return 'ignored'; return 'ignored';
} else if ((contentFileExts as readonly string[]).includes(ext)) { } else if (contentFileExts.includes(ext)) {
return 'content'; return 'content';
} else if (fileUrl.href === paths.config.url.href) { } else if (fileUrl.href === paths.config.url.href) {
return 'config'; return 'config';

4
packages/astro/src/content/vite-plugin-content-assets.ts
View file

@ -11,7 +11,7 @@ import { prependForwardSlash } from '../core/path.js';
import { getStylesForURL } from '../core/render/dev/css.js'; import { getStylesForURL } from '../core/render/dev/css.js';
import { getScriptsForURL } from '../core/render/dev/scripts.js'; import { getScriptsForURL } from '../core/render/dev/scripts.js';
import { import {
contentFileExts, defaultContentFileExts,
LINKS_PLACEHOLDER, LINKS_PLACEHOLDER,
PROPAGATED_ASSET_FLAG, PROPAGATED_ASSET_FLAG,
SCRIPTS_PLACEHOLDER, SCRIPTS_PLACEHOLDER,
@ -22,7 +22,7 @@ function isPropagatedAsset(viteId: string): boolean {
const url = new URL(viteId, 'file://'); const url = new URL(viteId, 'file://');
return ( return (
url.searchParams.has(PROPAGATED_ASSET_FLAG) && url.searchParams.has(PROPAGATED_ASSET_FLAG) &&
contentFileExts.some((ext) => url.pathname.endsWith(ext)) defaultContentFileExts.some((ext) => url.pathname.endsWith(ext))
); );
} }

48
packages/astro/src/content/vite-plugin-content-imports.ts
View file

@ -1,3 +1,4 @@
import { contentEntryTypes } from './~dream.js';
import * as devalue from 'devalue'; import * as devalue from 'devalue';
import type fsMod from 'node:fs'; import type fsMod from 'node:fs';
import { pathToFileURL } from 'url'; import { pathToFileURL } from 'url';
@ -6,7 +7,7 @@ import { AstroSettings } from '../@types/astro.js';
import { AstroErrorData } from '../core/errors/errors-data.js'; import { AstroErrorData } from '../core/errors/errors-data.js';
import { AstroError } from '../core/errors/errors.js'; import { AstroError } from '../core/errors/errors.js';
import { escapeViteEnvReferences, getFileInfo } from '../vite-plugin-utils/index.js'; import { escapeViteEnvReferences, getFileInfo } from '../vite-plugin-utils/index.js';
import { contentFileExts, CONTENT_FLAG } from './consts.js'; import { defaultContentFileExts, CONTENT_FLAG } from './consts.js';
import { import {
ContentConfig, ContentConfig,
getContentPaths, getContentPaths,
@ -19,8 +20,8 @@ import {
} from './utils.js'; } from './utils.js';
function isContentFlagImport(viteId: string) { function isContentFlagImport(viteId: string) {
const { pathname, searchParams } = new URL(viteId, 'file://'); const { searchParams } = new URL(viteId, 'file://');
return searchParams.has(CONTENT_FLAG) && contentFileExts.some((ext) => pathname.endsWith(ext)); return searchParams.has(CONTENT_FLAG);
} }
export function astroContentImportPlugin({ export function astroContentImportPlugin({
@ -31,6 +32,10 @@ export function astroContentImportPlugin({
settings: AstroSettings; settings: AstroSettings;
}): Plugin { }): Plugin {
const contentPaths = getContentPaths(settings.config, fs); const contentPaths = getContentPaths(settings.config, fs);
const contentFileExts = [
...defaultContentFileExts,
...contentEntryTypes.map((t) => t.extensions).flat(),
];
return { return {
name: 'astro:content-imports', name: 'astro:content-imports',
@ -69,11 +74,27 @@ export function astroContentImportPlugin({
}); });
} }
const rawContents = await fs.promises.readFile(fileId, 'utf-8'); const rawContents = await fs.promises.readFile(fileId, 'utf-8');
const { const contentEntryType = contentEntryTypes.find((entryType) =>
content: body, entryType.extensions.some((ext) => fileId.endsWith(ext))
data: unparsedData, );
matter: rawData = '', let body: string,
} = parseFrontmatter(rawContents, fileId); unvalidatedData: Record<string, unknown>,
unvalidatedSlug: string,
rawData: string;
if (contentEntryType) {
const info = await contentEntryType.getEntryInfo({ fileUrl: pathToFileURL(fileId) });
body = info.body;
unvalidatedData = info.data;
unvalidatedSlug = info.slug;
rawData = info.rawData;
} else {
const parsed = parseFrontmatter(rawContents, fileId);
body = parsed.content;
unvalidatedData = parsed.data;
unvalidatedSlug = parsed.data.slug;
rawData = parsed.matter;
}
const entryInfo = getEntryInfo({ const entryInfo = getEntryInfo({
entry: pathToFileURL(fileId), entry: pathToFileURL(fileId),
contentDir: contentPaths.contentDir, contentDir: contentPaths.contentDir,
@ -81,15 +102,14 @@ export function astroContentImportPlugin({
if (entryInfo instanceof Error) return; if (entryInfo instanceof Error) return;
const _internal = { filePath: fileId, rawData }; const _internal = { filePath: fileId, rawData };
const partialEntry = { data: unparsedData, body, _internal, ...entryInfo };
// TODO: move slug calculation to the start of the build // TODO: move slug calculation to the start of the build
// to generate a performant lookup map for `getEntryBySlug` // to generate a performant lookup map for `getEntryBySlug`
const slug = getEntrySlug(partialEntry); const slug = getEntrySlug({ ...entryInfo, unvalidatedSlug });
const collectionConfig = contentConfig?.collections[entryInfo.collection]; const collectionConfig = contentConfig?.collections[entryInfo.collection];
const data = collectionConfig const data = collectionConfig
? await getEntryData(partialEntry, collectionConfig) ? await getEntryData({ ...entryInfo, _internal, unvalidatedData }, collectionConfig)
: unparsedData; : unvalidatedData;
const code = escapeViteEnvReferences(` const code = escapeViteEnvReferences(`
export const id = ${JSON.stringify(entryInfo.id)}; export const id = ${JSON.stringify(entryInfo.id)};
@ -99,7 +119,7 @@ export const body = ${JSON.stringify(body)};
export const data = ${devalue.uneval(data) /* TODO: reuse astro props serializer */}; export const data = ${devalue.uneval(data) /* TODO: reuse astro props serializer */};
export const _internal = { export const _internal = {
filePath: ${JSON.stringify(fileId)}, filePath: ${JSON.stringify(fileId)},
rawData: ${JSON.stringify(rawData)}, rawData: ${JSON.stringify(unvalidatedData)},
}; };
`); `);
return { code }; return { code };
@ -109,7 +129,7 @@ export const _internal = {
viteServer.watcher.on('all', async (event, entry) => { viteServer.watcher.on('all', async (event, entry) => {
if ( if (
['add', 'unlink', 'change'].includes(event) && ['add', 'unlink', 'change'].includes(event) &&
getEntryType(entry, contentPaths) === 'config' getEntryType(entry, contentPaths, contentFileExts) === 'config'
) { ) {
// Content modules depend on config, so we need to invalidate them. // Content modules depend on config, so we need to invalidate them.
for (const modUrl of viteServer.moduleGraph.urlToModuleMap.keys()) { for (const modUrl of viteServer.moduleGraph.urlToModuleMap.keys()) {

8
packages/astro/src/content/vite-plugin-content-virtual-mod.ts
View file

@ -4,8 +4,9 @@ import type { Plugin } from 'vite';
import { normalizePath } from 'vite'; import { normalizePath } from 'vite';
import type { AstroSettings } from '../@types/astro.js'; import type { AstroSettings } from '../@types/astro.js';
import { appendForwardSlash, prependForwardSlash } from '../core/path.js'; import { appendForwardSlash, prependForwardSlash } from '../core/path.js';
import { contentFileExts, VIRTUAL_MODULE_ID } from './consts.js'; import { defaultContentFileExts, VIRTUAL_MODULE_ID } from './consts.js';
import { getContentPaths } from './utils.js'; import { getContentPaths } from './utils.js';
import { contentEntryTypes } from './~dream.js';
interface AstroContentVirtualModPluginParams { interface AstroContentVirtualModPluginParams {
settings: AstroSettings; settings: AstroSettings;
@ -22,6 +23,11 @@ export function astroContentVirtualModPlugin({
) )
) )
); );
const contentFileExts = [
...defaultContentFileExts,
...contentEntryTypes.map((t) => t.extensions).flat(),
];
const entryGlob = `${relContentDir}**/*{${contentFileExts.join(',')}}`; const entryGlob = `${relContentDir}**/*{${contentFileExts.join(',')}}`;
const virtualModContents = fsMod const virtualModContents = fsMod
.readFileSync(contentPaths.virtualModTemplate, 'utf-8') .readFileSync(contentPaths.virtualModTemplate, 'utf-8')

44
packages/astro/src/content/~dream.ts Normal file
View file

@ -0,0 +1,44 @@
import fs from 'node:fs';
import { fileURLToPath } from 'node:url';
import { parseFrontmatter } from './utils.js';
// Register things for typesafety
declare module 'astro:content' {
interface FancyRender {
'.mdoc': {
getParsed(): string;
getTransformed(): Promise<string>;
};
}
}
type ContentEntryType = {
extensions: string[];
getEntryInfo(params: { fileUrl: URL }): Promise<{
data: Record<string, unknown>;
/**
* Used for error hints to point to correct line and location
* Should be the untouched data as read from the file,
* including newlines
*/
rawData: string;
body: string;
slug: string;
}>;
};
export const contentEntryTypes: ContentEntryType[] = [
{
extensions: ['.mdoc'],
async getEntryInfo({ fileUrl }) {
const rawContents = await fs.promises.readFile(fileUrl, 'utf-8');
const parsed = parseFrontmatter(rawContents, fileURLToPath(fileUrl));
return {
data: parsed.data,
body: parsed.content,
slug: parsed.data.slug,
rawData: parsed.matter,
};
},
},
];