michael/astro
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

wip: get dream API for file loader working

Browse source
This commit is contained in:
bholmesdev 2023-02-07 14:00:23 -05:00
parent 926c82d8a5
commit 65846f4151
9 changed files with 122 additions and 37 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
examples/with-markdoc/src/content/blog/test.mdoc Normal file
View file

@ -0,0 +1,5 @@
---
title: Example!
---
# Hey there

4
examples/with-markdoc/src/pages/index.astro
View file

@ -5,6 +5,10 @@ import RenderMarkdoc from '../renderer/RenderMarkdoc.astro';
import { getTransformed } from '../components/test.mdoc';
import { Code } from 'astro/components';
import Marquee from '../components/Marquee.astro';
import { getEntryBySlug } from 'astro:content';
const mdocEntry = await getEntryBySlug('blog', 'test');
console.log(mdocEntry);
---
<html lang="en">

3
packages/astro/src/content/consts.ts
View file

@ -1,4 +1,5 @@
export const contentFileExts = ['.md', '.mdx'];
/** TODO as const*/
export const defaultContentFileExts = ['.md', '.mdx'];
export const PROPAGATED_ASSET_FLAG = 'astroPropagatedAssets';
export const CONTENT_FLAG = 'astroContent';
export const VIRTUAL_MODULE_ID = 'astro:content';

11
packages/astro/src/content/types-generator.ts
View file

@ -8,7 +8,7 @@ import type { AstroSettings } from '../@types/astro.js';
import { AstroError, AstroErrorData } from '../core/errors/index.js';
import { info, LogOptions, warn } from '../core/logger/core.js';
import { isRelativePath } from '../core/path.js';
import { CONTENT_TYPES_FILE } from './consts.js';
import { CONTENT_TYPES_FILE, defaultContentFileExts } from './consts.js';
import {
ContentConfig,
ContentObservable,
@ -22,6 +22,7 @@ import {
NoCollectionError,
parseFrontmatter,
} from './utils.js';
import { contentEntryTypes } from './~dream.js';
type ChokidarEvent = 'add' | 'addDir' | 'change' | 'unlink' | 'unlinkDir';
type RawContentEvent = { name: ChokidarEvent; entry: string };
@ -57,6 +58,10 @@ export async function createContentTypesGenerator({
}: CreateContentGeneratorParams) {
const contentTypes: ContentTypes = {};
const contentPaths = getContentPaths(settings.config, fs);
const contentFileExts = [
...defaultContentFileExts,
...contentEntryTypes.map((t) => t.extensions).flat(),
];
let events: EventWithOptions[] = [];
let debounceTimeout: NodeJS.Timeout | undefined;
@ -121,7 +126,7 @@ export async function createContentTypesGenerator({
}
return { shouldGenerateTypes: true };
}
const fileType = getEntryType(fileURLToPath(event.entry), contentPaths);
const fileType = getEntryType(fileURLToPath(event.entry), contentPaths, contentFileExts);
if (fileType === 'ignored') {
return { shouldGenerateTypes: false };
}
@ -300,7 +305,7 @@ async function parseSlug({
// on dev server startup or production build init.
const rawContents = await fs.promises.readFile(event.entry, 'utf-8');
const { data: frontmatter } = parseFrontmatter(rawContents, fileURLToPath(event.entry));
return getEntrySlug({ ...entryInfo, data: frontmatter });
return getEntrySlug({ ...entryInfo, unvalidatedSlug: frontmatter.slug });
}
function setEntry(

32
packages/astro/src/content/utils.ts
View file

@ -7,6 +7,7 @@ import { ErrorPayload as ViteErrorPayload, normalizePath, ViteDevServer } from '
import { z } from 'zod';
import { AstroConfig, AstroSettings } from '../@types/astro.js';
import { AstroError, AstroErrorData } from '../core/errors/index.js';
import { appendForwardSlash } from '../core/path.js';
import { contentFileExts, CONTENT_TYPES_FILE } from './consts.js';
export const collectionConfigParser = z.object({
@ -29,14 +30,7 @@ export const contentConfigParser = z.object({
export type CollectionConfig = z.infer<typeof collectionConfigParser>;
export type ContentConfig = z.infer<typeof contentConfigParser>;
type Entry = {
id: string;
collection: string;
slug: string;
data: any;
body: string;
_internal: { rawData: string; filePath: string };
};
type EntryInternal = { rawData: string; filePath: string };
export type EntryInfo = {
id: string;
@ -53,10 +47,10 @@ export function getEntrySlug({
id,
collection,
slug,
data: unparsedData,
}: Pick<Entry, 'id' | 'collection' | 'slug' | 'data'>) {
unvalidatedSlug,
}: EntryInfo & { unvalidatedSlug?: unknown }) {
try {
return z.string().default(slug).parse(unparsedData.slug);
return z.string().default(slug).parse(unvalidatedSlug);
} catch {
throw new AstroError({
...AstroErrorData.InvalidContentEntrySlugError,
@ -65,9 +59,12 @@ export function getEntrySlug({
}
}
export async function getEntryData(entry: Entry, collectionConfig: CollectionConfig) {
export async function getEntryData(
entry: EntryInfo & { unvalidatedData: Record<string, unknown>; _internal: EntryInternal },
collectionConfig: CollectionConfig
) {
// Remove reserved `slug` field before parsing data
let { slug, ...data } = entry.data;
let { slug, ...data } = entry.unvalidatedData;
if (collectionConfig.schema) {
// TODO: remove for 2.0 stable release
if (
@ -94,7 +91,9 @@ export async function getEntryData(entry: Entry, collectionConfig: CollectionCon
});
}
// Use `safeParseAsync` to allow async transforms
const parsed = await collectionConfig.schema.safeParseAsync(entry.data, { errorMap });
const parsed = await collectionConfig.schema.safeParseAsync(entry.unvalidatedData, {
errorMap,
});
if (parsed.success) {
data = parsed.data;
} else {
@ -160,14 +159,15 @@ export function getEntryInfo({
export function getEntryType(
entryPath: string,
paths: Pick<ContentPaths, 'config' | 'contentDir'>
paths: Pick<ContentPaths, 'config' | 'contentDir'>,
contentFileExts: string[]
): 'content' | 'config' | 'ignored' | 'unsupported' {
const { ext, base } = path.parse(entryPath);
const fileUrl = pathToFileURL(entryPath);
if (hasUnderscoreBelowContentDirectoryPath(fileUrl, paths.contentDir) || isOnIgnoreList(base)) {
return 'ignored';
} else if ((contentFileExts as readonly string[]).includes(ext)) {
} else if (contentFileExts.includes(ext)) {
return 'content';
} else if (fileUrl.href === paths.config.url.href) {
return 'config';

4
packages/astro/src/content/vite-plugin-content-assets.ts
View file

@ -11,7 +11,7 @@ import { prependForwardSlash } from '../core/path.js';
import { getStylesForURL } from '../core/render/dev/css.js';
import { getScriptsForURL } from '../core/render/dev/scripts.js';
import {
contentFileExts,
defaultContentFileExts,
LINKS_PLACEHOLDER,
PROPAGATED_ASSET_FLAG,
SCRIPTS_PLACEHOLDER,
@ -22,7 +22,7 @@ function isPropagatedAsset(viteId: string): boolean {
const url = new URL(viteId, 'file://');
return (
url.searchParams.has(PROPAGATED_ASSET_FLAG) &&
contentFileExts.some((ext) => url.pathname.endsWith(ext))
defaultContentFileExts.some((ext) => url.pathname.endsWith(ext))
);
}

48
packages/astro/src/content/vite-plugin-content-imports.ts
View file

@ -1,3 +1,4 @@
import { contentEntryTypes } from './~dream.js';
import * as devalue from 'devalue';
import type fsMod from 'node:fs';
import { pathToFileURL } from 'url';
@ -6,7 +7,7 @@ import { AstroSettings } from '../@types/astro.js';
import { AstroErrorData } from '../core/errors/errors-data.js';
import { AstroError } from '../core/errors/errors.js';
import { escapeViteEnvReferences, getFileInfo } from '../vite-plugin-utils/index.js';
import { contentFileExts, CONTENT_FLAG } from './consts.js';
import { defaultContentFileExts, CONTENT_FLAG } from './consts.js';
import {
ContentConfig,
getContentPaths,
@ -19,8 +20,8 @@ import {
} from './utils.js';
function isContentFlagImport(viteId: string) {
const { pathname, searchParams } = new URL(viteId, 'file://');
return searchParams.has(CONTENT_FLAG) && contentFileExts.some((ext) => pathname.endsWith(ext));
const { searchParams } = new URL(viteId, 'file://');
return searchParams.has(CONTENT_FLAG);
}
export function astroContentImportPlugin({
@ -31,6 +32,10 @@ export function astroContentImportPlugin({
settings: AstroSettings;
}): Plugin {
const contentPaths = getContentPaths(settings.config, fs);
const contentFileExts = [
...defaultContentFileExts,
...contentEntryTypes.map((t) => t.extensions).flat(),
];
return {
name: 'astro:content-imports',
@ -69,11 +74,27 @@ export function astroContentImportPlugin({
});
}
const rawContents = await fs.promises.readFile(fileId, 'utf-8');
const {
content: body,
data: unparsedData,
matter: rawData = '',
} = parseFrontmatter(rawContents, fileId);
const contentEntryType = contentEntryTypes.find((entryType) =>
entryType.extensions.some((ext) => fileId.endsWith(ext))
);
let body: string,
unvalidatedData: Record<string, unknown>,
unvalidatedSlug: string,
rawData: string;
if (contentEntryType) {
const info = await contentEntryType.getEntryInfo({ fileUrl: pathToFileURL(fileId) });
body = info.body;
unvalidatedData = info.data;
unvalidatedSlug = info.slug;
rawData = info.rawData;
} else {
const parsed = parseFrontmatter(rawContents, fileId);
body = parsed.content;
unvalidatedData = parsed.data;
unvalidatedSlug = parsed.data.slug;
rawData = parsed.matter;
}
const entryInfo = getEntryInfo({
entry: pathToFileURL(fileId),
contentDir: contentPaths.contentDir,
@ -81,15 +102,14 @@ export function astroContentImportPlugin({
if (entryInfo instanceof Error) return;
const _internal = { filePath: fileId, rawData };
const partialEntry = { data: unparsedData, body, _internal, ...entryInfo };
// TODO: move slug calculation to the start of the build
// to generate a performant lookup map for `getEntryBySlug`
const slug = getEntrySlug(partialEntry);
const slug = getEntrySlug({ ...entryInfo, unvalidatedSlug });
const collectionConfig = contentConfig?.collections[entryInfo.collection];
const data = collectionConfig
? await getEntryData(partialEntry, collectionConfig)
: unparsedData;
? await getEntryData({ ...entryInfo, _internal, unvalidatedData }, collectionConfig)
: unvalidatedData;
const code = escapeViteEnvReferences(`
export const id = ${JSON.stringify(entryInfo.id)};
@ -99,7 +119,7 @@ export const body = ${JSON.stringify(body)};
export const data = ${devalue.uneval(data) /* TODO: reuse astro props serializer */};
export const _internal = {
filePath: ${JSON.stringify(fileId)},
rawData: ${JSON.stringify(rawData)},
rawData: ${JSON.stringify(unvalidatedData)},
};
`);
return { code };
@ -109,7 +129,7 @@ export const _internal = {
viteServer.watcher.on('all', async (event, entry) => {
if (
['add', 'unlink', 'change'].includes(event) &&
getEntryType(entry, contentPaths) === 'config'
getEntryType(entry, contentPaths, contentFileExts) === 'config'
) {
// Content modules depend on config, so we need to invalidate them.
for (const modUrl of viteServer.moduleGraph.urlToModuleMap.keys()) {

8
packages/astro/src/content/vite-plugin-content-virtual-mod.ts
View file

@ -4,8 +4,9 @@ import type { Plugin } from 'vite';
import { normalizePath } from 'vite';
import type { AstroSettings } from '../@types/astro.js';
import { appendForwardSlash, prependForwardSlash } from '../core/path.js';
import { contentFileExts, VIRTUAL_MODULE_ID } from './consts.js';
import { defaultContentFileExts, VIRTUAL_MODULE_ID } from './consts.js';
import { getContentPaths } from './utils.js';
import { contentEntryTypes } from './~dream.js';
interface AstroContentVirtualModPluginParams {
settings: AstroSettings;
@ -22,6 +23,11 @@ export function astroContentVirtualModPlugin({
)
)
);
const contentFileExts = [
...defaultContentFileExts,
...contentEntryTypes.map((t) => t.extensions).flat(),
];
const entryGlob = `${relContentDir}**/*{${contentFileExts.join(',')}}`;
const virtualModContents = fsMod
.readFileSync(contentPaths.virtualModTemplate, 'utf-8')

44
packages/astro/src/content/~dream.ts Normal file
View file

@ -0,0 +1,44 @@
import fs from 'node:fs';
import { fileURLToPath } from 'node:url';
import { parseFrontmatter } from './utils.js';
// Register things for typesafety
declare module 'astro:content' {
interface FancyRender {
'.mdoc': {
getParsed(): string;
getTransformed(): Promise<string>;
};
}
}
type ContentEntryType = {
extensions: string[];
getEntryInfo(params: { fileUrl: URL }): Promise<{
data: Record<string, unknown>;
/**
* Used for error hints to point to correct line and location
* Should be the untouched data as read from the file,
* including newlines
*/
rawData: string;
body: string;
slug: string;
}>;
};
export const contentEntryTypes: ContentEntryType[] = [
{
extensions: ['.mdoc'],
async getEntryInfo({ fileUrl }) {
const rawContents = await fs.promises.readFile(fileUrl, 'utf-8');
const parsed = parseFrontmatter(rawContents, fileURLToPath(fileUrl));
return {
data: parsed.data,
body: parsed.content,
slug: parsed.data.slug,
rawData: parsed.matter,
};
},
},
];