From 9cdada0bcc5fe3fad6f645ffda5f7e5934c738b5 Mon Sep 17 00:00:00 2001 From: Nate Moore Date: Fri, 21 May 2021 15:52:20 -0500 Subject: [PATCH] Markdown issue cleanup (#224) * fix: markdown issues * chore: add changeset * chore: add missing dep * perf: parallelize compileHtml for children --- .changeset/spotty-ways-leave.md | 6 + packages/astro-parser/src/parse/state/text.ts | 10 +- packages/astro/package.json | 1 + packages/astro/src/@types/astro.ts | 8 + packages/astro/src/compiler/codegen/index.ts | 334 ++++++++++-------- packages/astro/src/compiler/index.ts | 7 +- .../astro/src/compiler/markdown/codeblock.ts | 41 +++ .../src/compiler/markdown/micromark.d.ts | 3 + .../src/compiler/markdown/remark-mdx-lite.ts | 39 +- .../compiler/markdown/remark-scoped-styles.ts | 4 +- .../astro/src/compiler/transform/prism.ts | 19 +- packages/astro/src/compiler/utils.ts | 23 +- packages/astro/src/config.ts | 1 + packages/astro/src/frontend/markdown.ts | 26 -- packages/astro/src/runtime.ts | 7 +- yarn.lock | 57 ++- 16 files changed, 380 insertions(+), 206 deletions(-) create mode 100644 .changeset/spotty-ways-leave.md create mode 100644 packages/astro/src/compiler/markdown/codeblock.ts delete mode 100644 packages/astro/src/frontend/markdown.ts diff --git a/.changeset/spotty-ways-leave.md b/.changeset/spotty-ways-leave.md new file mode 100644 index 000000000..17e16b696 --- /dev/null +++ b/.changeset/spotty-ways-leave.md @@ -0,0 +1,6 @@ +--- +'astro': patch +'astro-parser': patch +--- + +Fixes a few edge case bugs with Astro's handling of Markdown content diff --git a/packages/astro-parser/src/parse/state/text.ts b/packages/astro-parser/src/parse/state/text.ts index eac810a0a..bde2ec5e4 100644 --- a/packages/astro-parser/src/parse/state/text.ts +++ b/packages/astro-parser/src/parse/state/text.ts @@ -8,7 +8,15 @@ export default function text(parser: Parser) { let data = ''; - while (parser.index < parser.template.length && !parser.match('---') && !parser.match('<') && !parser.match('{') && !parser.match('`')) { + const shouldContinue = () => { + // Special case 'code' content to avoid tripping up on user code + if (parser.current().name === 'code') { + return !parser.match('<') && !parser.match('{'); + } + return !parser.match('---') && !parser.match('<') && !parser.match('{') && !parser.match('`'); + } + + while (parser.index < parser.template.length && shouldContinue()) { data += parser.template[parser.index++]; } diff --git a/packages/astro/package.json b/packages/astro/package.json index 29a4fff4d..6b2dfb1d8 100644 --- a/packages/astro/package.json +++ b/packages/astro/package.json @@ -61,6 +61,7 @@ "locate-character": "^2.0.5", "magic-string": "^0.25.3", "mdast-util-mdx": "^0.1.1", + "micromark-extension-mdxjs": "^0.3.0", "mime": "^2.5.2", "moize": "^6.0.1", "node-fetch": "^2.6.1", diff --git a/packages/astro/src/@types/astro.ts b/packages/astro/src/@types/astro.ts index 5df93635f..2f9983f53 100644 --- a/packages/astro/src/@types/astro.ts +++ b/packages/astro/src/@types/astro.ts @@ -8,12 +8,20 @@ export interface AstroConfigRaw { export type ValidExtensionPlugins = 'astro' | 'react' | 'preact' | 'svelte' | 'vue'; +export interface AstroMarkdownOptions { + /** Enable or disable footnotes syntax extension */ + footnotes: boolean; + /** Enable or disable GitHub-flavored Markdown syntax extension */ + gfm: boolean; +} export interface AstroConfig { dist: string; projectRoot: URL; astroRoot: URL; public: URL; extensions?: Record; + /** Options for rendering markdown content */ + markdownOptions?: Partial; /** Options specific to `astro build` */ buildOptions: { /** Your public domain, e.g.: https://my-site.dev/. Used to generate sitemaps and canonical URLs. */ diff --git a/packages/astro/src/compiler/codegen/index.ts b/packages/astro/src/compiler/codegen/index.ts index 64d7c1822..fb6bac9be 100644 --- a/packages/astro/src/compiler/codegen/index.ts +++ b/packages/astro/src/compiler/codegen/index.ts @@ -1,12 +1,13 @@ import type { Ast, Script, Style, TemplateNode } from 'astro-parser'; import type { CompileOptions } from '../../@types/compiler'; -import type { AstroConfig, TransformResult, ValidExtensionPlugins } from '../../@types/astro'; +import type { AstroConfig, AstroMarkdownOptions, TransformResult, ValidExtensionPlugins } from '../../@types/astro'; import 'source-map-support/register.js'; import eslexer from 'es-module-lexer'; import esbuild from 'esbuild'; import path from 'path'; -import { walk } from 'estree-walker'; +import { parse } from 'astro-parser'; +import { walk, asyncWalk } from 'estree-walker'; import _babelGenerator from '@babel/generator'; import babelParser from '@babel/parser'; import { codeFrameColumns } from '@babel/code-frame'; @@ -16,6 +17,7 @@ import { error, warn } from '../../logger.js'; import { fetchContent } from './content.js'; import { isFetchContent } from './utils.js'; import { yellow } from 'kleur/colors'; +import { MarkdownRenderingOptions, renderMarkdown } from '../utils'; const traverse: typeof babelTraverse.default = (babelTraverse.default as any).default; @@ -306,7 +308,7 @@ interface CodegenState { components: Components; css: string[]; markers: { - insideMarkdown: boolean | string; + insideMarkdown: boolean | Record; }; importExportStatements: Set; dynamicImports: DynamicImportMap; @@ -538,160 +540,210 @@ function compileCss(style: Style, state: CodegenState) { }); } +/** dedent markdown */ +function dedent(str: string) { + let arr = str.match(/^[ \t]*(?=\S)/gm); + let first = !!arr && arr.find((x) => x.length > 0)?.length; + return !arr || !first ? str : str.replace(new RegExp(`^[ \\t]{0,${first}}`, 'gm'), ''); +} + + /** Compile page markup */ -function compileHtml(enterNode: TemplateNode, state: CodegenState, compileOptions: CompileOptions) { - const { components, css, importExportStatements, dynamicImports, filename } = state; - const { astroConfig } = compileOptions; +async function compileHtml(enterNode: TemplateNode, state: CodegenState, compileOptions: CompileOptions): Promise { + return new Promise((resolve) => { + const { components, css, importExportStatements, dynamicImports, filename } = state; + const { astroConfig } = compileOptions; - let outSource = ''; - walk(enterNode, { - enter(node: TemplateNode, parent: TemplateNode) { - switch (node.type) { - case 'Expression': { - let children: string[] = []; - for (const child of node.children || []) { - children.push(compileHtml(child, state, compileOptions)); - } - let raw = ''; - let nextChildIndex = 0; - for (const chunk of node.codeChunks) { - raw += chunk; - if (nextChildIndex < children.length) { - raw += children[nextChildIndex++]; - } - } - // TODO Do we need to compile this now, or should we compile the entire module at the end? - let code = compileExpressionSafe(raw).trim().replace(/\;$/, ''); - outSource += `,(${code})`; - this.skip(); - break; - } - case 'MustacheTag': - case 'Comment': - return; - case 'Fragment': - break; - case 'Slot': - case 'Head': - case 'InlineComponent': - case 'Title': - case 'Element': { - const name: string = node.name; - if (!name) { - throw new Error('AHHHH'); - } - try { - const attributes = getAttributes(node.attributes); + let paren = -1; + let buffers = { + out: '', + markdown: '', + }; + let curr: keyof typeof buffers = 'out'; - outSource += outSource === '' ? '' : ','; - if (node.type === 'Slot') { - outSource += `(children`; - return; - } - const COMPONENT_NAME_SCANNER = /^[A-Z]/; - if (!COMPONENT_NAME_SCANNER.test(name)) { - outSource += `h("${name}", ${attributes ? generateAttributes(attributes) : 'null'}`; - if (state.markers.insideMarkdown) { - outSource += `,h(__astroMarkdownRender, null`; + /** renders markdown stored in `buffers.markdown` to JSX and pushes that to `buffers.out` */ + async function pushMarkdownToBuffer() { + const md = buffers.markdown; + const { markdownOptions = {} } = astroConfig; + const { $scope: scopedClassName } = state.markers.insideMarkdown as Record<'$scope', any>; + let { content: rendered } = await renderMarkdown(dedent(md), { ...markdownOptions as AstroMarkdownOptions, mode: 'astro-md', $: { scopedClassName: scopedClassName.slice(1, -1) } }); + const ast = parse(rendered); + const result = await compileHtml(ast.html, {...state, markers: {...state.markers, insideMarkdown: false }}, compileOptions); + + buffers.out += ',' + result; + buffers.markdown = ''; + curr = 'out'; + } + + asyncWalk(enterNode, { + async enter(node: TemplateNode, parent: TemplateNode) { + switch (node.type) { + case 'Expression': { + const children: string[] = await Promise.all((node.children ?? []).map(child => compileHtml(child, state, compileOptions))); + let raw = ''; + let nextChildIndex = 0; + for (const chunk of node.codeChunks) { + raw += chunk; + if (nextChildIndex < children.length) { + raw += children[nextChildIndex++]; } - return; } - const [componentName, componentKind] = name.split(':'); - const componentImportData = components[componentName]; - if (!componentImportData) { - throw new Error(`Unknown Component: ${componentName}`); + // TODO Do we need to compile this now, or should we compile the entire module at the end? + let code = compileExpressionSafe(raw).trim().replace(/\;$/, ''); + if (state.markers.insideMarkdown) { + buffers[curr] += `{${code}}`; + } else { + buffers[curr] += `,(${code})`; } - if (componentImportData.type === '.astro') { - if (componentName === 'Markdown') { - const attributeStr = attributes ? generateAttributes(attributes) : 'null'; - state.markers.insideMarkdown = attributeStr; - outSource += `h(__astroMarkdownRender, ${attributeStr}`; + this.skip(); + break; + } + case 'MustacheTag': + case 'Comment': + return; + case 'Fragment': + break; + case 'Slot': + case 'Head': + case 'InlineComponent': + case 'Title': + case 'Element': { + const name: string = node.name; + if (!name) { + throw new Error('AHHHH'); + } + try { + const attributes = getAttributes(node.attributes); + + buffers.out += buffers.out === '' ? '' : ','; + + if (node.type === 'Slot') { + buffers[curr] += `(children`; + paren++; return; } - } - const { wrapper, wrapperImport } = getComponentWrapper(name, components[componentName], { astroConfig, dynamicImports, filename }); - if (wrapperImport) { - importExportStatements.add(wrapperImport); - } + const COMPONENT_NAME_SCANNER = /^[A-Z]/; + if (!COMPONENT_NAME_SCANNER.test(name)) { + if (curr === 'markdown') { + await pushMarkdownToBuffer(); + } + buffers[curr] += `h("${name}", ${attributes ? generateAttributes(attributes) : 'null'}`; + paren++; + return; + } + const [componentName, componentKind] = name.split(':'); + const componentImportData = components[componentName]; + if (!componentImportData) { + throw new Error(`Unknown Component: ${componentName}`); + } + if (componentImportData.type === '.astro') { + if (componentName === 'Markdown') { + const { $scope } = attributes ?? {}; + state.markers.insideMarkdown = { $scope }; + curr = 'markdown'; + return; + } + } + const { wrapper, wrapperImport } = getComponentWrapper(name, components[componentName], { astroConfig, dynamicImports, filename }); + if (wrapperImport) { + importExportStatements.add(wrapperImport); + } + if (curr === 'markdown') { + await pushMarkdownToBuffer(); + } - outSource += `h(${wrapper}, ${attributes ? generateAttributes(attributes) : 'null'}`; - if (state.markers.insideMarkdown) { - const attributeStr = state.markers.insideMarkdown; - outSource += `,h(__astroMarkdownRender, ${attributeStr}`; + paren++; + buffers[curr] += `h(${wrapper}, ${attributes ? generateAttributes(attributes) : 'null'}`; + } catch (err) { + // handle errors in scope with filename + const rel = filename.replace(astroConfig.projectRoot.pathname, ''); + // TODO: return actual codeframe here + error(compileOptions.logging, rel, err.toString()); } - } catch (err) { - // handle errors in scope with filename - const rel = filename.replace(astroConfig.projectRoot.pathname, ''); - // TODO: return actual codeframe here - error(compileOptions.logging, rel, err.toString()); - } - return; - } - case 'Attribute': { - this.skip(); - return; - } - case 'Style': { - css.push(node.content.styles); // if multiple