fix: markdown issues

packages/astro-parser/src/parse/state/text.ts

@@ -8,7 +8,15 @@ export default function text(parser: Parser) {
 
   let data = '';
 
-  while (parser.index < parser.template.length && !parser.match('---') && !parser.match('<') && !parser.match('{') && !parser.match('`')) {
+  const shouldContinue = () => {
+    // Special case 'code' content to avoid tripping up on user code
+    if (parser.current().name === 'code') {
+      return !parser.match('<') && !parser.match('{');
+    }
+    return !parser.match('---') && !parser.match('<') && !parser.match('{') && !parser.match('`');
+  }
+
+  while (parser.index < parser.template.length && shouldContinue()) {
     data += parser.template[parser.index++];
   }
 

packages/astro/src/@types/astro.ts

@@ -8,12 +8,20 @@ export interface AstroConfigRaw {
 
 export type ValidExtensionPlugins = 'astro' | 'react' | 'preact' | 'svelte' | 'vue';
 
+export interface AstroMarkdownOptions {
+  /** Enable or disable footnotes syntax extension */
+  footnotes: boolean;
+  /** Enable or disable GitHub-flavored Markdown syntax extension */
+  gfm: boolean;
+}
 export interface AstroConfig {
   dist: string;
   projectRoot: URL;
   astroRoot: URL;
   public: URL;
   extensions?: Record<string, ValidExtensionPlugins>;
+  /** Options for rendering markdown content */
+  markdownOptions?: Partial<AstroMarkdownOptions>;
   /** Options specific to `astro build` */
   buildOptions: {
     /** Your public domain, e.g.: https://my-site.dev/. Used to generate sitemaps and canonical URLs. */

packages/astro/src/compiler/codegen/index.ts

@@ -1,12 +1,13 @@
 import type { Ast, Script, Style, TemplateNode } from 'astro-parser';
 import type { CompileOptions } from '../../@types/compiler';
-import type { AstroConfig, TransformResult, ValidExtensionPlugins } from '../../@types/astro';
+import type { AstroConfig, AstroMarkdownOptions, TransformResult, ValidExtensionPlugins } from '../../@types/astro';
 
 import 'source-map-support/register.js';
 import eslexer from 'es-module-lexer';
 import esbuild from 'esbuild';
 import path from 'path';
-import { walk } from 'estree-walker';
+import { parse } from 'astro-parser';
+import { walk, asyncWalk } from 'estree-walker';
 import _babelGenerator from '@babel/generator';
 import babelParser from '@babel/parser';
 import { codeFrameColumns } from '@babel/code-frame';
@@ -16,6 +17,7 @@ import { error, warn } from '../../logger.js';
 import { fetchContent } from './content.js';
 import { isFetchContent } from './utils.js';
 import { yellow } from 'kleur/colors';
+import { MarkdownRenderingOptions, renderMarkdown } from '../utils';
 
 const traverse: typeof babelTraverse.default = (babelTraverse.default as any).default;
 
@@ -306,7 +308,7 @@ interface CodegenState {
   components: Components;
   css: string[];
   markers: {
-    insideMarkdown: boolean | string;
+    insideMarkdown: boolean | Record<string, any>;
   };
   importExportStatements: Set<string>;
   dynamicImports: DynamicImportMap;
@@ -538,19 +540,48 @@ function compileCss(style: Style, state: CodegenState) {
   });
 }
 
+/** dedent markdown */
+function dedent(str: string) {
+  let arr = str.match(/^[ \t]*(?=\S)/gm);
+  let first = !!arr && arr.find((x) => x.length > 0)?.length;
+  return !arr || !first ? str : str.replace(new RegExp(`^[ \\t]{0,${first}}`, 'gm'), '');
+}
+
+
 /** Compile page markup */
-function compileHtml(enterNode: TemplateNode, state: CodegenState, compileOptions: CompileOptions) {
+async function compileHtml(enterNode: TemplateNode, state: CodegenState, compileOptions: CompileOptions): Promise<string> {
+  return new Promise((resolve) => {
     const { components, css, importExportStatements, dynamicImports, filename } = state;
     const { astroConfig } = compileOptions;
 
-  let outSource = '';
+    let paren = -1;
-  walk(enterNode, {
+    let buffers = {
-    enter(node: TemplateNode, parent: TemplateNode) {
+      out: '',
+      markdown: '',
+    };
+    let curr: keyof typeof buffers = 'out';
+
+    /** renders markdown stored in `buffers.markdown` to JSX and pushes that to `buffers.out` */
+    async function pushMarkdownToBuffer() {
+      const md = buffers.markdown;
+      const { markdownOptions = {} } = astroConfig;
+      const { $scope: scopedClassName } = state.markers.insideMarkdown as Record<'$scope', any>;
+      let { content: rendered } = await renderMarkdown(dedent(md), { ...markdownOptions as AstroMarkdownOptions, mode: 'astro-md', $: { scopedClassName: scopedClassName.slice(1, -1) } });
+      const ast = parse(rendered);
+      const result = await compileHtml(ast.html, {...state, markers: {...state.markers, insideMarkdown: false }}, compileOptions);
+
+      buffers.out += ',' + result;
+      buffers.markdown = '';
+      curr = 'out';
+    }
+
+    asyncWalk(enterNode, {
+      async enter(node: TemplateNode, parent: TemplateNode) {
         switch (node.type) {
           case 'Expression': {
             let children: string[] = [];
             for (const child of node.children || []) {
-            children.push(compileHtml(child, state, compileOptions));
+              children.push(await compileHtml(child, state, compileOptions));
             }
             let raw = '';
             let nextChildIndex = 0;
@@ -562,7 +593,11 @@ function compileHtml(enterNode: TemplateNode, state: CodegenState, compileOption
             }
             // TODO Do we need to compile this now, or should we compile the entire module at the end?
             let code = compileExpressionSafe(raw).trim().replace(/\;$/, '');
-          outSource += `,(${code})`;
+            if (state.markers.insideMarkdown) {
+              buffers[curr] += `{${code}}`;
+            } else {
+              buffers[curr] += `,(${code})`;
+            }
             this.skip();
             break;
           }
@@ -583,17 +618,20 @@ function compileHtml(enterNode: TemplateNode, state: CodegenState, compileOption
             try {
               const attributes = getAttributes(node.attributes);
 
-            outSource += outSource === '' ? '' : ',';
+              buffers.out += buffers.out === '' ? '' : ',';
+
               if (node.type === 'Slot') {
-              outSource += `(children`;
+                buffers[curr] += `(children`;
+                paren++;
                 return;
               }
               const COMPONENT_NAME_SCANNER = /^[A-Z]/;
               if (!COMPONENT_NAME_SCANNER.test(name)) {
-              outSource += `h("${name}", ${attributes ? generateAttributes(attributes) : 'null'}`;
+                if (curr === 'markdown') {
-              if (state.markers.insideMarkdown) {
+                  await pushMarkdownToBuffer();
-                outSource += `,h(__astroMarkdownRender, null`;
                 }
+                buffers[curr] += `h("${name}", ${attributes ? generateAttributes(attributes) : 'null'}`;
+                paren++;
                 return;
               }
               const [componentName, componentKind] = name.split(':');
@@ -603,9 +641,9 @@ function compileHtml(enterNode: TemplateNode, state: CodegenState, compileOption
               }
               if (componentImportData.type === '.astro') {
                 if (componentName === 'Markdown') {
-                const attributeStr = attributes ? generateAttributes(attributes) : 'null';
+                  const { $scope } = attributes ?? {};
-                state.markers.insideMarkdown = attributeStr;
+                  state.markers.insideMarkdown = { $scope };
-                outSource += `h(__astroMarkdownRender, ${attributeStr}`;
+                  curr = 'markdown';
                   return;
                 }
               }
@@ -613,12 +651,12 @@ function compileHtml(enterNode: TemplateNode, state: CodegenState, compileOption
               if (wrapperImport) {
                 importExportStatements.add(wrapperImport);
               }
-
+              if (curr === 'markdown') {
-            outSource += `h(${wrapper}, ${attributes ? generateAttributes(attributes) : 'null'}`;
+                await pushMarkdownToBuffer();
-            if (state.markers.insideMarkdown) {
-              const attributeStr = state.markers.insideMarkdown;
-              outSource += `,h(__astroMarkdownRender, ${attributeStr}`;
               }
+
+              paren++;
+              buffers[curr] += `h(${wrapper}, ${attributes ? generateAttributes(attributes) : 'null'}`;
             } catch (err) {
               // handle errors in scope with filename
               const rel = filename.replace(astroConfig.projectRoot.pathname, '');
@@ -638,47 +676,60 @@ function compileHtml(enterNode: TemplateNode, state: CodegenState, compileOption
           }
           case 'CodeSpan':
           case 'CodeFence': {
-          outSource += ',' + JSON.stringify(node.raw);
+            if (state.markers.insideMarkdown) {
+              if (curr === 'out') curr = 'markdown';
+              buffers[curr] += node.raw;
+              return;
+            }
+            buffers[curr] += ',' + JSON.stringify(node.data);
             return;
           }
           case 'Text': {
-          const text = getTextFromAttribute(node);
+            let text = getTextFromAttribute(node);
-          // Whitespace is significant if we are immediately inside of <Markdown>,
+            if (state.markers.insideMarkdown) {
-          // but not if we're inside of another component in <Markdown>
+              if (curr === 'out') curr = 'markdown';
+              buffers[curr] += text;
+              return;
+            }
             if (parent.name !== 'Markdown' && !text.trim()) {
               return;
             }
-          outSource += ',' + JSON.stringify(text);
+            if (parent.name === 'code') {
+              // Special case, escaped { characters from markdown content
+              text = node.raw.replace(/&#x26;#123;/g, '{');
+            }
+            buffers[curr] += ',' + JSON.stringify(text);
             return;
           }
           default:
             throw new Error('Unexpected (enter) node type: ' + node.type);
         }
       },
-    leave(node, parent, prop, index) {
+      async leave(node, parent, prop, index) {
         switch (node.type) {
           case 'Text':
-        case 'CodeSpan':
-        case 'CodeFence':
           case 'Attribute':
           case 'Comment':
           case 'Fragment':
           case 'Expression':
           case 'MustacheTag':
             return;
+          case 'CodeSpan':
+          case 'CodeFence':
+            return;
           case 'Slot':
           case 'Head':
           case 'Body':
           case 'Title':
           case 'Element':
           case 'InlineComponent': {
-          if (node.type === 'InlineComponent' && node.name === 'Markdown') {
+            if (node.type === 'InlineComponent' && curr === 'markdown' && buffers.markdown !== '') {
-            state.markers.insideMarkdown = false;
+              await pushMarkdownToBuffer();
             }
-          if (state.markers.insideMarkdown) {
+            if (paren !== -1) {
-            outSource += ')';
+              buffers.out += ')';
+              paren--;
             }
-          outSource += ')';
             return;
           }
           case 'Style': {
@@ -689,9 +740,13 @@ function compileHtml(enterNode: TemplateNode, state: CodegenState, compileOption
             throw new Error('Unexpected (leave) node type: ' + node.type);
         }
       },
+    }).then(() => {
+      const content = buffers.out.replace(/^\,/, '').replace(/\,\)/g, ')').replace(/\,+/g, ',').replace(/\)h/g, '),h');
+      buffers.out = '';
+      buffers.markdown = '';
+      return resolve(content);
+    });
   });
-
-  return outSource;
 }
 
 /**
@@ -721,7 +776,7 @@ export async function codegen(ast: Ast, { compileOptions, filename }: CodeGenOpt
 
   compileCss(ast.css, state);
 
-  const html = compileHtml(ast.html, state, compileOptions);
+  const html = await compileHtml(ast.html, state, compileOptions);
 
   return {
     script: script,

packages/astro/src/compiler/index.ts

@@ -29,7 +29,7 @@ interface ConvertAstroOptions {
  * 2. Transform
  * 3. Codegen
  */
-async function convertAstroToJsx(template: string, opts: ConvertAstroOptions): Promise<TransformResult> {
+export async function convertAstroToJsx(template: string, opts: ConvertAstroOptions): Promise<TransformResult> {
   const { filename } = opts;
 
   // 1. Parse
@@ -48,11 +48,12 @@ async function convertAstroToJsx(template: string, opts: ConvertAstroOptions): P
  * .md -> .astro source
  */
 export async function convertMdToAstroSource(contents: string, { filename }: { filename: string }): Promise<string> {
-  const {
+  let {
     content,
     frontmatter: { layout, ...frontmatter },
     ...data
   } = await renderMarkdownWithFrontmatter(contents);
+
   if (frontmatter['astro'] !== undefined) {
     throw new Error(`"astro" is a reserved word but was used as a frontmatter value!\n\tat ${filename}`);
   }
@@ -109,7 +110,6 @@ export async function compileComponent(
 ): Promise<CompileResult> {
   const result = await transformFromSource(source, { compileOptions, filename, projectRoot });
   const site = compileOptions.astroConfig.buildOptions.site || `http://localhost:${compileOptions.astroConfig.devOptions.port}`;
-  const usesMarkdown = !!result.imports.find((spec) => spec.indexOf('Markdown') > -1);
 
   // return template
   let modJsx = `
@@ -120,7 +120,6 @@ ${result.imports.join('\n')}
 
 // \`__render()\`: Render the contents of the Astro module.
 import { h, Fragment } from '${internalImport('h.js')}';
-${usesMarkdown ? `import __astroMarkdownRender from '${internalImport('markdown.js')}';` : ''};
 const __astroRequestSymbol = Symbol('astro.request');
 async function __render(props, ...children) {
   const Astro = {

packages/astro/src/compiler/markdown/codeblock.ts

@@ -0,0 +1,41 @@
+import { visit } from 'unist-util-visit';
+
+/**  */
+export function remarkCodeBlock() {
+  const visitor = (node: any) => {
+    const { data, lang, meta } = node;
+    let currentClassName = data?.hProperties?.class ?? '';
+    node.data = node.data || {};
+    node.data.hProperties = node.data.hProperties || {};
+    node.data.hProperties = { ...node.data.hProperties, class: `language-${lang} ${currentClassName}`.trim(), lang, meta }
+
+    return node;
+  };
+  return () => (tree: any) => visit(tree, 'code', visitor);
+}
+
+/**  */
+export function rehypeCodeBlock() {
+  const escapeCode = (code: any) => {
+    code.children = code.children.map((child: any) => {
+      if (child.type === 'text') {
+        return { ...child, value: child.value.replace(/\{/g, '{') };
+      }
+      return child;
+    })
+  }
+  const visitor = (node: any) => {
+    if (node.tagName === 'code') {
+      escapeCode(node);
+      return;
+    }
+
+    if (node.tagName !== 'pre') return;
+    const code = node.children[0];
+    if (code.tagName !== 'code') return;
+    node.properties = { ...code.properties };
+
+    return node;
+  };
+  return () => (tree: any) => visit(tree, 'element', visitor);
+}

packages/astro/src/compiler/markdown/micromark.d.ts

@@ -2,6 +2,9 @@ declare module '@silvenon/remark-smartypants' {
   export default function (): any;
 }
 
+declare module 'mdast-util-mdx';
+declare module 'micromark-extension-mdxjs';
+
 declare module 'mdast-util-mdx/from-markdown.js' {
   export default function (): any;
 }

packages/astro/src/compiler/markdown/remark-mdx-lite.ts

@@ -1,26 +1,31 @@
-import fromMarkdown from 'mdast-util-mdx/from-markdown.js';
-import toMarkdown from 'mdast-util-mdx/to-markdown.js';
 
-/** See https://github.com/micromark/micromark-extension-mdx-md */
+import syntaxMdxjs from 'micromark-extension-mdxjs'
-const syntax = { disable: { null: ['autolink', 'codeIndented'] } };
+import {fromMarkdown, toMarkdown} from 'mdast-util-mdx'
 
 /**
- * Lite version of https://github.com/mdx-js/mdx/tree/main/packages/remark-mdx
+ * Add the micromark and mdast extensions for MDX.js (JS aware MDX).
- * We don't need all the features MDX does because all components are precompiled
+ *
- * to HTML. We just want to disable a few MD features that cause issues.
+ * @this {Processor}
+ * @param {MdxOptions} [options]
+ * @return {void}
  */
-function mdxLite(this: any) {
+export function remarkMdx(this: any, options: any) {
-  let data = this.data();
+  let data = this.data()
 
-  add('micromarkExtensions', syntax);
+  add('micromarkExtensions', syntaxMdxjs(options))
-  add('fromMarkdownExtensions', fromMarkdown);
+  add('fromMarkdownExtensions', fromMarkdown)
-  add('toMarkdownExtensions', toMarkdown);
+  add('toMarkdownExtensions', toMarkdown)
 
-  /** Adds remark plugin */
+  /**
+   * @param {string} field
+   * @param {unknown} value
+   */
   function add(field: string, value: any) {
-    if (data[field]) data[field].push(value);
+    // Other extensions defined before this.
-    else data[field] = [value];
+    // Useful when externalizing.
+    /* c8 ignore next 2 */
+    // @ts-ignore Assume it’s an array.
+    if (data[field]) data[field].push(value)
+    else data[field] = [value]
   }
 }
-
-export default mdxLite;

packages/astro/src/compiler/markdown/remark-scoped-styles.ts

@@ -7,10 +7,10 @@ export default function scopedStyles(className: string) {
     if (noVisit.has(node.type)) return;
 
     const { data } = node;
-    const currentClassName = data?.hProperties?.class ?? '';
+    let currentClassName = data?.hProperties?.class ?? '';
     node.data = node.data || {};
     node.data.hProperties = node.data.hProperties || {};
-    node.data.hProperties.className = `${className} ${currentClassName}`.trim();
+    node.data.hProperties.class = `${className} ${currentClassName}`.trim();
 
     return node;
   };

packages/astro/src/compiler/transform/prism.ts

@@ -1,5 +1,5 @@
 import type { Transformer } from '../../@types/transformer';
-import type { Script } from 'astro-parser';
+import type { Script, TemplateNode } from 'astro-parser';
 import { getAttrValue } from '../../ast.js';
 
 const PRISM_IMPORT = `import Prism from 'astro/components/Prism.astro';\n`;
@@ -8,7 +8,17 @@ const prismImportExp = /import Prism from ['"]astro\/components\/Prism.astro['"]
 function escape(code: string) {
   return code.replace(/[`$]/g, (match) => {
     return '\\' + match;
-  });
+  }).replace(/{/g, '{');
+}
+
+/** Unescape { characters transformed by Markdown generation */
+function unescapeCode(code: TemplateNode) {
+  code.children = code.children?.map(child => {
+    if (child.type === 'Text') {
+      return { ...child, raw: child.raw.replace(/{/g, '{') }
+    }
+    return child;
+  })
 }
 /** default export - Transform prism   */
 export default function (module: Script): Transformer {
@@ -19,6 +29,11 @@ export default function (module: Script): Transformer {
       html: {
         Element: {
           enter(node) {
+            if (node.name === 'code') {
+              unescapeCode(node);
+              return;
+            }
+
             if (node.name !== 'pre') return;
             const codeEl = node.children && node.children[0];
             if (!codeEl || codeEl.name !== 'code') return;

packages/astro/src/compiler/utils.ts

@@ -1,20 +1,20 @@
-import mdxLite from './markdown/remark-mdx-lite.js';
+import type { AstroMarkdownOptions } from '../@types/astro';
 import createCollectHeaders from './markdown/rehype-collect-headers.js';
 import scopedStyles from './markdown/remark-scoped-styles.js';
+import { remarkCodeBlock, rehypeCodeBlock } from './markdown/codeblock.js';
 import raw from 'rehype-raw';
+
 import unified from 'unified';
 import markdown from 'remark-parse';
 import markdownToHtml from 'remark-rehype';
-import smartypants from '@silvenon/remark-smartypants';
+// import smartypants from '@silvenon/remark-smartypants';
-import stringify from 'rehype-stringify';
+import rehypeStringify from 'rehype-stringify';
 
-export interface MarkdownRenderingOptions {
+export interface MarkdownRenderingOptions extends Partial<AstroMarkdownOptions> {
   $?: {
     scopedClassName: string | null;
   };
-  footnotes?: boolean;
+  mode: 'md'|'astro-md';
-  gfm?: boolean;
-  plugins?: any[];
 }
 
 /** Internal utility for rendering a full markdown file and extracting Frontmatter data */
@@ -22,16 +22,16 @@ export async function renderMarkdownWithFrontmatter(contents: string, opts?: Mar
   // Dynamic import to ensure that "gray-matter" isn't built by Snowpack
   const { default: matter } = await import('gray-matter');
   const { data: frontmatter, content } = matter(contents);
-  const value = await renderMarkdown(content, opts);
+  const value = await renderMarkdown(content, { ...opts, mode: 'md' });
   return { ...value, frontmatter };
 }
 
 /** Shared utility for rendering markdown */
 export async function renderMarkdown(content: string, opts?: MarkdownRenderingOptions | null) {
-  const { $: { scopedClassName = null } = {}, footnotes: useFootnotes = true, gfm: useGfm = true, plugins = [] } = opts ?? {};
+  const { $: { scopedClassName = null } = {}, mode = 'astro-md', footnotes: useFootnotes = true, gfm: useGfm = true } = opts ?? {};
   const { headers, rehypeCollectHeaders } = createCollectHeaders();
 
-  let parser = unified().use(markdown).use(mdxLite).use(smartypants);
+  let parser = unified().use(markdown).use(remarkCodeBlock());
 
   if (scopedClassName) {
     parser = parser.use(scopedStyles(scopedClassName));
@@ -53,7 +53,8 @@ export async function renderMarkdown(content: string, opts?: MarkdownRenderingOp
       .use(markdownToHtml, { allowDangerousHtml: true, passThrough: ['raw'] })
       .use(raw)
       .use(rehypeCollectHeaders)
-      .use(stringify)
+      .use(rehypeCodeBlock())
+      .use(rehypeStringify)
       .process(content);
     result = vfile.contents.toString();
   } catch (err) {

packages/astro/src/config.ts

@@ -54,6 +54,7 @@ function configDefaults(userConfig?: any): any {
   if (!config.devOptions) config.devOptions = {};
   if (!config.devOptions.port) config.devOptions.port = 3000;
   if (!config.buildOptions) config.buildOptions = {};
+  if (!config.markdownOptions) config.markdownOptions = {};
   if (typeof config.buildOptions.sitemap === 'undefined') config.buildOptions.sitemap = true;
 
   return config;

packages/astro/src/frontend/markdown.ts

@@ -1,26 +0,0 @@
-import { renderMarkdown } from '../compiler/utils.js';
-
-/**
- * Functional component which uses Astro's built-in Markdown rendering
- * to render out its children.
- *
- * Note: the children have already been properly escaped/rendered
- * by the parser and Astro, so at this point we're just rendering
- * out plain markdown, no need for JSX support
- */
-export default async function Markdown(props: { $scope: string | null }, ...children: string[]): Promise<string> {
-  const { $scope = null } = props ?? {};
-  const text = dedent(children.join('').trimEnd());
-  let { content } = await renderMarkdown(text, { $: { scopedClassName: $scope } });
-  if (content.split('<p>').length === 2) {
-    content = content.replace(/^\<p\>/i, '').replace(/\<\/p\>$/i, '');
-  }
-  return content;
-}
-
-/** Remove leading indentation based on first line */
-function dedent(str: string) {
-  let arr = str.match(/^[ \t]*(?=\S)/gm);
-  let first = !!arr && arr.find((x) => x.length > 0)?.length;
-  return !arr || !first ? str : str.replace(new RegExp(`^[ \\t]{0,${first}}`, 'gm'), '');
-}

packages/astro/src/runtime.ts

@@ -314,7 +314,12 @@ async function createSnowpack(astroConfig: AstroConfig, options: CreateSnowpackO
     },
     packageOptions: {
       knownEntrypoints: ['preact-render-to-string'],
-      external: ['@vue/server-renderer', 'node-fetch', 'prismjs/components/index.js'],
+      external: [
+        '@vue/server-renderer',
+        'node-fetch',
+        'prismjs/components/index.js',
+        'gray-matter',
+      ],
     },
   });
 

yarn.lock

@@ -2218,6 +2218,14 @@ astral-regex@^2.0.0:
 
 "astro-languageserver@file:tools/astro-languageserver":
   version "0.4.0"
+  dependencies:
+    source-map "^0.7.3"
+    typescript "^4.3.1-rc"
+    vscode-css-languageservice "^5.1.1"
+    vscode-emmet-helper "2.1.2"
+    vscode-html-languageservice "^3.0.3"
+    vscode-languageserver "6.1.1"
+    vscode-languageserver-textdocument "^1.0.1"
 
 "astro-scripts@file:scripts":
   version "0.0.1"