From 54ba9f5ee17a68f0e5a917011ce696de397220dc Mon Sep 17 00:00:00 2001 From: Matthew Phillips Date: Thu, 1 Apr 2021 16:34:11 -0400 Subject: [PATCH] Fix complex MDX parsing (#50) * Fix complex MDX parsing This allows fully MDX support using the micromark MDX extension. One caveat is that if you do something like use the less than sign, you need to escape it because the parser expects these to be tags otherwise. * Move micromark definition --- .../astro/pages/concepts/dev-server.md | 2 +- .../snowpack/astro/pages/guides/web-worker.md | 20 +------- .../posts/2020-05-26-snowpack-2-0-release.md | 2 +- .../astro/pages/reference/configuration.md | 4 +- .../astro/pages/tutorials/getting-started.md | 2 +- package-lock.json | 47 +++++++++++++++++++ package.json | 2 + src/@types/micromark.ts | 12 +++++ src/compiler/index.ts | 10 ++-- .../markdown}/micromark-collect-headers.ts | 0 .../markdown}/micromark-encode.ts | 13 +++-- src/compiler/markdown/micromark-mdx-astro.ts | 23 +++++++++ src/compiler/markdown/micromark.d.ts | 11 +++++ test/astro-markdown.test.js | 8 ++++ .../astro-markdown/astro/components/Hello.jsx | 5 ++ .../astro-markdown/astro/pages/complex.md | 11 +++++ 16 files changed, 136 insertions(+), 36 deletions(-) create mode 100644 src/@types/micromark.ts rename src/{ => compiler/markdown}/micromark-collect-headers.ts (100%) rename src/{ => compiler/markdown}/micromark-encode.ts (60%) create mode 100644 src/compiler/markdown/micromark-mdx-astro.ts create mode 100644 src/compiler/markdown/micromark.d.ts create mode 100644 test/fixtures/astro-markdown/astro/components/Hello.jsx create mode 100644 test/fixtures/astro-markdown/astro/pages/complex.md diff --git a/examples/snowpack/astro/pages/concepts/dev-server.md b/examples/snowpack/astro/pages/concepts/dev-server.md index 2566d760d..b15bde96b 100644 --- a/examples/snowpack/astro/pages/concepts/dev-server.md +++ b/examples/snowpack/astro/pages/concepts/dev-server.md @@ -6,6 +6,6 @@ description: Snowpack's dev server is fast because it only rebuilds the files yo ![dev command output example](/img/snowpack-dev-startup-2.png) -`snowpack dev` - Snowpack's dev server is an instant dev environment for [unbundled development.](/concepts/how-snowpack-works) The dev server will build a file only when it's requested by the browser. That means that Snowpack can start up instantly (usually in **<50ms**) and scale to infinitely large projects without slowing down. In contrast, it's common to see 30+ second dev startup times when building large apps with a traditional bundler. +`snowpack dev` - Snowpack's dev server is an instant dev environment for [unbundled development.](/concepts/how-snowpack-works) The dev server will build a file only when it's requested by the browser. That means that Snowpack can start up instantly (usually in **\<50ms**) and scale to infinitely large projects without slowing down. In contrast, it's common to see 30+ second dev startup times when building large apps with a traditional bundler. Snowpack supports JSX & TypeScript source code by default. You can extend your build even further with [custom plugins](/plugins) that connect Snowpack with your favorite build tools: TypeScript, Babel, Vue, Svelte, PostCSS, Sass... go wild! diff --git a/examples/snowpack/astro/pages/guides/web-worker.md b/examples/snowpack/astro/pages/guides/web-worker.md index ed64c22d6..4329c489a 100644 --- a/examples/snowpack/astro/pages/guides/web-worker.md +++ b/examples/snowpack/astro/pages/guides/web-worker.md @@ -28,22 +28,4 @@ const worker = new Worker(new URL('./esm-worker.js', import.meta.url), { name: 'my-worker', type: 'module', }); -``` - - +``` \ No newline at end of file diff --git a/examples/snowpack/astro/pages/posts/2020-05-26-snowpack-2-0-release.md b/examples/snowpack/astro/pages/posts/2020-05-26-snowpack-2-0-release.md index 989ed8b69..3882b1071 100644 --- a/examples/snowpack/astro/pages/posts/2020-05-26-snowpack-2-0-release.md +++ b/examples/snowpack/astro/pages/posts/2020-05-26-snowpack-2-0-release.md @@ -10,7 +10,7 @@ date: 2020-05-26 After 40+ beta versions & release candidates we are very excited to introduce **Snowpack 2.0: A build system for the modern web.** -- Starts up in <50ms and stays fast in large projects. +- Starts up in \<50ms and stays fast in large projects. - Bundle-free development with bundled production builds. - Built-in support for TypeScript, JSX, CSS Modules and more. - Works with React, Preact, Vue, Svelte, and all your favorite libraries. diff --git a/examples/snowpack/astro/pages/reference/configuration.md b/examples/snowpack/astro/pages/reference/configuration.md index 1469b4cbb..d1ed2fc34 100644 --- a/examples/snowpack/astro/pages/reference/configuration.md +++ b/examples/snowpack/astro/pages/reference/configuration.md @@ -84,8 +84,6 @@ Example: You can further customize this the build behavior for any mounted directory by using the expanded object notation: - - ```js // snowpack.config.js // Example: expanded object notation "mount" usage @@ -441,7 +439,7 @@ Run Snowpack's build pipeline through a file watcher. This option works best for Toggles whether HTML fragments are transformed like full HTML pages. -HTML fragments are HTML files not starting with "". +HTML fragments are HTML files not starting with "\". ### buildOptions.jsxFactory diff --git a/examples/snowpack/astro/pages/tutorials/getting-started.md b/examples/snowpack/astro/pages/tutorials/getting-started.md index 85037db89..07687dffa 100644 --- a/examples/snowpack/astro/pages/tutorials/getting-started.md +++ b/examples/snowpack/astro/pages/tutorials/getting-started.md @@ -48,7 +48,7 @@ npm install --save-dev snowpack ## Snowpack's development server -Adding a basic HTML file allows us to run Snowpack's development server, an instant development environment for unbundled development. The development server builds a file only when it's requested by the browser. That means that Snowpack can start up instantly (usually in **<50 ms**) and scale to infinitely large projects without slowing down. In contrast, it's common to see 30+ second development startup times when building large apps with a traditional bundler. +Adding a basic HTML file allows us to run Snowpack's development server, an instant development environment for unbundled development. The development server builds a file only when it's requested by the browser. That means that Snowpack can start up instantly (usually in **\<50 ms**) and scale to infinitely large projects without slowing down. In contrast, it's common to see 30+ second development startup times when building large apps with a traditional bundler. Create an `index.html` in your project with the following contents: diff --git a/package-lock.json b/package-lock.json index 83d45dc04..11dba039c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -354,6 +354,11 @@ "resolved": "https://registry.npmjs.org/@types/scheduler/-/scheduler-0.16.1.tgz", "integrity": "sha512-EaCxbanVeyxDRTQBkdLb3Bvl/HK7PBK6UJjsSixB0iHKoWxE5uu2Q/DgtpOhPIojN0Zl1whvOd7PoHs2P0s5eA==" }, + "@types/unist": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@types/unist/-/unist-2.0.3.tgz", + "integrity": "sha512-FvUupuM3rlRsRtCN+fDudtmytGO6iHJuuRKS1Ss0pG5z8oX0diNEw94UEL7hgDbpN94rgaK5R7sWm6RrSkZuAQ==" + }, "@types/yargs-parser": { "version": "20.2.0", "resolved": "https://registry.npmjs.org/@types/yargs-parser/-/yargs-parser-20.2.0.tgz", @@ -1641,6 +1646,11 @@ "integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==", "dev": true }, + "estree-util-is-identifier-name": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/estree-util-is-identifier-name/-/estree-util-is-identifier-name-1.1.0.tgz", + "integrity": "sha512-OVJZ3fGGt9By77Ix9NhaRbzfbDV/2rx9EP7YIDJTmsZSEc5kYn2vWcNccYyahJL2uAQZK2a5Or2i0wtIKTPoRQ==" + }, "estree-walker": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.0.tgz", @@ -2534,6 +2544,26 @@ "micromark": "~2.11.0" } }, + "micromark-extension-mdx-expression": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/micromark-extension-mdx-expression/-/micromark-extension-mdx-expression-0.3.2.tgz", + "integrity": "sha512-Sh8YHLSAlbm/7TZkVKEC4wDcJE8XhVpZ9hUXBue1TcAicrrzs/oXu7PHH3NcyMemjGyMkiVS34Y0AHC5KG3y4A==", + "requires": { + "micromark": "~2.11.0", + "vfile-message": "^2.0.0" + } + }, + "micromark-extension-mdx-jsx": { + "version": "0.3.3", + "resolved": "https://registry.npmjs.org/micromark-extension-mdx-jsx/-/micromark-extension-mdx-jsx-0.3.3.tgz", + "integrity": "sha512-kG3VwaJlzAPdtIVDznfDfBfNGMTIzsHqKpTmMlew/iPnUCDRNkX+48ElpaOzXAtK5axtpFKE3Hu3VBriZDnRTQ==", + "requires": { + "estree-util-is-identifier-name": "^1.0.0", + "micromark": "~2.11.0", + "micromark-extension-mdx-expression": "^0.3.2", + "vfile-message": "^2.0.0" + } + }, "micromatch": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.2.tgz", @@ -3798,6 +3828,14 @@ "crypto-random-string": "^2.0.0" } }, + "unist-util-stringify-position": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/unist-util-stringify-position/-/unist-util-stringify-position-2.0.3.tgz", + "integrity": "sha512-3faScn5I+hy9VleOq/qNbAd6pAx7iH5jYBMS9I1HgQVijz/4mv5Bvw5iw1sC/90CODiKo81G/ps8AJrISn687g==", + "requires": { + "@types/unist": "^2.0.2" + } + }, "untildify": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/untildify/-/untildify-4.0.0.tgz", @@ -3877,6 +3915,15 @@ "spdx-expression-parse": "^3.0.0" } }, + "vfile-message": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/vfile-message/-/vfile-message-2.0.4.tgz", + "integrity": "sha512-DjssxRGkMvifUOJre00juHoP9DPWuzjxKuMDrhNbk2TdaYYBNMStsNhEOt3idrtI12VQYM/1+iM0KOzXi4pxwQ==", + "requires": { + "@types/unist": "^2.0.0", + "unist-util-stringify-position": "^2.0.0" + } + }, "vue": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/vue/-/vue-3.0.7.tgz", diff --git a/package.json b/package.json index 0f4b70f6f..8c947396c 100644 --- a/package.json +++ b/package.json @@ -54,6 +54,8 @@ "magic-string": "^0.25.3", "micromark": "^2.11.4", "micromark-extension-gfm": "^0.3.3", + "micromark-extension-mdx-expression": "^0.3.2", + "micromark-extension-mdx-jsx": "^0.3.3", "node-fetch": "^2.6.1", "postcss": "^8.2.8", "react": "^17.0.1", diff --git a/src/@types/micromark.ts b/src/@types/micromark.ts new file mode 100644 index 000000000..0e0dc2465 --- /dev/null +++ b/src/@types/micromark.ts @@ -0,0 +1,12 @@ + +export interface MicromarkExtensionContext { + sliceSerialize(node: any): string; + raw(value: string): void; +} + +export type MicromarkExtensionCallback = (this: MicromarkExtensionContext, node: any) => void; + +export interface MicromarkExtension { + enter?: Record; + exit?: Record; +} \ No newline at end of file diff --git a/src/compiler/index.ts b/src/compiler/index.ts index 541bae21e..112b7881e 100644 --- a/src/compiler/index.ts +++ b/src/compiler/index.ts @@ -8,8 +8,9 @@ import matter from 'gray-matter'; import gfmHtml from 'micromark-extension-gfm/html.js'; import { parse } from '../parser/index.js'; -import { createMarkdownHeadersCollector } from '../micromark-collect-headers.js'; -import { encodeMarkdown } from '../micromark-encode.js'; +import { createMarkdownHeadersCollector } from './markdown/micromark-collect-headers.js'; +import { encodeMarkdown } from './markdown/micromark-encode.js'; +import { encodeAstroMdx } from './markdown/micromark-mdx-astro.js'; import { optimize } from './optimize/index.js'; import { codegen } from './codegen.js'; @@ -56,10 +57,11 @@ async function convertMdToJsx( ): Promise { const { data: frontmatterData, content } = matter(contents); const { headers, headersExtension } = createMarkdownHeadersCollector(); + const { htmlAstro, mdAstro } = encodeAstroMdx(); const mdHtml = micromark(content, { allowDangerousHtml: true, - extensions: [gfmSyntax()], - htmlExtensions: [gfmHtml, encodeMarkdown, headersExtension], + extensions: [gfmSyntax(), ...htmlAstro], + htmlExtensions: [gfmHtml, encodeMarkdown, headersExtension, mdAstro], }); // TODO: Warn if reserved word is used in "frontmatterData" diff --git a/src/micromark-collect-headers.ts b/src/compiler/markdown/micromark-collect-headers.ts similarity index 100% rename from src/micromark-collect-headers.ts rename to src/compiler/markdown/micromark-collect-headers.ts diff --git a/src/micromark-encode.ts b/src/compiler/markdown/micromark-encode.ts similarity index 60% rename from src/micromark-encode.ts rename to src/compiler/markdown/micromark-encode.ts index f9e863fdd..635ab3b54 100644 --- a/src/micromark-encode.ts +++ b/src/compiler/markdown/micromark-encode.ts @@ -1,4 +1,5 @@ -import type { HtmlExtension, Token, Tokenize } from 'micromark/dist/shared-types'; +import type { Token } from 'micromark/dist/shared-types'; +import type { MicromarkExtension, MicromarkExtensionContext } from '../../@types/micromark'; const characterReferences = { '"': 'quot', @@ -19,15 +20,13 @@ function encode(value: string): string { } /** Encode Markdown node */ -function encodeToken(this: Record void>) { +function encodeToken(this: MicromarkExtensionContext) { const token: Token = arguments[0]; - const serialize = (this.sliceSerialize as unknown) as (t: Token) => string; - const raw = (this.raw as unknown) as (s: string) => void; - const value = serialize(token); - raw(encode(value)); + const value = this.sliceSerialize(token); + this.raw(encode(value)); } -const plugin: HtmlExtension = { +const plugin: MicromarkExtension = { exit: { codeTextData: encodeToken, codeFlowValue: encodeToken, diff --git a/src/compiler/markdown/micromark-mdx-astro.ts b/src/compiler/markdown/micromark-mdx-astro.ts new file mode 100644 index 000000000..0ffd69fb1 --- /dev/null +++ b/src/compiler/markdown/micromark-mdx-astro.ts @@ -0,0 +1,23 @@ +import type { MicromarkExtension } from '../../@types/micromark'; +import mdxExpression from 'micromark-extension-mdx-expression'; +import mdxJsx from 'micromark-extension-mdx-jsx'; + + +/** + * Keep MDX. + */ +export function encodeAstroMdx() { + const extension: MicromarkExtension = { + enter: { + mdxJsxFlowTag(node: any) { + const mdx = this.sliceSerialize(node); + this.raw(mdx); + } + } + }; + + return { + htmlAstro: [mdxExpression(), mdxJsx()], + mdAstro: extension + }; +} \ No newline at end of file diff --git a/src/compiler/markdown/micromark.d.ts b/src/compiler/markdown/micromark.d.ts new file mode 100644 index 000000000..1f389e473 --- /dev/null +++ b/src/compiler/markdown/micromark.d.ts @@ -0,0 +1,11 @@ +declare module 'micromark-extension-mdx-expression' { + import type { HtmlExtension } from 'micromark/dist/shared-types'; + + export default function(): HtmlExtension; +} + +declare module 'micromark-extension-mdx-jsx' { + import type { HtmlExtension } from 'micromark/dist/shared-types'; + + export default function(): HtmlExtension; +} \ No newline at end of file diff --git a/test/astro-markdown.test.js b/test/astro-markdown.test.js index a07f692c0..572569466 100644 --- a/test/astro-markdown.test.js +++ b/test/astro-markdown.test.js @@ -42,4 +42,12 @@ Markdown('Can load markdown pages with hmx', async () => { assert.ok($('#test').length, 'There is a div added via a component from markdown'); }); +Markdown('Can load more complex jsxy stuff', async () => { + const result = await runtime.load('/complex'); + + const $ = doc(result.contents); + const $el = $('#test'); + assert.equal($el.text(), 'Hello world'); +}); + Markdown.run(); diff --git a/test/fixtures/astro-markdown/astro/components/Hello.jsx b/test/fixtures/astro-markdown/astro/components/Hello.jsx new file mode 100644 index 000000000..787ca587b --- /dev/null +++ b/test/fixtures/astro-markdown/astro/components/Hello.jsx @@ -0,0 +1,5 @@ +import { h } from 'preact'; + +export default function({ name }) { + return
Hello {name}
+} \ No newline at end of file diff --git a/test/fixtures/astro-markdown/astro/pages/complex.md b/test/fixtures/astro-markdown/astro/pages/complex.md new file mode 100644 index 000000000..ff7582c84 --- /dev/null +++ b/test/fixtures/astro-markdown/astro/pages/complex.md @@ -0,0 +1,11 @@ +--- +layout: ../layouts/content.astro +title: My Blog Post +description: This is a post about some stuff. +import: + Hello: '../components/Hello.jsx' +--- + +## Interesting Topic + +