Allow HMX components in markdown (#19)

* Allow HMX components in markdown

This adds support for HMX components in markdown. The mechanism for importing is via frontmatter. We could do this differently (setup script maybe?) but since this was the easiest to implement I thought it was a good first-pass option.

* Remove node-fetch from snowpack config

* Assert that the runtime is created successfully

* Add back in the micromark extension for encoding entities

* Encode both codeTextData and codeFlowValue

* Install snowpack app's deps

.github/workflows/nodejs.yml

@@ -19,6 +19,10 @@ jobs:
         node-version: ${{ matrix.node-version }}
     - name: npm install, build, and test
       run: |
+        cd examples/snowpack
+        npm ci
+        cd ../..
+
         npm ci
         npm run build
         npm test

examples/snowpack/snowpack.config.js

@@ -10,11 +10,7 @@ module.exports = {
     '@snowpack/plugin-svelte',
     '@snowpack/plugin-vue',
   ],
-  packageOptions: {
-    external: [
-      'node-fetch'
-    ]
-  },
+  packageOptions: {},
   buildOptions: {
     out: '_site',
   },

src/dev.ts

@@ -57,6 +57,8 @@ export default async function (astroConfig: AstroConfig) {
             break;
           }
         }
+        res.statusCode = 500;
+        res.end(formatErrorForBrowser(result.error));
         break;
       }
     }
@@ -66,3 +68,8 @@ export default async function (astroConfig: AstroConfig) {
     console.log(`Server running at http://${hostname}:${port}/`);
   });
 }
+
+function formatErrorForBrowser(error: Error) {
+  // TODO make this pretty.
+  return error.toString();
+}

src/micromark-encode.ts

@@ -0,0 +1,35 @@
+import type { HtmlExtension, Token, Tokenize } from 'micromark/dist/shared-types';
+
+const characterReferences = {
+  '"': 'quot',
+  '&': 'amp',
+  '<': 'lt',
+  '>': 'gt',
+  '{': 'lbrace',
+  '}': 'rbrace',
+};
+
+type EncodedChars = '"' | '&' | '<' | '>' | '{' | '}';
+
+function encode(value: string): string {
+  return value.replace(/["&<>{}]/g, (raw: string) => {
+    return '&' + characterReferences[raw as EncodedChars] + ';';
+  });
+}
+
+function encodeToken(this: Record<string, () => void>) {
+  const token: Token = arguments[0];
+  const serialize = (this.sliceSerialize as unknown) as (t: Token) => string;
+  const raw = (this.raw as unknown) as (s: string) => void;
+  const value = serialize(token);
+  raw(encode(value));
+}
+
+const plugin: HtmlExtension = {
+  exit: {
+    codeTextData: encodeToken,
+    codeFlowValue: encodeToken,
+  },
+};
+
+export { plugin as encodeMarkdown };

src/transform2.ts

@@ -8,6 +8,7 @@ import gfmHtml from 'micromark-extension-gfm/html.js';
 import { CompileResult, TransformResult } from './@types/astro';
 import { parse } from './compiler/index.js';
 import { createMarkdownHeadersCollector } from './micromark-collect-headers.js';
+import { encodeMarkdown } from './micromark-encode.js';
 import { defaultLogOptions } from './logger.js';
 import { optimize } from './optimize/index.js';
 import { codegen } from './codegen/index.js';
@@ -54,8 +55,9 @@ async function convertMdToJsx(
   const { data: _frontmatterData, content } = matter(contents);
   const { headers, headersExtension } = createMarkdownHeadersCollector();
   const mdHtml = micromark(content, {
+    allowDangerousHtml: true,
     extensions: [gfmSyntax()],
-    htmlExtensions: [gfmHtml, headersExtension],
+    htmlExtensions: [gfmHtml, encodeMarkdown, headersExtension],
   });
 
   const setupContext = {
@@ -68,19 +70,26 @@ async function convertMdToJsx(
     },
   };
 
+  let imports = '';
+  for(let [ComponentName, specifier] of Object.entries(_frontmatterData.import || {})) {
+    imports += `import ${ComponentName} from '${specifier}';\n`;
+  }
+
   // </script> can't be anywhere inside of a JS string, otherwise the HTML parser fails.
   // Break it up here so that the HTML parser won't detect it.
   const stringifiedSetupContext = JSON.stringify(setupContext).replace(/\<\/script\>/g, `</scrip" + "t>`);
 
-  return convertHmxToJsx(
-    `<script astro>
-      ${_frontmatterData.layout ? `export const layout = ${JSON.stringify(_frontmatterData.layout)};` : ''}
-      export function setup({context}) {
-        return {context: ${stringifiedSetupContext} };
-      }
-    </script><section>{${JSON.stringify(mdHtml)}}</section>`,
-    { compileOptions, filename, fileID }
-  );
+  const raw =  `<script astro>
+  ${imports}
+  ${_frontmatterData.layout ? `export const layout = ${JSON.stringify(_frontmatterData.layout)};` : ''}
+  export function setup({context}) {
+    return {context: ${stringifiedSetupContext} };
+  }
+</script><section>${mdHtml}</section>`;
+
+  const convertOptions = { compileOptions, filename, fileID };
+
+  return convertHmxToJsx(raw, convertOptions);
 }
 
 async function transformFromSource(

test/fixtures/hmx-markdown/astro.config.mjs

@@ -0,0 +1,9 @@
+
+export default {
+  projectRoot: '.',
+  hmxRoot: './astro',
+  dist: './_site',
+  extensions: {
+    '.jsx': 'preact'
+  }
+}

test/fixtures/hmx-markdown/astro/components/Example.jsx

@@ -0,0 +1,5 @@
+import { h } from 'preact';
+
+export default function() {
+  return <div id="test">Testing</div>
+}

test/fixtures/hmx-markdown/astro/layouts/content.hmx

@@ -0,0 +1,3 @@
+<div class="container">
+  <slot></slot>
+</div>

test/fixtures/hmx-markdown/astro/pages/index.hmx

@@ -0,0 +1,13 @@
+<script astro>
+  export function setup() {
+    return {
+      props: {}
+    }
+  }
+</script>
+
+<astro:head>
+  <!-- Head Stuff -->
+</astro:head>
+
+<h1>Hello world!</h1>

test/fixtures/hmx-markdown/astro/pages/post.md

@@ -0,0 +1,13 @@
+---
+layout: layouts/content.hmx
+title: My Blog Post
+description: This is a post about some stuff.
+import:
+  Example: '../components/Example.jsx'
+---
+
+## Interesting Topic
+
+<div id="first">Some content</div>
+
+<Example />

test/fixtures/hmx-markdown/snowpack.config.js

@@ -0,0 +1,5 @@
+export default {
+  mount: {
+
+  }
+};

test/hmx-markdown.test.js

@@ -0,0 +1,45 @@
+import { suite } from 'uvu';
+import * as assert from 'uvu/assert';
+import { createRuntime } from '../lib/runtime.js';
+import { loadConfig } from '../lib/config.js';
+import { doc } from './test-utils.js';
+
+const HMXMD = suite('HMX Markdown');
+
+let runtime, setupError;
+
+HMXMD.before(async () => {
+  const astroConfig = await loadConfig(new URL('./fixtures/hmx-markdown', import.meta.url).pathname);
+  
+  const logging = {
+    level: 'error',
+    dest: process.stderr
+  };
+
+  try {
+    runtime = await createRuntime(astroConfig, logging);
+  } catch(err) {
+    console.error(err);
+    setupError = err;
+  }
+});
+
+HMXMD.after(async () => {
+  runtime && runtime.shutdown();
+});
+
+HMXMD('No errors creating a runtime', () => {
+  assert.equal(setupError, undefined);
+});
+
+HMXMD('Can load markdown pages with hmx', async () => {
+  const result = await runtime.load('/post');
+
+  assert.equal(result.statusCode, 200);
+
+  const $ = doc(result.contents);
+  assert.ok($('#first').length, 'There is a div added in markdown');
+  assert.ok($('#test').length, 'There is a div added via a component from markdown');
+});
+
+HMXMD.run();

test/react-component.test.js

@@ -6,7 +6,7 @@ import { doc } from './test-utils.js';
 
 const React = suite('React Components');
 
-let runtime;
+let runtime, setupError;
 
 React.before(async () => {
   const astroConfig = await loadConfig(new URL('./fixtures/react-component', import.meta.url).pathname);
@@ -20,7 +20,7 @@ React.before(async () => {
     runtime = await createRuntime(astroConfig, logging);
   } catch(err) {
     console.error(err);
-    throw err;
+    setupError = err;
   }
 });
 
@@ -28,6 +28,10 @@ React.after(async () => {
   await runtime.shutdown();
 });
 
+React('No error creating the runtime', () => {
+  assert.equal(setupError, undefined);
+});
+
 React('Can load hmx page', async () => {
   const result = await runtime.load('/');
 

test/snowpack-integration.test.js

@@ -10,7 +10,7 @@ const { readdir, stat } = fsPromises;
 
 const SnowpackDev = suite('snowpack.dev');
 
-let runtime, cwd;
+let runtime, cwd, setupError;
 
 SnowpackDev.before(async () => {
 // Bug: Snowpack config is still loaded relative to the current working directory.
@@ -28,7 +28,7 @@ SnowpackDev.before(async () => {
     runtime = await createRuntime(astroConfig, logging);
   } catch(err) {
     console.error(err);
-    throw err;
+    setupError = err;
   }
 });
 
@@ -58,6 +58,10 @@ async function* allPages(root) {
   }
 }
 
+SnowpackDev('No error creating the runtime', () => {
+  assert.equal(setupError, undefined);
+});
+
 SnowpackDev('Can load every page', async () => {
   const failed = [];