Use npm package names to load internal deps (#294)

* Use npm package names to load internal deps

This is necessary so that published Astro components work. These components will be built by esinstall and therefore they cannot rely on `_astro_internal`. The fix is to use npm specifiers everywhere.

* Move most of frontend to internal

* Mark astro/internal/markdown.js as external

* Move markdown stuff to its own package

This moves the markdown stuff to its own package so that we can externalize it in the markdown component.

* Add the changeset

.changeset/forty-apricots-film.md

@@ -0,0 +1,6 @@
+---
+'astro': patch
+'@astrojs/markdown-support': patch
+---
+
+Fixes issues with using astro via the create script

package.json

@@ -6,7 +6,7 @@
     "release": "yarn build && yarn changeset publish",
     "benchmark": "yarn workspace astro run benchmark",
     "build": "yarn build:core",
-    "build:core": "lerna run build --scope astro --scope astro-parser",
+    "build:core": "lerna run build --scope astro --scope astro-parser --scope @astrojs/markdown-support",
     "build:vscode": "lerna run build --scope astro-languageserver --scope astro-vscode --scope astro-parser",
     "dev:vscode": "lerna run dev --scope astro-languageserver --scope astro-vscode --scope astro-parser --parallel --stream",
     "format": "prettier -w \"**/*.{js,jsx,ts,tsx,md,json}\"",

packages/astro/components/Markdown.astro

@@ -1,5 +1,5 @@
 ---
-import { renderMarkdown } from 'astro/dist/frontend/markdown.js';
+import { renderMarkdown } from '@astrojs/markdown-support';
 
 export let content: string;
 export let $scope: string;

packages/astro/package.json

@@ -12,7 +12,7 @@
     "./components": "./components/index.js",
     "./components/*": "./components/*",
     "./runtime/svelte": "./dist/frontend/runtime/svelte.js",
-    "./dist/frontend/markdown.js": "./dist/frontend/markdown.js"
+    "./internal/*": "./dist/internal/*"
   },
   "imports": {
     "#astro/compiler": "./dist/compiler/index.js",
@@ -28,13 +28,14 @@
     "astro.mjs"
   ],
   "scripts": {
-    "build": "astro-scripts build \"src/*.ts\" \"src/compiler/index.ts\" \"src/frontend/**/*.ts\" && tsc",
+    "build": "astro-scripts build \"src/*.ts\" \"src/compiler/index.ts\" \"src/frontend/**/*.ts\" \"src/internal/**/*.ts\" && tsc",
     "postbuild": "astro-scripts copy \"src/**/*.astro\"",
     "dev": "astro-scripts dev \"src/**/*.ts\"",
     "benchmark": "node test/benchmark/dev.bench.js && node test/benchmark/build.bench.js",
     "test": "uvu test -i fixtures -i benchmark -i test-utils.js"
   },
   "dependencies": {
+    "@astrojs/markdown-support": "0.1.0",
     "@astrojs/renderer-preact": "0.1.0",
     "@astrojs/renderer-react": "0.1.0",
     "@astrojs/renderer-svelte": "0.1.0",
@@ -60,7 +61,6 @@
     "fast-xml-parser": "^3.19.0",
     "fdir": "^5.0.0",
     "find-up": "^5.0.0",
-    "github-slugger": "^1.3.0",
     "gray-matter": "^4.0.2",
     "gzip-size": "^6.0.0",
     "hast-to-hyperscript": "~9.0.0",
@@ -76,9 +76,6 @@
     "postcss": "^8.2.15",
     "postcss-icss-keyframes": "^0.2.1",
     "prismjs": "^1.23.0",
-    "rehype-parse": "^7.0.1",
-    "rehype-raw": "^5.1.0",
-    "rehype-stringify": "^8.0.0",
     "remark-footnotes": "^3.0.0",
     "remark-gfm": "^1.0.0",
     "remark-parse": "^9.0.0",
@@ -89,11 +86,10 @@
     "sass": "^1.32.13",
     "shorthash": "^0.0.2",
     "slash": "^4.0.0",
-    "snowpack": "^3.5.4",
+    "snowpack": "^3.5.5",
     "source-map-support": "^0.5.19",
     "string-width": "^5.0.0",
     "tiny-glob": "^0.2.8",
-    "unified": "^9.2.1",
     "yargs-parser": "^20.2.7"
   },
   "devDependencies": {
@@ -103,13 +99,11 @@
     "@types/babel__parser": "^7.1.1",
     "@types/babel__traverse": "^7.11.1",
     "@types/estree": "0.0.46",
-    "@types/github-slugger": "^1.3.0",
     "@types/mime": "^2.0.3",
     "@types/node": "^14.14.31",
     "@types/sass": "^1.16.0",
     "@types/yargs-parser": "^20.2.0",
-    "astro-scripts": "0.0.1",
-    "unist-util-visit": "^3.1.0"
+    "astro-scripts": "0.0.1"
   },
   "engines": {
     "node": ">=14.0.0",

packages/astro/snowpack-plugin.cjs

@@ -33,7 +33,7 @@ module.exports = (snowpackConfig, { resolvePackageUrl, hmrPort, renderers, astro
       */
     async transform({contents, id, fileExt}) {
       if (fileExt === '.js' && /__astro_component\.js/g.test(id)) {
-        const rendererServerPackages = await Promise.all(renderers.map(({ server }) => resolvePackageUrl(server)));
+        const rendererServerPackages = renderers.map(({ server }) => server);
         const rendererClientPackages = await Promise.all(renderers.map(({ client }) => resolvePackageUrl(client)));
         const result = `${rendererServerPackages.map((pkg, i) => `import __renderer_${i} from "${pkg}";`).join('\n')}
 let __rendererSources = [${rendererClientPackages.map(pkg => `"${pkg}"`).join(', ')}];

packages/astro/src/compiler/codegen/index.ts

@@ -17,7 +17,8 @@ import { error, warn } from '../../logger.js';
 import { fetchContent } from './content.js';
 import { isFetchContent } from './utils.js';
 import { yellow } from 'kleur/colors';
-import { isComponentTag, renderMarkdown } from '../utils';
+import { isComponentTag } from '../utils';
+import { renderMarkdown } from '@astrojs/markdown-support';
 import { transform } from '../transform/index.js';
 import { PRISM_IMPORT } from '../transform/prism.js';
 
@@ -41,11 +42,6 @@ interface CodeGenOptions {
   fileID: string;
 }
 
-/** Format Astro internal import URL */
-function internalImport(internalPath: string) {
-  return `/_astro_internal/${internalPath}`;
-}
-
 /** Retrieve attributes from TemplateNode */
 function getAttributes(attrs: Attribute[]): Record<string, string> {
   let result: Record<string, string> = {};
@@ -169,7 +165,7 @@ function getComponentWrapper(_name: string, { url, importSpecifier }: ComponentI
   const importInfo = kind ? { componentUrl: getComponentUrl(), componentExport: getComponentExport() } : {};
   return {
     wrapper: `__astro_component(${name}, ${JSON.stringify({ hydrate: kind, displayName: _name, ...importInfo })})`,
-    wrapperImport: `import {__astro_component} from '${internalImport('__astro_component.js')}';`,
+    wrapperImport: `import {__astro_component} from 'astro/internal/__astro_component.js';`,
   };
 }
 

packages/astro/src/compiler/index.ts

@@ -3,7 +3,7 @@ import type { CompileResult, TransformResult } from '../@types/astro';
 import type { CompileOptions } from '../@types/compiler.js';
 
 import path from 'path';
-import { renderMarkdownWithFrontmatter } from './utils.js';
+import { renderMarkdownWithFrontmatter } from '@astrojs/markdown-support';
 
 import { parse } from 'astro-parser';
 import { transform } from './transform/index.js';
@@ -11,11 +11,6 @@ import { codegen } from './codegen/index.js';
 
 export { scopeRule } from './transform/postcss-scoped-styles/index.js';
 
-/** Return Astro internal import URL */
-function internalImport(internalPath: string) {
-  return `/_astro_internal/${internalPath}`;
-}
-
 interface ConvertAstroOptions {
   compileOptions: CompileOptions;
   filename: string;
@@ -119,7 +114,7 @@ import fetch from 'node-fetch';
 ${result.imports.join('\n')}
 
 // \`__render()\`: Render the contents of the Astro module.
-import { h, Fragment } from '${internalImport('h.js')}';
+import { h, Fragment } from 'astro/internal/h.js';
 const __astroRequestSymbol = Symbol('astro.request');
 async function __render(props, ...children) {
   const Astro = {

packages/astro/src/compiler/markdown/micromark.d.ts

@@ -1,14 +0,0 @@
-declare module '@silvenon/remark-smartypants' {
-  export default function (): any;
-}
-
-declare module 'mdast-util-mdx';
-declare module 'micromark-extension-mdxjs';
-
-declare module 'mdast-util-mdx/from-markdown.js' {
-  export default function (): any;
-}
-
-declare module 'mdast-util-mdx/to-markdown.js' {
-  export default function (): any;
-}

packages/astro/src/compiler/markdown/remark-mdx-lite.ts

@@ -1,30 +0,0 @@
-import syntaxMdxjs from 'micromark-extension-mdxjs';
-import { fromMarkdown, toMarkdown } from 'mdast-util-mdx';
-
-/**
- * Add the micromark and mdast extensions for MDX.js (JS aware MDX).
- *
- * @this {Processor}
- * @param {MdxOptions} [options]
- * @return {void}
- */
-export function remarkMdx(this: any, options: any) {
-  let data = this.data();
-
-  add('micromarkExtensions', syntaxMdxjs(options));
-  add('fromMarkdownExtensions', fromMarkdown);
-  add('toMarkdownExtensions', toMarkdown);
-
-  /**
-   * @param {string} field
-   * @param {unknown} value
-   */
-  function add(field: string, value: any) {
-    // Other extensions defined before this.
-    // Useful when externalizing.
-    /* c8 ignore next 2 */
-    // @ts-ignore Assume it’s an array.
-    if (data[field]) data[field].push(value);
-    else data[field] = [value];
-  }
-}

packages/astro/src/compiler/utils.ts

@@ -1,72 +1,3 @@
-import type { AstroMarkdownOptions } from '../@types/astro';
-import createCollectHeaders from './markdown/rehype-collect-headers.js';
-import scopedStyles from './markdown/remark-scoped-styles.js';
-import { remarkCodeBlock, rehypeCodeBlock } from './markdown/codeblock.js';
-import raw from 'rehype-raw';
-
-import unified from 'unified';
-import markdown from 'remark-parse';
-import markdownToHtml from 'remark-rehype';
-// import smartypants from '@silvenon/remark-smartypants';
-import rehypeStringify from 'rehype-stringify';
-
-export interface MarkdownRenderingOptions extends Partial<AstroMarkdownOptions> {
-  $?: {
-    scopedClassName: string | null;
-  };
-  mode: 'md' | 'astro-md';
-}
-
-/** Internal utility for rendering a full markdown file and extracting Frontmatter data */
-export async function renderMarkdownWithFrontmatter(contents: string, opts?: MarkdownRenderingOptions | null) {
-  // Dynamic import to ensure that "gray-matter" isn't built by Snowpack
-  const { default: matter } = await import('gray-matter');
-  const { data: frontmatter, content } = matter(contents);
-  const value = await renderMarkdown(content, { ...opts, mode: 'md' });
-  return { ...value, frontmatter };
-}
-
-/** Shared utility for rendering markdown */
-export async function renderMarkdown(content: string, opts?: MarkdownRenderingOptions | null) {
-  const { $: { scopedClassName = null } = {}, mode = 'astro-md', footnotes: useFootnotes = true, gfm: useGfm = true } = opts ?? {};
-  const { headers, rehypeCollectHeaders } = createCollectHeaders();
-
-  let parser = unified().use(markdown).use(remarkCodeBlock());
-
-  if (scopedClassName) {
-    parser = parser.use(scopedStyles(scopedClassName));
-  }
-
-  if (useGfm) {
-    const { default: gfm } = await import('remark-gfm');
-    parser = parser.use(gfm);
-  }
-
-  if (useFootnotes) {
-    const { default: footnotes } = await import('remark-footnotes');
-    parser = parser.use(footnotes);
-  }
-
-  let result: string;
-  try {
-    const vfile = await parser
-      .use(markdownToHtml, { allowDangerousHtml: true, passThrough: ['raw'] })
-      .use(raw)
-      .use(rehypeCollectHeaders)
-      .use(rehypeCodeBlock())
-      .use(rehypeStringify)
-      .process(content);
-    result = vfile.contents.toString();
-  } catch (err) {
-    throw err;
-  }
-
-  return {
-    astro: { headers, source: content },
-    content: result.toString(),
-  };
-}
-
 /** Is the given string a valid component tag */
 export function isComponentTag(tag: string) {
   return /^[A-Z]/.test(tag) || /^[a-z]+\./.test(tag);

packages/astro/src/external.ts

@@ -17,7 +17,7 @@ const isAstroRenderer = (name: string) => {
 
 // These packages should NOT be built by `esinstall`
 // But might not be explicit dependencies of `astro`
-const denyList = ['prismjs/components/index.js', '@vue/server-renderer', 'astro/dist/frontend/markdown.js'];
+const denyList = ['prismjs/components/index.js', '@vue/server-renderer', '@astrojs/markdown-support'];
 
 export default Object.keys(pkg.dependencies)
   // Filter out packages that should be loaded threw Snowpack

packages/astro/src/frontend/markdown.ts

@@ -1 +0,0 @@
-export { renderMarkdown } from '../compiler/utils';

packages/astro/src/internal/__astro_component.ts

@@ -56,7 +56,7 @@ async function generateHydrateScript({ renderer, astroId, props }: any, { hydrat
   const rendererSource = __rendererSources[__renderers.findIndex((r) => r === renderer)];
 
   const script = `<script type="module">
-import setup from '/_astro_internal/hydrate/${hydrate}.js';
+import setup from '/_astro_frontend/hydrate/${hydrate}.js';
 setup("${astroId}", async () => {
   const [{ ${componentExport.value}: Component }, { default: hydrate }] = await Promise.all([import("${componentUrl}"), import("${rendererSource}")]);
   return (el, children) => hydrate(el)(Component, ${serialize(props)}, children);

packages/astro/src/runtime.ts

@@ -310,7 +310,7 @@ async function createSnowpack(astroConfig: AstroConfig, options: CreateSnowpackO
   const { projectRoot, pages: pagesRoot, renderers = DEFAULT_RENDERERS } = astroConfig;
   const { env, mode, resolvePackageUrl, target } = options;
 
-  const internalPath = new URL('./frontend/', import.meta.url);
+  const frontendPath = new URL('./frontend/', import.meta.url);
   const resolveDependency = (dep: string) => resolve.sync(dep, { basedir: fileURLToPath(projectRoot) });
   const isHmrEnabled = mode === 'development' && target === 'backend';
 
@@ -329,7 +329,7 @@ async function createSnowpack(astroConfig: AstroConfig, options: CreateSnowpackO
   const mountOptions = {
     [fileURLToPath(pagesRoot)]: '/_astro/pages',
     ...(existsSync(astroConfig.public) ? { [fileURLToPath(astroConfig.public)]: '/' } : {}),
-    [fileURLToPath(internalPath)]: '/_astro_internal',
+    [fileURLToPath(frontendPath)]: '/_astro_frontend',
     [fileURLToPath(projectRoot)]: '/_astro', // must be last (greediest)
   };
 

packages/astro/src/search.ts

@@ -97,7 +97,7 @@ export function searchForPage(url: URL, pagesRoot: URL): SearchResult {
       statusCode: 200,
       location: {
         fileURL: new URL('./frontend/500.astro', import.meta.url),
-        snowpackURL: `/_astro_internal/500.astro.js`,
+        snowpackURL: `/_astro_frontend/500.astro.js`,
       },
       pathname: reqPath,
     };

packages/astro/test/astro-children.test.js

@@ -10,7 +10,6 @@ setupBuild(ComponentChildren, './fixtures/astro-children');
 
 ComponentChildren('Passes string children to framework components', async ({ runtime }) => {
   let result = await runtime.load('/strings');
-  console.log(JSON.stringify(result, null, 2));
   if (result.error) throw new Error(result);
 
   const $ = doc(result.contents);

packages/markdown-support/package.json

@@ -0,0 +1,23 @@
+{
+  "name": "@astrojs/markdown-support",
+  "version": "0.1.0",
+  "main": "./dist/index.js",
+  "type": "commonjs",
+  "exports": {
+    ".": "./dist/index.js"
+  },
+  "scripts": {
+    "prepublish": "yarn build",
+    "build": "astro-scripts build --format cjs \"src/**/*.ts\" && tsc -p tsconfig.json",
+    "dev": "astro-scripts dev \"src/**/*.ts\""
+  },
+  "devDependencies": {
+    "@types/github-slugger": "^1.3.0",
+    "github-slugger": "^1.3.0",
+    "rehype-parse": "^7.0.1",
+    "rehype-raw": "^5.1.0",
+    "rehype-stringify": "^8.0.0",
+    "unified": "^9.2.1",
+    "unist-util-visit": "^3.1.0"
+  }
+}

packages/markdown-support/src/index.ts

@@ -0,0 +1,69 @@
+import type { AstroMarkdownOptions } from './types';
+
+import createCollectHeaders from './rehype-collect-headers.js';
+import scopedStyles from './remark-scoped-styles.js';
+import { remarkCodeBlock, rehypeCodeBlock } from './codeblock.js';
+import raw from 'rehype-raw';
+
+import unified from 'unified';
+import markdown from 'remark-parse';
+import markdownToHtml from 'remark-rehype';
+// import smartypants from '@silvenon/remark-smartypants';
+import rehypeStringify from 'rehype-stringify';
+
+export interface MarkdownRenderingOptions extends Partial<AstroMarkdownOptions> {
+  $?: {
+    scopedClassName: string | null;
+  };
+  mode: 'md' | 'astro-md';
+}
+
+/** Internal utility for rendering a full markdown file and extracting Frontmatter data */
+export async function renderMarkdownWithFrontmatter(contents: string, opts?: MarkdownRenderingOptions | null) {
+  // Dynamic import to ensure that "gray-matter" isn't built by Snowpack
+  const { default: matter } = await import('gray-matter');
+  const { data: frontmatter, content } = matter(contents);
+  const value = await renderMarkdown(content, { ...opts, mode: 'md' });
+  return { ...value, frontmatter };
+}
+
+/** Shared utility for rendering markdown */
+export async function renderMarkdown(content: string, opts?: MarkdownRenderingOptions | null) {
+  const { $: { scopedClassName = null } = {}, mode = 'astro-md', footnotes: useFootnotes = true, gfm: useGfm = true } = opts ?? {};
+  const { headers, rehypeCollectHeaders } = createCollectHeaders();
+
+  let parser = unified().use(markdown).use(remarkCodeBlock());
+
+  if (scopedClassName) {
+    parser = parser.use(scopedStyles(scopedClassName));
+  }
+
+  if (useGfm) {
+    const { default: gfm } = await import('remark-gfm');
+    parser = parser.use(gfm);
+  }
+
+  if (useFootnotes) {
+    const { default: footnotes } = await import('remark-footnotes');
+    parser = parser.use(footnotes);
+  }
+
+  let result: string;
+  try {
+    const vfile = await parser
+      .use(markdownToHtml, { allowDangerousHtml: true, passThrough: ['raw'] })
+      .use(raw)
+      .use(rehypeCollectHeaders)
+      .use(rehypeCodeBlock())
+      .use(rehypeStringify)
+      .process(content);
+    result = vfile.contents.toString();
+  } catch (err) {
+    throw err;
+  }
+
+  return {
+    astro: { headers, source: content },
+    content: result.toString(),
+  };
+}

packages/markdown-support/src/types.ts

@@ -0,0 +1,6 @@
+export interface AstroMarkdownOptions {
+  /** Enable or disable footnotes syntax extension */
+  footnotes: boolean;
+  /** Enable or disable GitHub-flavored Markdown syntax extension */
+  gfm: boolean;
+}

packages/markdown-support/tsconfig.json

@@ -0,0 +1,10 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "include": ["src"],
+  "compilerOptions": {
+    "allowJs": true,
+    "target": "ES2020",
+    "module": "CommonJS",
+    "outDir": "./dist"
+  }
+}

yarn.lock

@@ -8906,10 +8906,10 @@ smartwrap@^1.2.3:
     wcwidth "^1.0.1"
     yargs "^15.1.0"
 
-snowpack@^3.5.4:
-  version "3.5.4"
-  resolved "https://registry.yarnpkg.com/snowpack/-/snowpack-3.5.4.tgz#6f341714825f4080aeb2f7aa40a778c04c55934d"
-  integrity sha512-knm8Xv1Zh1/UW0jLuqu2f0VARN5WjZVeRWsFoSzdoXTsXaxctROVungRRirr++m4KhzHC32EG9K4+y28p1nknA==
+snowpack@^3.5.5:
+  version "3.5.5"
+  resolved "https://registry.yarnpkg.com/snowpack/-/snowpack-3.5.5.tgz#80729286d5f1d7db57e13aff44af750888155f3b"
+  integrity sha512-Ij+ETrYo6S1/AQ4O+AVixbXmbBHb+iOzmM9WWLLIu9QE4U12Im05lWobqbEBBXG9V4dNwQKdGe5rP1d40JnglA==
   dependencies:
     cli-spinners "^2.5.0"
     default-browser-id "^2.0.0"