From ad9c3b1d8dbf1c3aff75497271347ed36ea38a0b Mon Sep 17 00:00:00 2001 From: Matthew Phillips Date: Fri, 9 Apr 2021 14:09:13 -0400 Subject: [PATCH] Parse inner JSX as Astro (#67) * Parse inner JSX as Astro This completes the compiler changes, updating the parser so that it parses inner "JSX" as Astro. It does this by finding the start and end of HTML tags and feeds that back into the parser. The result is a structure like this: ``` { type: 'MustacheTag', expression: [ { type: 'Expression', codeStart: 'colors.map(color => (', codeEnd: '}}' children: [ { type: 'Fragment', children: [ { type: 'Element', name: 'div' } ] } ] } ] } ``` There is a new Node type, `Expression`. Note that `MustacheTag` remains in the tree, all it contains is an Expression though. I could spend some time trying to remove it, there's just a few places that expect it to exist. * Update import to the transform * Transform prism components into expressions --- src/@types/{optimizer.ts => transformer.ts} | 4 +- src/build/bundle.ts | 4 +- src/compiler/codegen.ts | 67 ++--- src/compiler/index.ts | 8 +- .../{optimize => transform}/doctype.ts | 6 +- src/compiler/{optimize => transform}/index.ts | 36 +-- .../{optimize => transform}/module-scripts.ts | 6 +- .../postcss-scoped-styles/index.ts | 0 src/compiler/{optimize => transform}/prism.ts | 11 +- .../{optimize => transform}/styles.ts | 10 +- src/parser/interfaces.ts | 12 +- src/parser/parse/acorn.ts | 42 --- src/parser/parse/read/context.ts | 2 +- src/parser/parse/read/expression.ts | 279 ++++++++++++++++-- src/parser/parse/read/script.ts | 4 +- src/parser/parse/state/mustache.ts | 4 +- src/parser/parse/state/tag.ts | 4 +- test/astro-expr.test.js | 42 ++- test/astro-scoped-styles.test.js | 2 +- .../astro-expr/astro/components/Color.jsx | 2 +- .../astro-expr/astro/pages/index.astro | 2 +- .../astro/pages/line-comments.astro | 17 ++ .../astro/pages/multiline-comments.astro | 16 + .../astro-expr/astro/pages/strings.astro | 16 + 24 files changed, 432 insertions(+), 164 deletions(-) rename src/@types/{optimizer.ts => transformer.ts} (89%) rename src/compiler/{optimize => transform}/doctype.ts (88%) rename src/compiler/{optimize => transform}/index.ts (67%) rename src/compiler/{optimize => transform}/module-scripts.ts (86%) rename src/compiler/{optimize => transform}/postcss-scoped-styles/index.ts (100%) rename src/compiler/{optimize => transform}/prism.ts (85%) rename src/compiler/{optimize => transform}/styles.ts (96%) delete mode 100644 src/parser/parse/acorn.ts create mode 100644 test/fixtures/astro-expr/astro/pages/line-comments.astro create mode 100644 test/fixtures/astro-expr/astro/pages/multiline-comments.astro create mode 100644 test/fixtures/astro-expr/astro/pages/strings.astro diff --git a/src/@types/optimizer.ts b/src/@types/transformer.ts similarity index 89% rename from src/@types/optimizer.ts rename to src/@types/transformer.ts index b6459ab51..7f4167558 100644 --- a/src/@types/optimizer.ts +++ b/src/@types/transformer.ts @@ -8,7 +8,7 @@ export interface NodeVisitor { leave?: VisitorFn; } -export interface Optimizer { +export interface Transformer { visitors?: { html?: Record; css?: Record; @@ -16,7 +16,7 @@ export interface Optimizer { finalize: () => Promise; } -export interface OptimizeOptions { +export interface TransformOptions { compileOptions: CompileOptions; filename: string; fileID: string; diff --git a/src/build/bundle.ts b/src/build/bundle.ts index ba1b8f2c2..b55828c2e 100644 --- a/src/build/bundle.ts +++ b/src/build/bundle.ts @@ -7,7 +7,7 @@ import type { LogOptions } from '../logger'; import esbuild from 'esbuild'; import { promises as fsPromises } from 'fs'; import { parse } from '../parser/index.js'; -import { optimize } from '../compiler/optimize/index.js'; +import { transform } from '../compiler/transform/index.js'; import { getAttrValue } from '../ast.js'; import { walk } from 'estree-walker'; import babelParser from '@babel/parser'; @@ -86,7 +86,7 @@ export async function collectDynamicImports(filename: URL, { astroConfig, loggin return imports; } - await optimize(ast, { + await transform(ast, { filename: filename.pathname, fileID: '', compileOptions: { diff --git a/src/compiler/codegen.ts b/src/compiler/codegen.ts index 59cc2c702..8bcf3f49d 100644 --- a/src/compiler/codegen.ts +++ b/src/compiler/codegen.ts @@ -78,9 +78,10 @@ function getAttributes(attrs: Attribute[]): Record { continue; } switch (val.type) { - case 'MustacheTag': - result[attr.name] = '(' + val.content + ')'; + case 'MustacheTag': { + result[attr.name] = '(' + val.expression.codeStart + ')'; continue; + } case 'Text': result[attr.name] = JSON.stringify(getTextFromAttribute(val)); continue; @@ -93,13 +94,21 @@ function getAttributes(attrs: Attribute[]): Record { /** Get value from a TemplateNode Attribute (text attributes only!) */ function getTextFromAttribute(attr: any): string { - if (attr.raw !== undefined) { - return attr.raw; + switch(attr.type) { + case 'Text': { + if (attr.raw !== undefined) { + return attr.raw; + } + if (attr.data !== undefined) { + return attr.data; + } + break; + } + case 'MustacheTag': { + return attr.expression.codeStart; + } } - if (attr.data !== undefined) { - return attr.data; - } - throw new Error('UNKNOWN attr'); + throw new Error(`Unknown attribute type ${attr.type}`); } /** Convert TemplateNode attributes to string */ @@ -238,7 +247,7 @@ function getComponentWrapper(_name: string, { type, plugin, url }: ComponentInfo } } -/** Evaluate mustache expression (safely) */ +/** Evaluate expression (safely) */ function compileExpressionSafe(raw: string): string { let { code } = transformSync(raw, { loader: 'tsx', @@ -468,33 +477,19 @@ function compileHtml(enterNode: TemplateNode, state: CodegenState, compileOption walk(enterNode, { enter(node: TemplateNode) { switch (node.type) { - case 'MustacheTag': - let code = compileExpressionSafe(node.content); - - let matches: RegExpExecArray[] = []; - let match: RegExpExecArray | null | undefined; - const H_COMPONENT_SCANNER = /h\(['"]?([A-Z].*?)['"]?,/gs; - const regex = new RegExp(H_COMPONENT_SCANNER); - while ((match = regex.exec(code))) { - matches.push(match); + case 'Expression': { + let child = ''; + if(node.children!.length) { + child = compileHtml(node.children![0], state, compileOptions); } - for (const astroComponent of matches.reverse()) { - const name = astroComponent[1]; - const [componentName, componentKind] = name.split(':'); - if (!components[componentName]) { - throw new Error(`Unknown Component: ${componentName}`); - } - const { wrapper, wrapperImport } = getComponentWrapper(name, components[componentName], { astroConfig, dynamicImports, filename }); - if (wrapperImport) { - importExportStatements.add(wrapperImport); - } - if (wrapper !== name) { - code = code.slice(0, astroComponent.index + 2) + wrapper + code.slice(astroComponent.index + astroComponent[0].length - 1); - } - } - outSource += `,(${code.trim().replace(/\;$/, '')})`; + let raw = node.codeStart + child + node.codeEnd; + // TODO Do we need to compile this now, or should we compile the entire module at the end? + let code = compileExpressionSafe(raw).trim().replace(/\;$/, ''); + outSource += `,(${code})`; this.skip(); - return; + break; + } + case 'MustacheTag': case 'Comment': return; case 'Fragment': @@ -557,11 +552,11 @@ function compileHtml(enterNode: TemplateNode, state: CodegenState, compileOption leave(node, parent, prop, index) { switch (node.type) { case 'Text': - case 'MustacheTag': case 'Attribute': case 'Comment': - return; case 'Fragment': + case 'Expression': + case 'MustacheTag': return; case 'Slot': case 'Head': diff --git a/src/compiler/index.ts b/src/compiler/index.ts index d33527b9b..db50abec8 100644 --- a/src/compiler/index.ts +++ b/src/compiler/index.ts @@ -11,7 +11,7 @@ import { parse } from '../parser/index.js'; import { createMarkdownHeadersCollector } from './markdown/micromark-collect-headers.js'; import { encodeMarkdown } from './markdown/micromark-encode.js'; import { encodeAstroMdx } from './markdown/micromark-mdx-astro.js'; -import { optimize } from './optimize/index.js'; +import { transform } from './transform/index.js'; import { codegen } from './codegen.js'; /** Return Astro internal import URL */ @@ -29,7 +29,7 @@ interface ConvertAstroOptions { * .astro -> .jsx * Core function processing .astro files. Initiates all 3 phases of compilation: * 1. Parse - * 2. Optimize + * 2. Transform * 3. Codegen */ async function convertAstroToJsx(template: string, opts: ConvertAstroOptions): Promise { @@ -40,8 +40,8 @@ async function convertAstroToJsx(template: string, opts: ConvertAstroOptions): P filename, }); - // 2. Optimize the AST - await optimize(ast, opts); + // 2. Transform the AST + await transform(ast, opts); // 3. Turn AST into JSX return await codegen(ast, opts); diff --git a/src/compiler/optimize/doctype.ts b/src/compiler/transform/doctype.ts similarity index 88% rename from src/compiler/optimize/doctype.ts rename to src/compiler/transform/doctype.ts index 176880c08..d19b01f81 100644 --- a/src/compiler/optimize/doctype.ts +++ b/src/compiler/transform/doctype.ts @@ -1,7 +1,7 @@ -import { Optimizer } from '../../@types/optimizer'; +import { Transformer } from '../../@types/transformer'; -/** Optimize tg */ -export default function (_opts: { filename: string; fileID: string }): Optimizer { +/** Transform tg */ +export default function (_opts: { filename: string; fileID: string }): Transformer { let hasDoctype = false; return { diff --git a/src/compiler/optimize/index.ts b/src/compiler/transform/index.ts similarity index 67% rename from src/compiler/optimize/index.ts rename to src/compiler/transform/index.ts index fcbd6e950..6a81b92b0 100644 --- a/src/compiler/optimize/index.ts +++ b/src/compiler/transform/index.ts @@ -1,13 +1,13 @@ import type { Ast, TemplateNode } from '../../parser/interfaces'; -import type { NodeVisitor, OptimizeOptions, Optimizer, VisitorFn } from '../../@types/optimizer'; +import type { NodeVisitor, TransformOptions, Transformer, VisitorFn } from '../../@types/transformer'; import { walk } from 'estree-walker'; -// Optimizers -import optimizeStyles from './styles.js'; -import optimizeDoctype from './doctype.js'; -import optimizeModuleScripts from './module-scripts.js'; -import optimizeCodeBlocks from './prism.js'; +// Transformers +import transformStyles from './styles.js'; +import transformDoctype from './doctype.js'; +import transformModuleScripts from './module-scripts.js'; +import transformCodeBlocks from './prism.js'; interface VisitorCollection { enter: Map; @@ -24,23 +24,23 @@ function addVisitor(visitor: NodeVisitor, collection: VisitorCollection, nodeNam collection[event].set(nodeName, visitors); } -/** Compile visitor actions from optimizer */ -function collectVisitors(optimizer: Optimizer, htmlVisitors: VisitorCollection, cssVisitors: VisitorCollection, finalizers: Array<() => Promise>) { - if (optimizer.visitors) { - if (optimizer.visitors.html) { - for (const [nodeName, visitor] of Object.entries(optimizer.visitors.html)) { +/** Compile visitor actions from transformer */ +function collectVisitors(transformer: Transformer, htmlVisitors: VisitorCollection, cssVisitors: VisitorCollection, finalizers: Array<() => Promise>) { + if (transformer.visitors) { + if (transformer.visitors.html) { + for (const [nodeName, visitor] of Object.entries(transformer.visitors.html)) { addVisitor(visitor, htmlVisitors, nodeName, 'enter'); addVisitor(visitor, htmlVisitors, nodeName, 'leave'); } } - if (optimizer.visitors.css) { - for (const [nodeName, visitor] of Object.entries(optimizer.visitors.css)) { + if (transformer.visitors.css) { + for (const [nodeName, visitor] of Object.entries(transformer.visitors.css)) { addVisitor(visitor, cssVisitors, nodeName, 'enter'); addVisitor(visitor, cssVisitors, nodeName, 'leave'); } } } - finalizers.push(optimizer.finalize); + finalizers.push(transformer.finalize); } /** Utility for formatting visitors */ @@ -74,17 +74,17 @@ function walkAstWithVisitors(tmpl: TemplateNode, collection: VisitorCollection) } /** - * Optimize + * Transform * Step 2/3 in Astro SSR. - * Optimize is the point at which we mutate the AST before sending off to + * Transform is the point at which we mutate the AST before sending off to * Codegen, and then to Snowpack. In some ways, it‘s a preprocessor. */ -export async function optimize(ast: Ast, opts: OptimizeOptions) { +export async function transform(ast: Ast, opts: TransformOptions) { const htmlVisitors = createVisitorCollection(); const cssVisitors = createVisitorCollection(); const finalizers: Array<() => Promise> = []; - const optimizers = [optimizeStyles(opts), optimizeDoctype(opts), optimizeModuleScripts(opts), optimizeCodeBlocks(ast.module)]; + const optimizers = [transformStyles(opts), transformDoctype(opts), transformModuleScripts(opts), transformCodeBlocks(ast.module)]; for (const optimizer of optimizers) { collectVisitors(optimizer, htmlVisitors, cssVisitors, finalizers); diff --git a/src/compiler/optimize/module-scripts.ts b/src/compiler/transform/module-scripts.ts similarity index 86% rename from src/compiler/optimize/module-scripts.ts rename to src/compiler/transform/module-scripts.ts index 9d4949215..aff1ec4f6 100644 --- a/src/compiler/optimize/module-scripts.ts +++ b/src/compiler/transform/module-scripts.ts @@ -1,11 +1,11 @@ -import type { Optimizer } from '../../@types/optimizer'; +import type { Transformer } from '../../@types/transformer'; import type { CompileOptions } from '../../@types/compiler'; import path from 'path'; import { getAttrValue, setAttrValue } from '../../ast.js'; -/** Optimize '; diff --git a/src/parser/parse/state/mustache.ts b/src/parser/parse/state/mustache.ts index 8ffac4f85..79372d8d9 100644 --- a/src/parser/parse/state/mustache.ts +++ b/src/parser/parse/state/mustache.ts @@ -397,7 +397,7 @@ export default function mustache(parser: Parser) { // }); throw new Error('@debug not yet supported'); } else { - const content = read_expression(parser); + const expression = read_expression(parser); parser.allow_whitespace(); parser.eat('}', true); @@ -407,7 +407,7 @@ export default function mustache(parser: Parser) { start, end: parser.index, type: 'MustacheTag', - content, + expression, }); } } diff --git a/src/parser/parse/state/tag.ts b/src/parser/parse/state/tag.ts index bacaffdef..a8b919a49 100644 --- a/src/parser/parse/state/tag.ts +++ b/src/parser/parse/state/tag.ts @@ -549,7 +549,7 @@ function read_sequence(parser: Parser, done: () => boolean): TemplateNode[] { flush(); parser.allow_whitespace(); - const content = read_expression(parser); + const expression = read_expression(parser); parser.allow_whitespace(); parser.eat('}', true); @@ -557,7 +557,7 @@ function read_sequence(parser: Parser, done: () => boolean): TemplateNode[] { start: index, end: parser.index, type: 'MustacheTag', - content, + expression, }); current_chunk = { diff --git a/test/astro-expr.test.js b/test/astro-expr.test.js index 9c73d719f..689c32ced 100644 --- a/test/astro-expr.test.js +++ b/test/astro-expr.test.js @@ -10,9 +10,47 @@ setup(Expressions, './fixtures/astro-expr'); Expressions('Can load page', async ({ runtime }) => { const result = await runtime.load('/'); - console.log(result); assert.equal(result.statusCode, 200); - console.log(result.contents); + + const $ = doc(result.contents); + + for(let col of ['red', 'yellow', 'blue']) { + assert.equal($('#' + col).length, 1); + } +}); + +Expressions('Ignores characters inside of strings', async ({ runtime }) => { + const result = await runtime.load('/strings'); + + assert.equal(result.statusCode, 200); + + const $ = doc(result.contents); + + for(let col of ['red', 'yellow', 'blue']) { + assert.equal($('#' + col).length, 1); + } +}); + +Expressions('Ignores characters inside of line comments', async ({ runtime }) => { + const result = await runtime.load('/line-comments'); + assert.equal(result.statusCode, 200); + + const $ = doc(result.contents); + + for(let col of ['red', 'yellow', 'blue']) { + assert.equal($('#' + col).length, 1); + } +}); + +Expressions('Ignores characters inside of multiline comments', async ({ runtime }) => { + const result = await runtime.load('/multiline-comments'); + assert.equal(result.statusCode, 200); + + const $ = doc(result.contents); + + for(let col of ['red', 'yellow', 'blue']) { + assert.equal($('#' + col).length, 1); + } }); Expressions.run(); diff --git a/test/astro-scoped-styles.test.js b/test/astro-scoped-styles.test.js index 5c01a31fb..295668b84 100644 --- a/test/astro-scoped-styles.test.js +++ b/test/astro-scoped-styles.test.js @@ -1,6 +1,6 @@ import { suite } from 'uvu'; import * as assert from 'uvu/assert'; -import { scopeRule } from '../lib/compiler/optimize/postcss-scoped-styles/index.js'; +import { scopeRule } from '../lib/compiler/transform/postcss-scoped-styles/index.js'; const ScopedStyles = suite('Astro PostCSS Scoped Styles Plugin'); diff --git a/test/fixtures/astro-expr/astro/components/Color.jsx b/test/fixtures/astro-expr/astro/components/Color.jsx index 13a5049aa..c2681cc9b 100644 --- a/test/fixtures/astro-expr/astro/components/Color.jsx +++ b/test/fixtures/astro-expr/astro/components/Color.jsx @@ -1,5 +1,5 @@ import { h } from 'preact'; export default function({ name }) { - return
{name}
+ return
{name}
} \ No newline at end of file diff --git a/test/fixtures/astro-expr/astro/pages/index.astro b/test/fixtures/astro-expr/astro/pages/index.astro index f0a4d2ab0..50af05d93 100644 --- a/test/fixtures/astro-expr/astro/pages/index.astro +++ b/test/fixtures/astro-expr/astro/pages/index.astro @@ -3,7 +3,7 @@ import Color from '../components/Color.jsx'; let title = 'My Site'; -const colors = ['red', 'yellow', 'blue'] +const colors = ['red', 'yellow', 'blue']; --- diff --git a/test/fixtures/astro-expr/astro/pages/line-comments.astro b/test/fixtures/astro-expr/astro/pages/line-comments.astro new file mode 100644 index 000000000..2fb7bf643 --- /dev/null +++ b/test/fixtures/astro-expr/astro/pages/line-comments.astro @@ -0,0 +1,17 @@ +--- +let title = 'My App'; + +let colors = ['red', 'yellow', 'blue']; +--- + + + + {title} + + + {colors.map(color => ( + // foo < > < } +
color
+ ))} + + \ No newline at end of file diff --git a/test/fixtures/astro-expr/astro/pages/multiline-comments.astro b/test/fixtures/astro-expr/astro/pages/multiline-comments.astro new file mode 100644 index 000000000..5c7016ee8 --- /dev/null +++ b/test/fixtures/astro-expr/astro/pages/multiline-comments.astro @@ -0,0 +1,16 @@ +--- +let title = 'My App'; + +let colors = ['red', 'yellow', 'blue']; +--- + + + + {title} + + + {colors.map(color => ( + /* foo < > < } */
color
+ ))} + + \ No newline at end of file diff --git a/test/fixtures/astro-expr/astro/pages/strings.astro b/test/fixtures/astro-expr/astro/pages/strings.astro new file mode 100644 index 000000000..712df6120 --- /dev/null +++ b/test/fixtures/astro-expr/astro/pages/strings.astro @@ -0,0 +1,16 @@ +--- +let title = 'My App'; + +let colors = ['red', 'yellow', 'blue']; +--- + + + + {title} + + + {colors.map(color => ( + 'foo < > < }' &&
color
+ ))} + + \ No newline at end of file