diff --git a/.changeset/great-cats-train.md b/.changeset/great-cats-train.md new file mode 100644 index 000000000..35ce4a4df --- /dev/null +++ b/.changeset/great-cats-train.md @@ -0,0 +1,5 @@ +--- +'astro': patch +--- + +Add CSS bundling diff --git a/examples/blog/package.json b/examples/blog/package.json index 15b0209b1..20f97487c 100644 --- a/examples/blog/package.json +++ b/examples/blog/package.json @@ -8,7 +8,7 @@ "astro-dev": "nodemon --delay 0.5 -w ../../packages/astro/dist -x '../../packages/astro/astro.mjs dev'" }, "devDependencies": { - "astro": "0.0.11", + "astro": "^0.0.12", "nodemon": "^2.0.7" }, "snowpack": { diff --git a/examples/kitchen-sink/package.json b/examples/kitchen-sink/package.json index ba35948fb..c2a99adb7 100644 --- a/examples/kitchen-sink/package.json +++ b/examples/kitchen-sink/package.json @@ -8,7 +8,7 @@ "astro-dev": "nodemon --delay 0.5 -w ../../packages/astro/dist -x '../../packages/astro/astro.mjs dev'" }, "devDependencies": { - "astro": "0.0.11", + "astro": "^0.0.12", "nodemon": "^2.0.7" }, "snowpack": { diff --git a/examples/portfolio/package.json b/examples/portfolio/package.json index b3567282d..ffe44b97a 100644 --- a/examples/portfolio/package.json +++ b/examples/portfolio/package.json @@ -8,7 +8,7 @@ "astro-dev": "nodemon --delay 0.5 -w ../../packages/astro/dist -x '../../packages/astro/astro.mjs dev'" }, "devDependencies": { - "astro": "0.0.11" + "astro": "^0.0.12" }, "snowpack": { "workspaceRoot": "../.." diff --git a/examples/snowpack/package.json b/examples/snowpack/package.json index 0b7dbfd90..8a095b13f 100644 --- a/examples/snowpack/package.json +++ b/examples/snowpack/package.json @@ -12,7 +12,7 @@ "lint": "prettier --check \"src/**/*.js\"" }, "dependencies": { - "astro": "0.0.11", + "astro": "^0.0.12", "date-fns": "^2.19.0", "deepmerge": "^4.2.2", "docsearch.js": "^2.6.3", @@ -26,7 +26,7 @@ "@11ty/eleventy-plugin-syntaxhighlight": "^3.0.4", "@contentful/rich-text-html-renderer": "^14.1.2", "@contentful/rich-text-types": "^14.1.2", - "astro": "0.0.11", + "astro": "^0.0.12", "eleventy-plugin-nesting-toc": "^1.2.0", "luxon": "^1.25.0", "markdown-it": "^12.0.2", diff --git a/examples/tailwindcss/package.json b/examples/tailwindcss/package.json index 7889890db..c00350964 100644 --- a/examples/tailwindcss/package.json +++ b/examples/tailwindcss/package.json @@ -8,7 +8,7 @@ "astro-dev": "nodemon --delay 0.5 -w ../../packages/astro/dist -x '../../packages/astro/astro.mjs dev'" }, "devDependencies": { - "astro": "0.0.11", + "astro": "^0.0.12", "tailwindcss": "^2.1.1" }, "snowpack": { diff --git a/package.json b/package.json index 314e216fd..9ad0ee961 100644 --- a/package.json +++ b/package.json @@ -32,18 +32,18 @@ "devDependencies": { "@typescript-eslint/eslint-plugin": "^4.22.0", "@typescript-eslint/parser": "^4.18.0", - "cheerio": "^1.0.0-rc.5", + "cheerio": "^1.0.0-rc.6", "cheerio-select-tmp": "^0.1.1", "del": "^6.0.0", + "esbuild": "^0.11.17", "eslint": "^7.25.0", "eslint-config-prettier": "^8.3.0", "eslint-plugin-prettier": "^3.4.0", "execa": "^5.0.0", "lerna": "^4.0.0", "prettier": "^2.2.1", - "tiny-glob": "^0.2.8", - "esbuild": "^0.11.17", "svelte": "^3.38.0", + "tiny-glob": "^0.2.8", "typescript": "^4.2.4", "uvu": "^0.5.1" }, diff --git a/packages/astro/package.json b/packages/astro/package.json index 81c5e0645..59505a3f9 100644 --- a/packages/astro/package.json +++ b/packages/astro/package.json @@ -43,8 +43,8 @@ "astro-parser": "0.0.9", "astro-prism": "0.0.2", "autoprefixer": "^10.2.5", - "cheerio": "^1.0.0-rc.5", - "domhandler": "^4.1.0", + "cheerio": "^1.0.0-rc.6", + "del": "^6.0.0", "es-module-lexer": "^0.4.1", "esbuild": "^0.10.1", "estree-walker": "^3.0.0", @@ -62,6 +62,7 @@ "micromark-extension-gfm": "^0.3.3", "micromark-extension-mdx-expression": "^0.3.2", "micromark-extension-mdx-jsx": "^0.3.3", + "mime": "^2.5.2", "moize": "^6.0.1", "node-fetch": "^2.6.1", "picomatch": "^2.2.3", @@ -76,6 +77,7 @@ "rollup": "^2.43.1", "rollup-plugin-terser": "^7.0.2", "sass": "^1.32.8", + "shorthash": "^0.0.2", "snowpack": "^3.3.7", "source-map-support": "^0.5.19", "string-width": "^5.0.0", @@ -92,6 +94,7 @@ "@types/babel__traverse": "^7.11.1", "@types/estree": "0.0.46", "@types/github-slugger": "^1.3.0", + "@types/mime": "^2.0.3", "@types/node": "^14.14.31", "@types/react": "^17.0.3", "@types/react-dom": "^17.0.2", diff --git a/packages/astro/src/@types/astro.ts b/packages/astro/src/@types/astro.ts index 049105970..df7dbc4d6 100644 --- a/packages/astro/src/@types/astro.ts +++ b/packages/astro/src/@types/astro.ts @@ -63,6 +63,36 @@ export type RuntimeMode = 'development' | 'production'; export type Params = Record; +/** Entire output of `astro build`, stored in memory */ +export interface BuildOutput { + [dist: string]: BuildFile; +} + +export interface BuildFile { + /** The original location. Needed for code frame errors. */ + srcPath: URL; + /** File contents */ + contents: string | Buffer; + /** File content type (to determine encoding, etc) */ + contentType: string; + /** Encoding */ + encoding?: 'utf8'; +} + +/** Mapping of every URL and its required assets. All URLs are absolute relative to the project. */ +export type BundleMap = { + [pageUrl: string]: PageDependencies; +}; + +export interface PageDependencies { + /** JavaScript files needed for page. No distinction between blocking/non-blocking or sync/async. */ + js: Set; + /** CSS needed for page, whether imported via , JS, or Astro component. */ + css: Set; + /** Images needed for page. Can be loaded via CSS, , or otherwise. */ + images: Set; +} + export interface CreateCollection { data: ({ params }: { params: Params }) => T[]; routes?: Params[]; diff --git a/packages/astro/src/@types/shorthash.d.ts b/packages/astro/src/@types/shorthash.d.ts new file mode 100644 index 000000000..02eb5ba51 --- /dev/null +++ b/packages/astro/src/@types/shorthash.d.ts @@ -0,0 +1,5 @@ +declare module 'shorthash' { + function unique(string: string): string; + + export default { unique }; +} diff --git a/packages/astro/src/build.ts b/packages/astro/src/build.ts index a09043db7..9661b5cc4 100644 --- a/packages/astro/src/build.ts +++ b/packages/astro/src/build.ts @@ -1,40 +1,24 @@ import 'source-map-support/register.js'; -import type { AstroConfig, RuntimeMode } from './@types/astro'; +import type { AstroConfig, BundleMap, BuildOutput, RuntimeMode, PageDependencies } from './@types/astro'; import type { LogOptions } from './logger'; -import type { AstroRuntime, LoadResult } from './runtime'; -import { existsSync, promises as fsPromises } from 'fs'; -import { bold, green, yellow } from 'kleur/colors'; +import fs from 'fs'; import path from 'path'; -import cheerio from 'cheerio'; import { fileURLToPath } from 'url'; +import { performance } from 'perf_hooks'; +import cheerio from 'cheerio'; +import del from 'del'; +import { bold, green, yellow } from 'kleur/colors'; +import mime from 'mime'; import { fdir } from 'fdir'; -import { defaultLogDestination, error, info, trapWarn } from './logger.js'; -import { createRuntime } from './runtime.js'; -import { bundle, collectDynamicImports } from './build/bundle.js'; -import { generateRSS } from './build/rss.js'; +import { bundleCSS } from './build/bundle/css.js'; +import { bundleJS, collectJSImports } from './build/bundle/js'; +import { buildCollectionPage, buildStaticPage, getPageType } from './build/page.js'; import { generateSitemap } from './build/sitemap.js'; -import { collectStatics } from './build/static.js'; -import { canonicalURL } from './build/util.js'; -import { createURLStats, mapBundleStatsToURLStats, logURLStats } from './build/stats.js'; - -const { mkdir, readFile, writeFile } = fsPromises; - -interface PageBuildOptions { - astroRoot: URL; - dist: URL; - filepath: URL; - runtime: AstroRuntime; - site?: string; - sitemap: boolean; - statics: Set; -} - -interface PageResult { - canonicalURLs: string[]; - rss?: string; - statusCode: number; -} +import { logURLStats, collectBundleStats, mapBundleStatsToURLStats } from './build/stats.js'; +import { getDistPath, sortSet, stopTimer } from './build/util.js'; +import { debug, defaultLogDestination, error, info, trapWarn } from './logger.js'; +import { createRuntime } from './runtime.js'; const logging: LogOptions = { level: 'debug', @@ -51,145 +35,20 @@ async function allPages(root: URL) { return files as string[]; } -/** Utility for merging two Set()s */ -function mergeSet(a: Set, b: Set) { - for (let str of b) { - a.add(str); - } - return a; -} - -/** Utility for writing to file (async) */ -async function writeFilep(outPath: URL, bytes: string | Buffer, encoding: 'utf8' | null) { - const outFolder = new URL('./', outPath); - await mkdir(outFolder, { recursive: true }); - await writeFile(outPath, bytes, encoding || 'binary'); -} - -interface WriteResultOptions { - srcPath: string; - result: LoadResult; - outPath: URL; - encoding: null | 'utf8'; -} - -/** Utility for writing a build result to disk */ -async function writeResult({ srcPath, result, outPath, encoding }: WriteResultOptions) { - if (result.statusCode === 500 || result.statusCode === 404) { - error(logging, 'build', ` Failed to build ${srcPath}\n${' '.repeat(9)}`, result.error?.message ?? `Unexpected load result (${result.statusCode})`); - } else if (result.statusCode !== 200) { - error(logging, 'build', ` Failed to build ${srcPath}\n${' '.repeat(9)}`, `Unexpected load result (${result.statusCode}) for ${fileURLToPath(outPath)}`); - } else { - const bytes = result.contents; - await writeFilep(outPath, bytes, encoding); - } -} - -/** Collection utility */ -function getPageType(filepath: URL): 'collection' | 'static' { - if (/\$[^.]+.astro$/.test(filepath.pathname)) return 'collection'; - return 'static'; -} - -/** Build collection */ -async function buildCollectionPage({ astroRoot, dist, filepath, runtime, site, statics }: PageBuildOptions): Promise { - const rel = path.relative(fileURLToPath(astroRoot) + '/pages', fileURLToPath(filepath)); // pages/index.astro - const pagePath = `/${rel.replace(/\$([^.]+)\.astro$/, '$1')}`; - const srcPath = fileURLToPath(new URL('pages/' + rel, astroRoot)); - const builtURLs = new Set(); // !important: internal cache that prevents building the same URLs - - /** Recursively build collection URLs */ - async function loadCollection(url: string): Promise { - if (builtURLs.has(url)) return; // this stops us from recursively building the same pages over and over - const result = await runtime.load(url); - builtURLs.add(url); - if (result.statusCode === 200) { - const outPath = new URL('./' + url + '/index.html', dist); - await writeResult({ srcPath, result, outPath, encoding: 'utf8' }); - mergeSet(statics, collectStatics(result.contents.toString('utf8'))); - } - return result; - } - - const result = (await loadCollection(pagePath)) as LoadResult; - - if (result.statusCode >= 500) { - throw new Error((result as any).error); - } - if (result.statusCode === 200 && !result.collectionInfo) { - throw new Error(`[${rel}]: Collection page must export createCollection() function`); - } - - let rss: string | undefined; - - // note: for pages that require params (/tag/:tag), we will get a 404 but will still get back collectionInfo that tell us what the URLs should be - if (result.collectionInfo) { - // build subsequent pages - await Promise.all( - [...result.collectionInfo.additionalURLs].map(async (url) => { - // for the top set of additional URLs, we render every new URL generated - const addlResult = await loadCollection(url); - builtURLs.add(url); - if (addlResult && addlResult.collectionInfo) { - // believe it or not, we may still have a few unbuilt pages left. this is our last crawl: - await Promise.all([...addlResult.collectionInfo.additionalURLs].map(async (url2) => loadCollection(url2))); - } - }) - ); - - if (result.collectionInfo.rss) { - if (!site) throw new Error(`[${rel}] createCollection() tried to generate RSS but "buildOptions.site" missing in astro.config.mjs`); - rss = generateRSS({ ...(result.collectionInfo.rss as any), site }, rel.replace(/\$([^.]+)\.astro$/, '$1')); - } - } - - return { - canonicalURLs: [...builtURLs].filter((url) => !url.endsWith('/1')), // note: canonical URLs are controlled by the collection, so these are canonical (but exclude "/1" pages as those are duplicates of the index) - statusCode: result.statusCode, - rss, - }; -} - -/** Build static page */ -async function buildStaticPage({ astroRoot, dist, filepath, runtime, sitemap, statics }: PageBuildOptions): Promise { - const rel = path.relative(fileURLToPath(astroRoot) + '/pages', fileURLToPath(filepath)); // pages/index.astro - const pagePath = `/${rel.replace(/\.(astro|md)$/, '')}`; - let canonicalURLs: string[] = []; - - let relPath = './' + rel.replace(/\.(astro|md)$/, '.html'); - if (!relPath.endsWith('index.html')) { - relPath = relPath.replace(/\.html$/, '/index.html'); - } - - const srcPath = fileURLToPath(new URL('pages/' + rel, astroRoot)); - const outPath = new URL(relPath, dist); - const result = await runtime.load(pagePath); - - await writeResult({ srcPath, result, outPath, encoding: 'utf8' }); - - if (result.statusCode === 200) { - mergeSet(statics, collectStatics(result.contents.toString('utf8'))); - - // get Canonical URL (if user has specified one manually, use that) - if (sitemap) { - const $ = cheerio.load(result.contents); - const canonicalTag = $('link[rel="canonical"]'); - canonicalURLs.push(canonicalTag.attr('href') || pagePath.replace(/index$/, '')); - } - } - - return { - canonicalURLs, - statusCode: result.statusCode, - }; +/** Is this URL remote? */ +function isRemote(url: string) { + if (url.startsWith('http://') || url.startsWith('https://') || url.startsWith('//')) return true; + return false; } /** The primary build action */ export async function build(astroConfig: AstroConfig): Promise<0 | 1> { const { projectRoot, astroRoot } = astroConfig; - const pageRoot = new URL('./pages/', astroRoot); - const componentRoot = new URL('./components/', astroRoot); const dist = new URL(astroConfig.dist + '/', projectRoot); + const pageRoot = new URL('./pages/', astroRoot); + const buildState: BuildOutput = {}; + const depTree: BundleMap = {}; + const timer: Record = {}; const runtimeLogging: LogOptions = { level: 'error', @@ -200,63 +59,34 @@ export async function build(astroConfig: AstroConfig): Promise<0 | 1> { const runtime = await createRuntime(astroConfig, { mode, logging: runtimeLogging }); const { runtimeConfig } = runtime; const { backendSnowpack: snowpack } = runtimeConfig; - const resolvePackageUrl = (pkgName: string) => snowpack.getUrlForPackage(pkgName); - - const imports = new Set(); - const statics = new Set(); - const collectImportsOptions = { astroConfig, logging, resolvePackageUrl, mode }; - - let builtURLs: string[] = []; - let urlStats = createURLStats(); - let importsToUrl = new Map>(); const pages = await allPages(pageRoot); + // 0. erase build directory + await del(fileURLToPath(dist)); + + /** + * 1. Build Pages + * Source files are built in parallel and stored in memory. Most assets are also gathered here, too. + */ + timer.build = performance.now(); try { info(logging, 'build', yellow('! building pages...')); - // Vue also console.warns, this silences it. - const release = trapWarn(); + const release = trapWarn(); // Vue also console.warns, this silences it. await Promise.all( pages.map(async (pathname) => { const filepath = new URL(`file://${pathname}`); - - const pageType = getPageType(filepath); - const pageOptions: PageBuildOptions = { astroRoot, dist, filepath, runtime, site: astroConfig.buildOptions.site, sitemap: astroConfig.buildOptions.sitemap, statics }; - let urls: string[]; - if (pageType === 'collection') { - const { canonicalURLs, rss } = await buildCollectionPage(pageOptions); - urls = canonicalURLs; - builtURLs.push(...canonicalURLs); - if (rss) { - const basename = path - .relative(fileURLToPath(astroRoot) + '/pages', pathname) - .replace(/^\$/, '') - .replace(/\.astro$/, ''); - await writeFilep(new URL(`file://${path.join(fileURLToPath(dist), 'feed', basename + '.xml')}`), rss, 'utf8'); - } - } else { - const { canonicalURLs } = await buildStaticPage(pageOptions); - urls = canonicalURLs; - builtURLs.push(...canonicalURLs); - } - - const dynamicImports = await collectDynamicImports(filepath, collectImportsOptions); - mergeSet(imports, dynamicImports); - - // Keep track of urls and dynamic imports for stats. - for(const url of urls) { - urlStats.set(url, { - dynamicImports, - stats: [] - }); - } - - for(let imp of dynamicImports) { - if(!importsToUrl.has(imp)) { - importsToUrl.set(imp, new Set()); - } - mergeSet(importsToUrl.get(imp)!, new Set(urls)); - } + const buildPage = getPageType(filepath) === 'collection' ? buildCollectionPage : buildStaticPage; + await buildPage({ + astroConfig, + buildState, + filepath, + logging, + mode, + resolvePackageUrl: (pkgName: string) => snowpack.getUrlForPackage(pkgName), + runtime, + site: astroConfig.buildOptions.site, + }); }) ); info(logging, 'build', green('✔'), 'pages built.'); @@ -266,19 +96,130 @@ export async function build(astroConfig: AstroConfig): Promise<0 | 1> { await runtime.shutdown(); return 1; } + debug(logging, 'build', `built pages [${stopTimer(timer.build)}]`); - info(logging, 'build', yellow('! scanning pages...')); - for (const pathname of await allPages(componentRoot)) { - mergeSet(imports, await collectDynamicImports(new URL(`file://${pathname}`), collectImportsOptions)); + // after pages are built, build depTree + timer.deps = performance.now(); + const scanPromises: Promise[] = []; + for (const id of Object.keys(buildState)) { + if (buildState[id].contentType !== 'text/html') continue; // only scan HTML files + const pageDeps = findDeps(buildState[id].contents as string, { + astroConfig, + srcPath: buildState[id].srcPath, + }); + depTree[id] = pageDeps; + + // while scanning we will find some unbuilt files; make sure those are all built while scanning + for (const url of [...pageDeps.js, ...pageDeps.css, ...pageDeps.images]) { + if (!buildState[url]) + scanPromises.push( + runtime.load(url).then((result) => { + if (result.statusCode !== 200) { + throw new Error((result as any).error); // there shouldn’t be a build error here + } + buildState[url] = { + srcPath: new URL(url, projectRoot), + contents: result.contents, + contentType: result.contentType || mime.getType(url) || '', + }; + }) + ); + } } - info(logging, 'build', green('✔'), 'pages scanned.'); + await Promise.all(scanPromises); + debug(logging, 'build', `scanned deps [${stopTimer(timer.deps)}]`); - if (imports.size > 0) { + /** + * 2. Bundling 1st Pass: In-memory + * Bundle CSS, and anything else that can happen in memory (for now, JS bundling happens after writing to disk) + */ + info(logging, 'build', yellow('! optimizing css...')); + timer.prebundle = performance.now(); + await Promise.all([ + bundleCSS({ buildState, astroConfig, logging, depTree }).then(() => { + debug(logging, 'build', `bundled CSS [${stopTimer(timer.prebundle)}]`); + }), + // TODO: optimize images? + ]); + // TODO: minify HTML? + info(logging, 'build', green('✔'), 'css optimized.'); + + /** + * 3. Write to disk + * Also clear in-memory bundle + */ + // collect stats output + const urlStats = await collectBundleStats(buildState, depTree); + + // collect JS imports for bundling + const jsImports = await collectJSImports(buildState); + + // write sitemap + if (astroConfig.buildOptions.sitemap && astroConfig.buildOptions.site) { + timer.sitemap = performance.now(); + info(logging, 'build', yellow('! creating sitemap...')); + const sitemap = generateSitemap(buildState, astroConfig.buildOptions.site); + const sitemapPath = new URL('sitemap.xml', dist); + await fs.promises.mkdir(path.dirname(fileURLToPath(sitemapPath)), { recursive: true }); + await fs.promises.writeFile(sitemapPath, sitemap, 'utf8'); + info(logging, 'build', green('✔'), 'sitemap built.'); + debug(logging, 'build', `built sitemap [${stopTimer(timer.sitemap)}]`); + } else if (astroConfig.buildOptions.sitemap) { + info(logging, 'tip', `Set "buildOptions.site" in astro.config.mjs to generate a sitemap.xml, or set "buildOptions.sitemap: false" to disable this message.`); + } + + timer.write = performance.now(); + + // write to disk and free up memory + await Promise.all( + Object.keys(buildState).map(async (id) => { + const outPath = new URL(`.${id}`, dist); + const parentDir = path.posix.dirname(fileURLToPath(outPath)); + await fs.promises.mkdir(parentDir, { recursive: true }); + await fs.promises.writeFile(outPath, buildState[id].contents, buildState[id].encoding); + delete buildState[id]; + delete depTree[id]; + }) + ); + debug(logging, 'build', `wrote files to disk [${stopTimer(timer.write)}]`); + + /** + * 4. Copy Public Assets + */ + if (fs.existsSync(astroConfig.public)) { + info(logging, 'build', yellow(`! copying public folder...`)); + timer.public = performance.now(); + const pub = astroConfig.public; + const publicFiles = (await new fdir().withFullPaths().crawl(fileURLToPath(pub)).withPromise()) as string[]; + await Promise.all( + publicFiles.map(async (filepath) => { + const fileUrl = new URL(`file://${filepath}`); + const rel = path.relative(fileURLToPath(pub), fileURLToPath(fileUrl)); + const outPath = new URL(path.join('.', rel), dist); + await fs.promises.mkdir(path.dirname(fileURLToPath(outPath)), { recursive: true }); + await fs.promises.copyFile(fileUrl, outPath); + }) + ); + debug(logging, 'build', `copied public folder [${stopTimer(timer.public)}]`); + info(logging, 'build', green('✔'), 'public folder copied.'); + } else { + if (path.basename(astroConfig.public.toString()) !== 'public') { + info(logging, 'tip', yellow(`! no public folder ${astroConfig.public} found...`)); + } + } + + /** + * 5. Bundling 2nd pass: On disk + * Bundle JS, which requires hard files to optimize + */ + info(logging, 'build', yellow(`! bundling...`)); + if (jsImports.size > 0) { try { - info(logging, 'build', yellow('! bundling client-side code.')); - const bundleStats = await bundle(imports, { dist, runtime, astroConfig }); - mapBundleStatsToURLStats(urlStats, importsToUrl, bundleStats); - info(logging, 'build', green('✔'), 'bundling complete.'); + timer.bundleJS = performance.now(); + const jsStats = await bundleJS(jsImports, { dist: new URL(dist + '/', projectRoot), runtime }); + mapBundleStatsToURLStats({ urlStats, depTree, bundleStats: jsStats }); + debug(logging, 'build', `bundled JS [${stopTimer(timer.bundleJS)}]`); + info(logging, 'build', green(`✔`), 'bundling complete.'); } catch (err) { error(logging, 'build', err); await runtime.shutdown(); @@ -286,45 +227,51 @@ export async function build(astroConfig: AstroConfig): Promise<0 | 1> { } } - for (let url of statics) { - const outPath = new URL('.' + url, dist); - const result = await runtime.load(url); - - await writeResult({ srcPath: url, result, outPath, encoding: null }); - } - - if (existsSync(astroConfig.public)) { - info(logging, 'build', yellow(`! copying public folder...`)); - const pub = astroConfig.public; - const publicFiles = (await new fdir().withFullPaths().crawl(fileURLToPath(pub)).withPromise()) as string[]; - for (const filepath of publicFiles) { - const fileUrl = new URL(`file://${filepath}`); - const rel = path.relative(pub.pathname, fileUrl.pathname); - const outUrl = new URL('./' + rel, dist); - - const bytes = await readFile(fileUrl); - await writeFilep(outUrl, bytes, null); - } - info(logging, 'build', green('✔'), 'public folder copied.'); - } else { - if (path.basename(astroConfig.public.toString()) !== 'public') { - info(logging, 'tip', yellow(`! no public folder ${astroConfig.public} found...`)); - } - } - // build sitemap - if (astroConfig.buildOptions.sitemap && astroConfig.buildOptions.site) { - info(logging, 'build', yellow('! creating a sitemap...')); - const sitemap = generateSitemap(builtURLs.map((url) => ({ canonicalURL: canonicalURL(url, astroConfig.buildOptions.site) }))); - await writeFile(new URL('./sitemap.xml', dist), sitemap, 'utf8'); - info(logging, 'build', green('✔'), 'sitemap built.'); - } else if (astroConfig.buildOptions.sitemap) { - info(logging, 'tip', `Set "buildOptions.site" in astro.config.mjs to generate a sitemap.xml`); - } - - // Log in a table-like view. - logURLStats(logging, urlStats, builtURLs); - + /** + * 6. Print stats + */ + logURLStats(logging, urlStats); await runtime.shutdown(); info(logging, 'build', bold(green('▶ Build Complete!'))); return 0; } + +/** Given an HTML string, collect and tags */ +export function findDeps(html: string, { astroConfig, srcPath }: { astroConfig: AstroConfig; srcPath: URL }): PageDependencies { + const pageDeps: PageDependencies = { + js: new Set(), + css: new Set(), + images: new Set(), + }; + + const $ = cheerio.load(html); + + $('script').each((i, el) => { + const src = $(el).attr('src'); + if (src && !isRemote(src)) { + pageDeps.js.add(getDistPath(src, { astroConfig, srcPath })); + } + }); + + $('link[href]').each((i, el) => { + const href = $(el).attr('href'); + if (href && !isRemote(href) && ($(el).attr('rel') === 'stylesheet' || $(el).attr('type') === 'text/css' || href.endsWith('.css'))) { + const dist = getDistPath(href, { astroConfig, srcPath }); + pageDeps.css.add(dist); + } + }); + + $('img[src]').each((i, el) => { + const src = $(el).attr('src'); + if (src && !isRemote(src)) { + pageDeps.images.add(getDistPath(src, { astroConfig, srcPath })); + } + }); + + // sort (makes things a bit more predictable) + pageDeps.js = sortSet(pageDeps.js); + pageDeps.css = sortSet(pageDeps.css); + pageDeps.images = sortSet(pageDeps.images); + + return pageDeps; +} diff --git a/packages/astro/src/build/bundle/css.ts b/packages/astro/src/build/bundle/css.ts new file mode 100644 index 000000000..11f978140 --- /dev/null +++ b/packages/astro/src/build/bundle/css.ts @@ -0,0 +1,139 @@ +import type { AstroConfig, BuildOutput, BundleMap } from '../../@types/astro'; +import type { LogOptions } from '../../logger.js'; + +import { performance } from 'perf_hooks'; +import shorthash from 'shorthash'; +import cheerio from 'cheerio'; +import esbuild from 'esbuild'; +import { getDistPath, getSrcPath, stopTimer } from '../util.js'; +import { debug } from '../../logger.js'; + +// config +const COMMON_URL = `/_astro/common-[HASH].css`; // [HASH] will be replaced + +/** + * Bundle CSS + * For files within dep tree, find ways to combine them. + * Current logic: + * - If CSS appears across multiple pages, combine into `/_astro/common.css` bundle + * - Otherwise, combine page CSS into one request as `/_astro/[page].css` bundle + * + * This operation _should_ be relatively-safe to do in parallel with other bundling, + * assuming other bundling steps don’t touch CSS. While this step does modify HTML, + * it doesn’t keep anything in local memory so other processes may modify HTML too. + * + * This operation mutates the original references of the buildOutput not only for + * safety (prevents possible conflicts), but for efficiency. + */ +export async function bundleCSS({ + astroConfig, + buildState, + logging, + depTree, +}: { + astroConfig: AstroConfig; + buildState: BuildOutput; + logging: LogOptions; + depTree: BundleMap; +}): Promise { + const timer: Record = {}; + const cssMap = new Map(); + + // 1. organize CSS into common or page-specific CSS + timer.bundle = performance.now(); + for (const [pageUrl, { css }] of Object.entries(depTree)) { + for (const cssUrl of css.keys()) { + if (cssMap.has(cssUrl)) { + // scenario 1: if multiple URLs require this CSS, upgrade to common chunk + cssMap.set(cssUrl, COMMON_URL); + } else { + // scenario 2: otherwise, assume this CSS is page-specific + cssMap.set(cssUrl, '/_astro' + pageUrl.replace(/.html$/, '').replace(/^\./, '') + '-[HASH].css'); + } + } + } + + // 2. bundle + timer.bundle = performance.now(); + await Promise.all( + Object.keys(buildState).map(async (id) => { + if (buildState[id].contentType !== 'text/css') return; + + const newUrl = cssMap.get(id); + if (!newUrl) return; + + // if new bundle, create + if (!buildState[newUrl]) { + buildState[newUrl] = { + srcPath: getSrcPath(id, { astroConfig }), // this isn’t accurate, but we can at least reference a file in the bundle + contents: '', + contentType: 'text/css', + encoding: 'utf8', + }; + } + + // append to bundle, delete old file + (buildState[newUrl] as any).contents += Buffer.isBuffer(buildState[id].contents) ? buildState[id].contents.toString('utf8') : buildState[id].contents; + delete buildState[id]; + }) + ); + debug(logging, 'css', `bundled [${stopTimer(timer.bundle)}]`); + + // 3. minify + timer.minify = performance.now(); + await Promise.all( + Object.keys(buildState).map(async (id) => { + if (buildState[id].contentType !== 'text/css') return; + const { code } = await esbuild.transform(buildState[id].contents as string, { + loader: 'css', + minify: true, + }); + buildState[id].contents = code; + }) + ); + debug(logging, 'css', `minified [${stopTimer(timer.minify)}]`); + + // 4. determine hashes based on CSS content (deterministic), and update HTML tags with final hashed URLs + timer.hashes = performance.now(); + const cssHashes = new Map(); + for (const id of Object.keys(buildState)) { + if (!id.includes('[HASH].css')) continue; // iterate through buildState, looking to replace [HASH] + + const hash = shorthash.unique(buildState[id].contents as string); + const newID = id.replace(/\[HASH\]/, hash); + cssHashes.set(id, newID); + buildState[newID] = buildState[id]; // copy ref without cloning to save memory + delete buildState[id]; // delete old ref + } + debug(logging, 'css', `built hashes [${stopTimer(timer.hashes)}]`); + + // 5. update HTML tags with final hashed URLs + timer.html = performance.now(); + await Promise.all( + Object.keys(buildState).map(async (id) => { + if (buildState[id].contentType !== 'text/html') return; + + const $ = cheerio.load(buildState[id].contents); + const pageCSS = new Set(); // keep track of page-specific CSS so we remove dupes + $('link[href]').each((i, el) => { + const srcPath = getSrcPath(id, { astroConfig }); + const oldHref = getDistPath($(el).attr('href') || '', { astroConfig, srcPath }); // note: this may be a relative URL; transform to absolute to find a buildOutput match + const newHref = cssMap.get(oldHref); + if (newHref) { + // note: link[href] will select too much, however, remote CSS and non-CSS link tags won’t be in cssMap + if (pageCSS.has(newHref)) { + $(el).remove(); // this is a dupe; remove + } else { + $(el).attr('href', cssHashes.get(newHref) || ''); // new CSS; update href (important! use cssHashes, not cssMap) + pageCSS.add(newHref); + } + // bonus: add [rel] and [type]. not necessary, but why not? + $(el).attr('rel', 'stylesheet'); + $(el).attr('type', 'text/css'); + } + }); + (buildState[id] as any).contents = $.html(); // save updated HTML in global buildState + }) + ); + debug(logging, 'css', `parsed html [${stopTimer(timer.html)}]`); +} diff --git a/packages/astro/src/build/bundle/js.ts b/packages/astro/src/build/bundle/js.ts new file mode 100644 index 000000000..0112f024a --- /dev/null +++ b/packages/astro/src/build/bundle/js.ts @@ -0,0 +1,95 @@ +import type { InputOptions, OutputOptions, OutputChunk } from 'rollup'; +import type { BuildOutput } from '../../@types/astro'; +import type { AstroRuntime } from '../../runtime'; + +import { fileURLToPath } from 'url'; +import { rollup } from 'rollup'; +import { terser } from 'rollup-plugin-terser'; +import { createBundleStats, addBundleStats, BundleStatsMap } from '../stats.js'; + +interface BundleOptions { + dist: URL; + runtime: AstroRuntime; +} + +/** Collect JS imports from build output */ +export function collectJSImports(buildState: BuildOutput): Set { + const imports = new Set(); + for (const id of Object.keys(buildState)) { + if (buildState[id].contentType === 'application/javascript') imports.add(id); + } + return imports; +} + +/** Bundle JS action */ +export async function bundleJS(imports: Set, { runtime, dist }: BundleOptions): Promise { + const ROOT = 'astro:root'; + const root = ` + ${[...imports].map((url) => `import '${url}';`).join('\n')} +`; + + const inputOptions: InputOptions = { + input: [...imports], + plugins: [ + { + name: 'astro:build', + resolveId(source: string, imported?: string) { + if (source === ROOT) { + return source; + } + if (source.startsWith('/')) { + return source; + } + + if (imported) { + const outUrl = new URL(source, 'http://example.com' + imported); + return outUrl.pathname; + } + + return null; + }, + async load(id: string) { + if (id === ROOT) { + return root; + } + + const result = await runtime.load(id); + + if (result.statusCode !== 200) { + return null; + } + + return result.contents.toString('utf-8'); + }, + }, + ], + }; + + const build = await rollup(inputOptions); + + const outputOptions: OutputOptions = { + dir: fileURLToPath(dist), + format: 'esm', + exports: 'named', + entryFileNames(chunk) { + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + return chunk.facadeModuleId!.substr(1); + }, + plugins: [ + // We are using terser for the demo, but might switch to something else long term + // Look into that rather than adding options here. + terser(), + ], + }; + + const stats = createBundleStats(); + const { output } = await build.write(outputOptions); + await Promise.all( + output.map(async (chunk) => { + const code = (chunk as OutputChunk).code || ''; + await addBundleStats(stats, code, chunk.fileName); + }) + ); + + return stats; +} diff --git a/packages/astro/src/build/bundle.ts b/packages/astro/src/build/page.ts similarity index 53% rename from packages/astro/src/build/bundle.ts rename to packages/astro/src/build/page.ts index c8bc37ece..89e2ac2a5 100644 --- a/packages/astro/src/build/bundle.ts +++ b/packages/astro/src/build/page.ts @@ -1,28 +1,139 @@ -import type { AstroConfig, RuntimeMode, ValidExtensionPlugins } from '../@types/astro'; import type { ImportDeclaration } from '@babel/types'; -import type { InputOptions, OutputOptions, OutputChunk } from 'rollup'; -import type { AstroRuntime } from '../runtime'; +import type { AstroConfig, BuildOutput, RuntimeMode, ValidExtensionPlugins } from '../@types/astro'; +import type { AstroRuntime, LoadResult } from '../runtime'; import type { LogOptions } from '../logger'; -import esbuild from 'esbuild'; -import { promises as fsPromises } from 'fs'; -import { fileURLToPath } from 'url'; -import { parse } from 'astro-parser'; -import { transform } from '../compiler/transform/index.js'; -import { convertMdToAstroSource } from '../compiler/index.js'; -import { getAttrValue } from '../ast.js'; -import { walk } from 'estree-walker'; -import babelParser from '@babel/parser'; +import fs from 'fs'; import path from 'path'; -import { rollup } from 'rollup'; -import { terser } from 'rollup-plugin-terser'; -import { createBundleStats, addBundleStats } from './stats.js'; - -const { transformSync } = esbuild; -const { readFile } = fsPromises; +import mime from 'mime'; +import { fileURLToPath } from 'url'; +import babelParser from '@babel/parser'; +import { parse } from 'astro-parser'; +import esbuild from 'esbuild'; +import { walk } from 'estree-walker'; +import { generateRSS } from './rss.js'; +import { getAttrValue } from '../ast.js'; +import { convertMdToAstroSource } from '../compiler/index.js'; +import { transform } from '../compiler/transform/index.js'; type DynamicImportMap = Map<'vue' | 'react' | 'react-dom' | 'preact' | 'svelte', string>; +interface PageBuildOptions { + astroConfig: AstroConfig; + buildState: BuildOutput; + logging: LogOptions; + filepath: URL; + mode: RuntimeMode; + resolvePackageUrl: (s: string) => Promise; + runtime: AstroRuntime; + site?: string; +} + +/** Collection utility */ +export function getPageType(filepath: URL): 'collection' | 'static' { + if (/\$[^.]+.astro$/.test(filepath.pathname)) return 'collection'; + return 'static'; +} + +/** Build collection */ +export async function buildCollectionPage({ astroConfig, filepath, logging, mode, runtime, site, resolvePackageUrl, buildState }: PageBuildOptions): Promise { + const rel = path.relative(fileURLToPath(astroConfig.astroRoot) + '/pages', fileURLToPath(filepath)); // pages/index.astro + const pagePath = `/${rel.replace(/\$([^.]+)\.astro$/, '$1')}`; + const srcPath = new URL('pages/' + rel, astroConfig.astroRoot); + const builtURLs = new Set(); // !important: internal cache that prevents building the same URLs + + /** Recursively build collection URLs */ + async function loadCollection(url: string): Promise { + if (builtURLs.has(url)) return; // this stops us from recursively building the same pages over and over + const result = await runtime.load(url); + builtURLs.add(url); + if (result.statusCode === 200) { + const outPath = path.posix.join('/', url, 'index.html'); + buildState[outPath] = { + srcPath, + contents: result.contents, + contentType: 'text/html', + encoding: 'utf8', + }; + } + return result; + } + + const [result] = await Promise.all([ + loadCollection(pagePath) as Promise, // first run will always return a result so assert type here + gatherRuntimes({ astroConfig, buildState, filepath, logging, resolvePackageUrl, mode, runtime }), + ]); + + if (result.statusCode >= 500) { + throw new Error((result as any).error); + } + if (result.statusCode === 200 && !result.collectionInfo) { + throw new Error(`[${rel}]: Collection page must export createCollection() function`); + } + + // note: for pages that require params (/tag/:tag), we will get a 404 but will still get back collectionInfo that tell us what the URLs should be + if (result.collectionInfo) { + // build subsequent pages + await Promise.all( + [...result.collectionInfo.additionalURLs].map(async (url) => { + // for the top set of additional URLs, we render every new URL generated + const addlResult = await loadCollection(url); + builtURLs.add(url); + if (addlResult && addlResult.collectionInfo) { + // believe it or not, we may still have a few unbuilt pages left. this is our last crawl: + await Promise.all([...addlResult.collectionInfo.additionalURLs].map(async (url2) => loadCollection(url2))); + } + }) + ); + + if (result.collectionInfo.rss) { + if (!site) throw new Error(`[${rel}] createCollection() tried to generate RSS but "buildOptions.site" missing in astro.config.mjs`); + const rss = generateRSS({ ...(result.collectionInfo.rss as any), site }, rel.replace(/\$([^.]+)\.astro$/, '$1')); + const feedURL = path.posix.join('/feed', `${pagePath}.xml`); + buildState[feedURL] = { + srcPath, + contents: rss, + contentType: 'application/rss+xml', + encoding: 'utf8', + }; + } + } +} + +/** Build static page */ +export async function buildStaticPage({ astroConfig, buildState, filepath, logging, mode, resolvePackageUrl, runtime }: PageBuildOptions): Promise { + const rel = path.relative(fileURLToPath(astroConfig.astroRoot) + '/pages', fileURLToPath(filepath)); // pages/index.astro + const pagePath = `/${rel.replace(/\.(astro|md)$/, '')}`; + + let relPath = path.posix.join('/', rel.replace(/\.(astro|md)$/, '.html')); + if (!relPath.endsWith('index.html')) { + relPath = relPath.replace(/\.html$/, '/index.html'); + } + + const srcPath = new URL('pages/' + rel, astroConfig.astroRoot); + + // build page in parallel with gathering runtimes + await Promise.all([ + runtime.load(pagePath).then((result) => { + if (result.statusCode === 200) { + buildState[relPath] = { srcPath, contents: result.contents, contentType: 'text/html', encoding: 'utf8' }; + } + }), + gatherRuntimes({ astroConfig, buildState, filepath, logging, resolvePackageUrl, mode, runtime }), + ]); +} + +/** Evaluate mustache expression (safely) */ +function compileExpressionSafe(raw: string): string { + let { code } = esbuild.transformSync(raw, { + loader: 'tsx', + jsxFactory: 'h', + jsxFragment: 'Fragment', + charset: 'utf8', + }); + return code; +} + /** Add framework runtimes when needed */ async function acquireDynamicComponentImports(plugins: Set, resolvePackageUrl: (s: string) => Promise): Promise { const importMap: DynamicImportMap = new Map(); @@ -50,17 +161,6 @@ async function acquireDynamicComponentImports(plugins: Set> = { '.jsx': 'react', '.tsx': 'react', @@ -68,39 +168,30 @@ const defaultExtensions: Readonly> = { '.vue': 'vue', }; -interface CollectDynamic { - astroConfig: AstroConfig; - resolvePackageUrl: (s: string) => Promise; - logging: LogOptions; - mode: RuntimeMode; -} - -/** Gather necessary framework runtimes for dynamic components */ -export async function collectDynamicImports(filename: URL, { astroConfig, logging, resolvePackageUrl, mode }: CollectDynamic) { +/** Gather necessary framework runtimes (React, Vue, Svelte, etc.) for dynamic components */ +async function gatherRuntimes({ astroConfig, buildState, filepath, logging, resolvePackageUrl, mode, runtime }: PageBuildOptions) { const imports = new Set(); // Only astro files - if (!filename.pathname.endsWith('.astro') && !filename.pathname.endsWith('.md')) { + if (!filepath.pathname.endsWith('.astro') && !filepath.pathname.endsWith('.md')) { return imports; } const extensions = astroConfig.extensions || defaultExtensions; - let source = await readFile(filename, 'utf-8'); - if (filename.pathname.endsWith('.md')) { + let source = await fs.promises.readFile(filepath, 'utf8'); + if (filepath.pathname.endsWith('.md')) { source = await convertMdToAstroSource(source); } - const ast = parse(source, { - filename, - }); + const ast = parse(source, { filepath }); if (!ast.module) { return imports; } await transform(ast, { - filename: fileURLToPath(filename), + filename: fileURLToPath(filepath), fileID: '', compileOptions: { astroConfig, @@ -224,13 +315,13 @@ export async function collectDynamicImports(filename: URL, { astroConfig, loggin } for (const foundImport of matches.reverse()) { const name = foundImport[1]; - appendImports(name, filename); + appendImports(name, filepath); } break; } case 'InlineComponent': { if (/^[A-Z]/.test(node.name)) { - appendImports(node.name, filename); + appendImports(node.name, filepath); return; } @@ -240,82 +331,21 @@ export async function collectDynamicImports(filename: URL, { astroConfig, loggin }, }); - return imports; -} - -interface BundleOptions { - runtime: AstroRuntime; - dist: URL; - astroConfig: AstroConfig; -} - -/** The primary bundling/optimization action */ -export async function bundle(imports: Set, { runtime, dist }: BundleOptions) { - const ROOT = 'astro:root'; - const root = ` - ${[...imports].map((url) => `import '${url}';`).join('\n')} - `; - - const inputOptions: InputOptions = { - input: [...imports], - plugins: [ - { - name: 'astro:build', - resolveId(source: string, imported?: string) { - if (source === ROOT) { - return source; - } - if (source.startsWith('/')) { - return source; - } - - if (imported) { - const outUrl = new URL(source, 'http://example.com' + imported); - return outUrl.pathname; - } - - return null; - }, - async load(id: string) { - if (id === ROOT) { - return root; - } - - const result = await runtime.load(id); - - if (result.statusCode !== 200) { - return null; - } - - return result.contents.toString('utf-8'); - }, - }, - ], - }; - - const build = await rollup(inputOptions); - - const outputOptions: OutputOptions = { - dir: fileURLToPath(dist), - format: 'esm', - exports: 'named', - entryFileNames(chunk) { - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - return chunk.facadeModuleId!.substr(1); - }, - plugins: [ - // We are using terser for the demo, but might switch to something else long term - // Look into that rather than adding options here. - terser(), - ], - }; - - const stats = createBundleStats(); - const {output} = await build.write(outputOptions); - await Promise.all(output.map(async chunk => { - const code = (chunk as OutputChunk).code || ''; - await addBundleStats(stats, code, chunk.fileName); - })); - - return stats; + // add all imports to build output + [...imports].map(async (url) => { + // don’t end up in an infinite loop building same URLs over and over + const alreadyBuilt = buildState[url]; + if (alreadyBuilt) return; + + // add new results to buildState + const result = await runtime.load(url); + if (result.statusCode === 200) { + buildState[url] = { + srcPath: filepath, + contents: result.contents, + contentType: result.contentType || mime.getType(url) || '', + encoding: 'utf8', + }; + } + }); } diff --git a/packages/astro/src/build/sitemap.ts b/packages/astro/src/build/sitemap.ts index 1cb3f3e40..5095019c7 100644 --- a/packages/astro/src/build/sitemap.ts +++ b/packages/astro/src/build/sitemap.ts @@ -1,14 +1,26 @@ -export interface PageMeta { - /** (required) The canonical URL of the page */ - canonicalURL: string; -} +import type { BuildOutput } from '../@types/astro'; + +import { canonicalURL } from './util'; /** Construct sitemap.xml given a set of URLs */ -export function generateSitemap(pages: PageMeta[]): string { +export function generateSitemap(buildState: BuildOutput, site: string): string { + const pages: string[] = []; + + // TODO: find way to respect URLs here + // TODO: find way to exclude pages from sitemap + + // look through built pages, only add HTML + for (const id of Object.keys(buildState)) { + if (buildState[id].contentType !== 'text/html' || id.endsWith('/1/index.html')) continue; // note: exclude auto-generated "page 1" pages (duplicates of index) + let url = canonicalURL(id.replace(/index\.html$/, ''), site); + pages.push(url); + } + + pages.sort((a, b) => a.localeCompare(b, 'en', { numeric: true })); // sort alphabetically so sitemap is same each time + let sitemap = ``; - pages.sort((a, b) => a.canonicalURL.localeCompare(b.canonicalURL, 'en', { numeric: true })); // sort alphabetically for (const page of pages) { - sitemap += `${page.canonicalURL}`; + sitemap += `${page}`; } sitemap += `\n`; return sitemap; diff --git a/packages/astro/src/build/static.ts b/packages/astro/src/build/static.ts deleted file mode 100644 index af99c33cb..000000000 --- a/packages/astro/src/build/static.ts +++ /dev/null @@ -1,28 +0,0 @@ -import type { Element } from 'domhandler'; -import cheerio from 'cheerio'; - -/** Given an HTML string, collect and tags */ -export function collectStatics(html: string) { - const statics = new Set(); - - const $ = cheerio.load(html); - - const append = (el: Element, attr: string) => { - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - const value: string = $(el).attr(attr)!; - if (value.startsWith('http') || $(el).attr('rel') === 'alternate') { - return; - } - statics.add(value); - }; - - $('link[href]').each((i, el) => { - append(el, 'href'); - }); - - $('img[src]').each((i, el) => { - append(el, 'src'); - }); - - return statics; -} diff --git a/packages/astro/src/build/stats.ts b/packages/astro/src/build/stats.ts index e29409994..909994a4a 100644 --- a/packages/astro/src/build/stats.ts +++ b/packages/astro/src/build/stats.ts @@ -1,3 +1,4 @@ +import type { BuildOutput, BundleMap } from '../@types/astro'; import type { LogOptions } from '../logger'; import { info, table } from '../logger.js'; @@ -30,19 +31,48 @@ export async function addBundleStats(bundleStatsMap: BundleStatsMap, code: strin bundleStatsMap.set(filename, { size: Buffer.byteLength(code), - gzipSize: gzsize + gzipSize: gzsize, }); } -export function mapBundleStatsToURLStats(urlStats: URLStatsMap, importsToUrl: Map>, bundleStats: BundleStatsMap) { - for(let [imp, stats] of bundleStats) { - for(let url of importsToUrl.get('/' + imp) || []) { - urlStats.get(url)?.stats.push(stats); +export function mapBundleStatsToURLStats({ urlStats, depTree, bundleStats }: { urlStats: URLStatsMap; depTree: BundleMap; bundleStats: BundleStatsMap }) { + for (let [srcPath, stats] of bundleStats) { + for (let url of urlStats.keys()) { + if (depTree[url] && depTree[url].js.has('/' + srcPath)) { + urlStats.get(url)?.stats.push(stats); + } } } } -export function logURLStats(logging: LogOptions, urlStats: URLStatsMap, builtURLs: string[]) { +export async function collectBundleStats(buildState: BuildOutput, depTree: BundleMap): Promise { + const urlStats = createURLStats(); + + await Promise.all( + Object.keys(buildState).map(async (id) => { + if (!depTree[id]) return; + const stats = await Promise.all( + [...depTree[id].js, ...depTree[id].css, ...depTree[id].images].map(async (url) => { + if (!buildState[url]) return undefined; + const stat = { + size: Buffer.byteLength(buildState[url].contents), + gzipSize: await gzipSize(buildState[url].contents), + }; + return stat; + }) + ); + urlStats.set(id, { + dynamicImports: new Set(), + stats: stats.filter((s) => !!s) as any, + }); + }) + ); + + return urlStats; +} + +export function logURLStats(logging: LogOptions, urlStats: URLStatsMap) { + const builtURLs = [...urlStats.keys()].map((url) => url.replace(/index\.html$/, '')); builtURLs.sort((a, b) => a.localeCompare(b, 'en', { numeric: true })); info(logging, null, ''); const log = table(logging, [60, 20]); @@ -51,11 +81,11 @@ export function logURLStats(logging: LogOptions, urlStats: URLStatsMap, builtURL const lastIndex = builtURLs.length - 1; builtURLs.forEach((url, index) => { const sep = index === 0 ? '┌' : index === lastIndex ? '└' : '├'; - const urlPart = (' ' + sep + ' ') + (url === '/' ? url : url + '/'); + const urlPart = ' ' + sep + ' ' + url; - const bytes = urlStats.get(url)?.stats.map(s => s.gzipSize).reduce((a, b) => a + b, 0) || 0; + const bytes = (urlStats.get(url) || urlStats.get(url + 'index.html'))?.stats.map((s) => s.gzipSize).reduce((a, b) => a + b, 0) || 0; const kb = (bytes * 0.001).toFixed(2); const sizePart = kb + ' kB'; log(info, urlPart, sizePart); }); -} \ No newline at end of file +} diff --git a/packages/astro/src/build/util.ts b/packages/astro/src/build/util.ts index 505e6f183..c22216388 100644 --- a/packages/astro/src/build/util.ts +++ b/packages/astro/src/build/util.ts @@ -1,4 +1,7 @@ +import type { AstroConfig } from '../@types/astro'; + import path from 'path'; +import { fileURLToPath, URL } from 'url'; /** Normalize URL to its canonical form */ export function canonicalURL(url: string, base?: string): string { @@ -7,3 +10,39 @@ export function canonicalURL(url: string, base?: string): string { base ).href; } + +/** Sort a Set */ +export function sortSet(set: Set): Set { + return new Set([...set].sort((a, b) => a.localeCompare(b, 'en', { numeric: true }))); +} + +/** Resolve final output URL */ +export function getDistPath(specifier: string, { astroConfig, srcPath }: { astroConfig: AstroConfig; srcPath: URL }): string { + if (specifier[0] === '/') return specifier; // assume absolute URLs are correct + + const fileLoc = path.posix.join(path.posix.dirname(fileURLToPath(srcPath)), specifier); + const projectLoc = path.posix.relative(fileURLToPath(astroConfig.astroRoot), fileLoc); + const pagesDir = fileURLToPath(new URL('/pages', astroConfig.astroRoot)); + // if this lives above src/pages, return that URL + if (fileLoc.includes(pagesDir)) { + const [, publicURL] = projectLoc.split(pagesDir); + return publicURL || '/index.html'; // if this is missing, this is the root + } + // otherwise, return /_astro/* url + return '/_astro/' + projectLoc; +} + +/** Given a final output URL, guess at src path (may be inaccurate) */ +export function getSrcPath(url: string, { astroConfig }: { astroConfig: AstroConfig }): URL { + if (url.startsWith('/_astro/')) { + return new URL(url.replace(/^\/_astro\//, ''), astroConfig.astroRoot); + } + let srcFile = url.replace(/^\//, '').replace(/\/index.html$/, '.astro'); + return new URL('./pages/' + srcFile, astroConfig.astroRoot); +} + +/** Stop timer & format time for profiling */ +export function stopTimer(start: number): string { + const diff = performance.now() - start; + return diff < 750 ? `${Math.round(diff)}ms` : `${(diff / 1000).toFixed(1)}s`; +} diff --git a/packages/astro/test/astro-css-bundling.test.js b/packages/astro/test/astro-css-bundling.test.js new file mode 100644 index 000000000..fe8d82c98 --- /dev/null +++ b/packages/astro/test/astro-css-bundling.test.js @@ -0,0 +1,50 @@ +import cheerio from 'cheerio'; +import { suite } from 'uvu'; +import * as assert from 'uvu/assert'; +import { setupBuild } from './helpers.js'; + +const CSSBundling = suite('CSS Bundling'); + +setupBuild(CSSBundling, './fixtures/astro-css-bundling'); + +// note: the hashes should be deterministic, but updating the file contents will change hashes +// be careful not to test that the HTML simply contains CSS, because it always will! filename and quanity matter here (bundling). +const EXPECTED_CSS = { + '/index.html': ['/_astro/common-ZVuUT3.css', '/_astro/index-Z2jH7pc.css'], + '/one/index.html': ['/_astro/common-ZVuUT3.css', '/_astro/one/index-2qFtfN.css'], + '/two/index.html': ['/_astro/common-ZVuUT3.css', '/_astro/two/index-2jKE68.css'], +}; +const UNEXPECTED_CSS = ['/_astro/components/nav.css', '../css/typography.css', '../css/colors.css', '../css/page-index.css', '../css/page-one.css', '../css/page-two.css']; + +CSSBundling('Bundles CSS', async (context) => { + await context.build(); + + const builtCSS = new Set(); + + // for all HTML files… + for (const [filepath, css] of Object.entries(EXPECTED_CSS)) { + const html = await context.readFile(filepath); + const $ = cheerio.load(html); + + // test 1: assert new bundled CSS is present + for (const href of css) { + builtCSS.add(href); + const link = $(`link[href="${href}"]`); + assert.equal(link.length, 1); + } + + // test 2: assert old CSS was removed + for (const href of UNEXPECTED_CSS) { + const link = $(`link[href="${href}"]`); + assert.equal(link.length, 0); + } + } + + // test 3: assert all bundled CSS was built and contains CSS + for (const url of builtCSS.keys()) { + const css = await context.readFile(url); + assert.ok(css, true); + } +}); + +CSSBundling.run(); diff --git a/packages/astro/test/astro-rss.test.js b/packages/astro/test/astro-rss.test.js index 1fc70a9a7..055150362 100644 --- a/packages/astro/test/astro-rss.test.js +++ b/packages/astro/test/astro-rss.test.js @@ -11,14 +11,8 @@ const snapshot = ']]>Tue, 19 Oct 1999 00:00:00 GMTmusic259true'; RSS('Generates RSS correctly', async (context) => { - let rss; - try { - await context.build(); - rss = await context.readFile('/feed/episodes.xml'); - assert.ok(true, 'Build successful'); - } catch (err) { - assert.ok(false, 'Build threw'); - } + await context.build(); + let rss = await context.readFile('/feed/episodes.xml'); assert.match(rss, snapshot); }); diff --git a/packages/astro/test/astro-sitemap.test.js b/packages/astro/test/astro-sitemap.test.js index 5e47c5d81..778816929 100644 --- a/packages/astro/test/astro-sitemap.test.js +++ b/packages/astro/test/astro-sitemap.test.js @@ -6,18 +6,12 @@ const Sitemap = suite('Sitemap Generation'); setupBuild(Sitemap, './fixtures/astro-rss'); -const snapshot = `https://mysite.dev/episode/fazers/https://mysite.dev/episode/rap-snitch-knishes/https://mysite.dev/episode/rhymes-like-dimes/https://mysite.dev/episodes/`; +const snapshot = `https://mysite.dev/episode/fazers/https://mysite.dev/episode/rap-snitch-knishes/https://mysite.dev/episode/rhymes-like-dimes/https://mysite.dev/episodes/\n`; Sitemap('Generates Sitemap correctly', async (context) => { - let rss; - try { - await context.build(); - rss = await context.readFile('/sitemap.xml'); - assert.ok(true, 'Build successful'); - } catch (err) { - assert.ok(false, 'Build threw'); - } - assert.match(rss, snapshot); + await context.build(); + let sitemap = await context.readFile('/sitemap.xml'); + assert.match(sitemap, snapshot); }); Sitemap.run(); diff --git a/packages/astro/test/fixtures/astro-css-bundling/src/components/Nav.astro b/packages/astro/test/fixtures/astro-css-bundling/src/components/Nav.astro new file mode 100644 index 000000000..deafe668c --- /dev/null +++ b/packages/astro/test/fixtures/astro-css-bundling/src/components/Nav.astro @@ -0,0 +1,9 @@ + + + \ No newline at end of file diff --git a/packages/astro/test/fixtures/astro-css-bundling/src/css/colors.css b/packages/astro/test/fixtures/astro-css-bundling/src/css/colors.css new file mode 100644 index 000000000..27bafe566 --- /dev/null +++ b/packages/astro/test/fixtures/astro-css-bundling/src/css/colors.css @@ -0,0 +1,4 @@ +:root { + --brown: burlywood; + --red: crimson; +} diff --git a/packages/astro/test/fixtures/astro-css-bundling/src/css/page-index.css b/packages/astro/test/fixtures/astro-css-bundling/src/css/page-index.css new file mode 100644 index 000000000..b553e8e49 --- /dev/null +++ b/packages/astro/test/fixtures/astro-css-bundling/src/css/page-index.css @@ -0,0 +1,3 @@ +.page__index { + background: var(--red); +} diff --git a/packages/astro/test/fixtures/astro-css-bundling/src/css/page-one.css b/packages/astro/test/fixtures/astro-css-bundling/src/css/page-one.css new file mode 100644 index 000000000..3e09a847f --- /dev/null +++ b/packages/astro/test/fixtures/astro-css-bundling/src/css/page-one.css @@ -0,0 +1,3 @@ +.page__one { + background: paleturquoise; +} diff --git a/packages/astro/test/fixtures/astro-css-bundling/src/css/page-two.css b/packages/astro/test/fixtures/astro-css-bundling/src/css/page-two.css new file mode 100644 index 000000000..dbef8bf34 --- /dev/null +++ b/packages/astro/test/fixtures/astro-css-bundling/src/css/page-two.css @@ -0,0 +1,3 @@ +.page__two { + background: var(--brown); +} diff --git a/packages/astro/test/fixtures/astro-css-bundling/src/css/typography.css b/packages/astro/test/fixtures/astro-css-bundling/src/css/typography.css new file mode 100644 index 000000000..810acd74a --- /dev/null +++ b/packages/astro/test/fixtures/astro-css-bundling/src/css/typography.css @@ -0,0 +1,6 @@ +/* Typography.css */ + +body { + font-size: 16px; + font-family: sans-serif; +} diff --git a/packages/astro/test/fixtures/astro-css-bundling/src/pages/index.astro b/packages/astro/test/fixtures/astro-css-bundling/src/pages/index.astro new file mode 100644 index 000000000..9479981c8 --- /dev/null +++ b/packages/astro/test/fixtures/astro-css-bundling/src/pages/index.astro @@ -0,0 +1,15 @@ +--- +import Nav from '../components/Nav.astro'; +--- + + + + + + + + +