Allow createCollection() to fetch remote data (#400)

* Allow createCollection() to fetch remote data

Fixes #378

* Update docs

* revert isomorphic-fetch, see if ci passes

Co-authored-by: Fred K. Schott <fkschott@gmail.com>

.changeset/five-ducks-knock.md

@@ -0,0 +1,5 @@
+---
+'astro': patch
+---
+
+Bugfix: createCollection() API can be used without fetchContent()

docs/api.md

@@ -91,7 +91,7 @@ When using the [Collections API][docs-collections], `createCollection()` is an a
 
 | Name        |             Type              | Description                                                                                                |
 | :---------- | :---------------------------: | :--------------------------------------------------------------------------------------------------------- |
-| `data`      | `async ({ params }) => any[]` | **Required.** Load data with this function to be returned.                                                 |
+| `data`      | `async ({ params }) => any[]` | **Required.** Load an array of data with this function to be returned.                                     |
 | `pageSize`  |           `number`            | Specify number of items per page (default: `25`).                                                          |
 | `routes`    |          `params[]`           | **Required for URL Params.** Return an array of all possible URL `param` values in `{ name: value }` form. |
 | `permalink` |   `({ params }) => string`    | **Required for URL Params.** Given a `param` object of `{ name: value }`, generate the final URL.\*        |

packages/astro/src/compiler/codegen/index.ts

@@ -249,7 +249,7 @@ function compileModule(module: Script, state: CodegenState, compileOptions: Comp
           } else if (node.declaration.type === 'FunctionDeclaration') {
             // case 2: createCollection (export async function)
             if (!node.declaration.id || node.declaration.id.name !== 'createCollection') break;
-            createCollection = module.content.substring(node.declaration.start || 0, node.declaration.end || 0);
+            createCollection = module.content.substring(node.start || 0, node.end || 0);
 
             // remove node
             body.splice(i, 1);
@@ -365,8 +365,7 @@ function compileModule(module: Script, state: CodegenState, compileOptions: Comp
                   imports += importStatement + '\n';
                 }
 
-                createCollection =
-                  imports + '\nexport ' + createCollection.substring(0, declaration.start || 0) + globResult.code + createCollection.substring(declaration.end || 0);
+                createCollection = imports + createCollection.substring(0, declaration.start || 0) + globResult.code + createCollection.substring(declaration.end || 0);
               }
               break;
             }

packages/astro/src/runtime.ts

@@ -7,9 +7,16 @@ import { existsSync, promises as fs } from 'fs';
 import { fileURLToPath, pathToFileURL } from 'url';
 import { posix as path } from 'path';
 import { performance } from 'perf_hooks';
-import { SnowpackDevServer, ServerRuntime as SnowpackServerRuntime, SnowpackConfig, NotFoundError } from 'snowpack';
+import {
+  loadConfiguration,
+  logger as snowpackLogger,
+  NotFoundError,
+  SnowpackDevServer,
+  ServerRuntime as SnowpackServerRuntime,
+  SnowpackConfig,
+  startServer as startSnowpackServer,
+} from 'snowpack';
 import { CompileError } from '@astrojs/parser';
-import { loadConfiguration, logger as snowpackLogger, startServer as startSnowpackServer } from 'snowpack';
 import { canonicalURL, getSrcPath, stopTimer } from './build/util.js';
 import { debug, info } from './logger.js';
 import { configureSnowpackLogger } from './snowpack-logger.js';
@@ -94,20 +101,22 @@ async function load(config: RuntimeConfig, rawPathname: string | undefined): Pro
 
     if (mod.exports.createCollection) {
       const createCollection: CreateCollection = await mod.exports.createCollection();
+      const VALID_KEYS = new Set(['data', 'routes', 'permalink', 'pageSize', 'rss']);
       for (const key of Object.keys(createCollection)) {
-        if (key !== 'data' && key !== 'routes' && key !== 'permalink' && key !== 'pageSize' && key !== 'rss') {
-          throw new Error(`[createCollection] unknown option: "${key}"`);
+        if (!VALID_KEYS.has(key)) {
+          throw new Error(`[createCollection] unknown option: "${key}". Expected one of ${[...VALID_KEYS].join(', ')}.`);
         }
       }
       let { data: loadData, routes, permalink, pageSize, rss: createRSS } = createCollection;
+      if (!loadData) throw new Error(`[createCollection] must return \`data()\` function to create a collection.`);
       if (!pageSize) pageSize = 25; // can’t be 0
       let currentParams: Params = {};
 
       // params
       if (routes || permalink) {
-        if (!routes || !permalink) {
-          throw new Error('createCollection() must have both routes and permalink options. Include both together, or omit both.');
-        }
+        if (!routes) throw new Error('[createCollection] `permalink` requires `routes` as well.');
+        if (!permalink) throw new Error('[createCollection] `routes` requires `permalink` as well.');
+
         let requestedParams = routes.find((p) => {
           const baseURL = (permalink as any)({ params: p });
           additionalURLs.add(baseURL);
@@ -120,6 +129,8 @@ async function load(config: RuntimeConfig, rawPathname: string | undefined): Pro
       }
 
       let data: any[] = await loadData({ params: currentParams });
+      if (!data) throw new Error(`[createCollection] \`data()\` returned nothing (empty data)"`);
+      if (!Array.isArray(data)) data = [data]; // note: this is supposed to be a little friendlier to the user, but should we error out instead?
 
       // handle RSS
       if (createRSS) {

packages/astro/test/astro-collection.test.js

@@ -42,4 +42,16 @@ Collections('generates pagination successfully', async ({ runtime }) => {
   assert.equal(next.length, 1); // this should be on-page
 });
 
+Collections('can load remote data', async ({ runtime }) => {
+  const result = await runtime.load('/remote');
+  if (result.error) throw new Error(result.error);
+  const $ = doc(result.contents);
+
+  const PACKAGES_TO_TEST = ['canvas-confetti', 'preact', 'svelte'];
+
+  for (const pkg of PACKAGES_TO_TEST) {
+    assert.ok($(`#pkg-${pkg}`).length);
+  }
+});
+
 Collections.run();

packages/astro/test/fixtures/astro-collection/src/pages/$remote.astro

@@ -0,0 +1,24 @@
+---
+export let collection: any;
+
+export async function createCollection() {
+  const data = await Promise.all([
+    fetch('https://api.skypack.dev/v1/package/canvas-confetti').then((res) => res.json()),
+    fetch('https://api.skypack.dev/v1/package/preact').then((res) => res.json()),
+    fetch('https://api.skypack.dev/v1/package/svelte').then((res) => res.json()),
+  ]);
+
+  return {
+    async data() {
+      return data;
+    }
+  }
+}
+
+---
+
+<div>
+  {collection.data.map((pkg) => (
+    <div id={`pkg-${pkg.name}`}>{pkg.name}</div>
+   ))}
+</div>