Adopt turborepo (#2167)

* chore: use turbo

* chore(lint): Prettier fix

* chore(actions): fix ci

* chore(actions): add turbo caching

* chore(actions): test turbo caching

* chore: remove test dependency on build

* chore: fix build outputs

* chore(actions): restore tests from cache

* chore(actions): fix turbo cache

* chore(actions): fix turbo cache

* chore(actions): restore turbo after artifacts

* chore(actions): caching is hard

* chore: force ci

* chore(actions): prefer-offline for yarn install

* chore(actions): attempt to cache turbo based on sha

* chore: remove lerna.json

* chore(action): update restore keys

* chore(action): sidestep turbo for windows tests

* chore(action): do not restore unhashed files

* chore(action): fix typo

* chore(action): fix typo

* chore(action): fix typo

* chore(action): fix artifacts paths

* chore(action): sync turbo cache

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

.github/workflows/ci.yml

@@ -37,7 +37,7 @@ jobs:
           cache: 'yarn'
 
       - name: Install NPM Dependencies
-        run: yarn install --prefer-offline --frozen-lockfile --ignore-engines
+        run: yarn install --prefer-offline --frozen-lockfile --ignore-engines --registry https://registry.npmjs.org --network-timeout 300000
         env:
           CI: true
       
@@ -83,19 +83,29 @@ jobs:
           node-version: 16
           cache: 'yarn'
 
+      - name: Cache Turbo
+        uses: actions/cache@v2
+        with:
+          path: "**/.turbo"
+          key: "turbo-${{ github.ref }}-${{ hashFiles('packages/**/src/**') }}-${{ github.sha }}"
+          restore-keys: |
+            turbo-${{ github.ref }}-${{ hashFiles('packages/**/src/**') }}-${{ github.sha }}
+            turbo-${{ github.ref }}-${{ hashFiles('packages/**/src/**') }}-
+            turbo-${{ github.ref }}-
+
       - name: Install NPM Dependencies
-        run: yarn install --frozen-lockfile --ignore-engines --registry https://registry.npmjs.org --network-timeout 300000
+        run: yarn install --prefer-offline --frozen-lockfile --ignore-engines --registry https://registry.npmjs.org --network-timeout 300000
         env:
           CI: true
 
       - name: Build Packages
-        run: yarn build:all
+        run: yarn build
 
       - name: Upload Package Artifacts
         uses: actions/upload-artifact@v2
         with:
           name: artifacts
-          path: packages/**/dist/*
+          path: packages/**/dist/**
           if-no-files-found: error
   
   # Test depends on Build's output, which allows us to skip any build process!
@@ -131,14 +141,30 @@ jobs:
       - name: Extract Artifacts
         run: ./.github/extract-artifacts.sh
 
+      - name: Cache Turbo
+        uses: actions/cache@v2
+        with:
+          path: "**/.turbo"
+          key: "turbo-${{ github.ref }}-${{ hashFiles('packages/**/src/**') }}-${{ github.sha }}"
+          restore-keys: |
+            turbo-${{ github.ref }}-${{ hashFiles('packages/**/src/**') }}-${{ github.sha }}
+            turbo-${{ github.ref }}-${{ hashFiles('packages/**/src/**') }}-
+            turbo-${{ github.ref }}-
+
       - name: Install NPM Dependencies
-        run: yarn install --prefer-offline --frozen-lockfile --ignore-engines
+        run: yarn install --prefer-offline --frozen-lockfile --ignore-engines --registry https://registry.npmjs.org --network-timeout 300000
         env:
           CI: true
 
       - name: Test
+        if: ${{ matrix.os != 'windows-latest' }}
         run: yarn test
 
+      # Turbo seems to make the tests hang on Windows, just run them directly
+      - name: Test (Windows)
+        if: ${{ matrix.os == 'windows-latest' }}
+        run: yarn workspace astro run test
+
   smoke:
     name: 'Smoke Test: ${{ matrix.target }}'
     runs-on: ubuntu-latest
@@ -163,8 +189,18 @@ jobs:
       - name: Extract Artifacts
         run: ./.github/extract-artifacts.sh
 
+      - name: Cache Turbo
+        uses: actions/cache@v2
+        with:
+          path: "**/.turbo"
+          key: "turbo-${{ github.ref }}-${{ hashFiles('packages/**/src/**') }}-${{ github.sha }}"
+          restore-keys: |
+            turbo-${{ github.ref }}-${{ hashFiles('packages/**/src/**') }}-${{ github.sha }}
+            turbo-${{ github.ref }}-${{ hashFiles('packages/**/src/**') }}-
+            turbo-${{ github.ref }}-
+
       - name: Install NPM Dependencies
-        run: yarn install --frozen-lockfile --ignore-engines --registry https://registry.npmjs.org --network-timeout 300000
+        run: yarn install --prefer-offline --frozen-lockfile --ignore-engines --registry https://registry.npmjs.org --network-timeout 300000
         env:
           CI: true
 
@@ -176,7 +212,7 @@ jobs:
   # We download all `dist/` artifacts from GitHub to skip the build process.
   changelog:
     name: Changelog PR or Release
-    if: ${{ github.ref_name == 'main' && github.repository_owner == 'withastro' }}
+    if: ${{ github.ref == 'main' && github.repository_owner == 'withastro' }}
     needs: [lint, test, smoke]
     runs-on: ubuntu-latest
     steps:
@@ -196,8 +232,18 @@ jobs:
       - name: Extract Artifacts
         run: ./.github/extract-artifacts.sh
 
+      - name: Cache Turbo
+        uses: actions/cache@v2
+        with:
+          path: "**/.turbo"
+          key: "turbo-${{ github.ref }}-${{ hashFiles('packages/**/src/**') }}-${{ github.sha }}"
+          restore-keys: |
+            turbo-${{ github.ref }}-${{ hashFiles('packages/**/src/**') }}-${{ github.sha }}
+            turbo-${{ github.ref }}-${{ hashFiles('packages/**/src/**') }}-
+            turbo-${{ github.ref }}-
+
       - name: Install NPM Dependencies
-        run: yarn install --prefer-offline --frozen-lockfile --ignore-engines
+        run: yarn install --prefer-offline --frozen-lockfile --ignore-engines --registry https://registry.npmjs.org --network-timeout 300000
         env:
           CI: true
       

.github/workflows/main.yml

@@ -46,7 +46,8 @@ jobs:
           cache: 'yarn'
 
       - name: Install NPM Dependencies
-        run: yarn install --prefer-offline --frozen-lockfile --ignore-engines
+        if: steps.cache-npm.outputs.cache-hit != 'true'
+        run: yarn install --prefer-offline --frozen-lockfile --ignore-engines --registry https://registry.npmjs.org --network-timeout 300000
         env:
           CI: true
       

.gitignore

@@ -6,6 +6,7 @@ dist/
 _site/
 *.log
 package-lock.json
+.turbo/
 
 # .vscode files other than at top-level
 **/.vscode
@@ -13,4 +14,4 @@ package-lock.json
 # do not commit .env files or any files that end with `.env`
 *.env
 
-!packages/astro/vendor/vite/dist
+!packages/astro/vendor/vite/dist

CONTRIBUTING.md

@@ -22,7 +22,7 @@ Astro uses yarn workspaces, so you should **always run `yarn install` from the t
 ```shell
 git clone && cd ...
 yarn install
-yarn build:all
+yarn build
 ```
 
 ### Development

lerna.json

@@ -1,10 +0,0 @@
-{
-  "ignoreChanges": ["**/test/**", "**/*.md"],
-  "useWorkspaces": true,
-  "version": "4.0.0",
-  "command": {
-    "run": {
-      "npmClient": "yarn"
-    }
-  }
-}

package.json

@@ -7,20 +7,46 @@
     "url": "https://github.com/withastro/astro.git"
   },
   "scripts": {
-    "release": "yarn build:all && changeset publish",
+    "release": "yarn build && changeset publish",
-    "benchmark": "yarn workspace astro run benchmark",
+    "build": "turbo run build --no-deps --scope=astro --scope=create-astro --scope=\"@astrojs/*\"",
-    "build": "yarn build:core",
+    "dev": "turbo run dev --no-deps --no-cache --parallel --scope=astro --scope=create-astro --scope=\"@astrojs/*\"",
-    "build:one": "lerna run build --scope",
+    "test": "turbo run test --scope=astro",
-    "build:all": "lerna run build --scope \"{astro,create-astro,@astrojs/*}\"",
+    "test:templates": "turbo run test --scope=create-astro",
-    "build:core": "lerna run build --scope \"{astro,@astrojs/parser,@astrojs/markdown-remark}\"",
+    "benchmark": "turbo run benchmark --scope=astro",
-    "dev": "yarn dev:core --parallel --stream",
-    "dev:one": "lerna run dev --scope --parallel --stream",
-    "dev:all": "lerna run dev --scope \"{astro,create-astro,@astrojs/*}\" --parallel --stream",
-    "dev:core": "lerna run dev --scope \"{astro,@astrojs/parser,@astrojs/markdown-remark}\" --parallel --stream",
-    "format": "prettier -w .",
     "lint": "eslint \"packages/**/*.ts\"",
-    "test": "yarn workspace astro run test",
+    "format": "prettier -w ."
-    "test:templates": "lerna run test --scope create-astro --stream"
+  },
+  "turbo": {
+    "baseBranch": "origin/main",
+    "pipeline": {
+      "build": {
+        "dependsOn": [
+          "^build"
+        ],
+        "outputs": [
+          "**/dist/**",
+          "!**/vendor/**"
+        ]
+      },
+      "dev": {
+        "cache": false
+      },
+      "test": {
+        "outputs": []
+      },
+      "test:templates": {
+        "outputs": []
+      },
+      "benchmark": {
+        "dependsOn": [
+          "^build"
+        ],
+        "outputs": []
+      },
+      "lint": {
+        "outputs": []
+      }
+    }
   },
   "workspaces": [
     "compiled/*",
@@ -53,9 +79,9 @@
     "eslint-config-prettier": "^8.3.0",
     "eslint-plugin-prettier": "^4.0.0",
     "execa": "^5.0.0",
-    "lerna": "^4.0.0",
     "prettier": "^2.4.1",
     "tiny-glob": "^0.2.8",
+    "turbo": "^1.0.0",
     "typescript": "4.4.4"
   },
   "engines": {