Fastdoc

.cargo/config.toml

@@ -0,0 +1,2 @@
+[registries.crates-io]
+protocol = "sparse"

.gitignore

@@ -0,0 +1 @@
+/target

Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+name = "fastdoc"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+anyhow = "1.0.71"
+async-walkdir = "0.2.0"
+clap = { version = "4.2.7", features = ["derive"] }
+console = "0.15.6"
+futures = "0.3.28"
+indicatif = { version = "0.17.3", features = ["tokio", "improved_unicode", "vt100"] }
+swc_common = { version = "0.31.10", features = ["tty-emitter"] }
+swc_ecma_parser = "0.134.6"
+tokio = { version = "1.28.1", features = ["full"] }

rustfmt.toml

@@ -0,0 +1,3 @@
+max_width = 80
+tab_spaces = 2
+wrap_comments = true

src/main.rs

@@ -0,0 +1,196 @@
+use std::path::{Path, PathBuf};
+use std::sync::mpsc;
+use std::thread;
+
+use anyhow::{Error, Result};
+use async_walkdir::{Filtering, WalkDir};
+use clap::Parser;
+use console::Term;
+use futures::{stream, StreamExt, TryStreamExt};
+use indicatif::{ProgressBar, ProgressStyle};
+use swc_common::errors::ColorConfig;
+use swc_common::{errors::Handler, sync::Lrc};
+use swc_common::{FileName, SourceMap};
+use swc_ecma_parser::EsConfig;
+use swc_ecma_parser::{
+  lexer::Lexer as EcmaLexer, Parser as EcmaParser, StringInput, Syntax,
+};
+use tokio::fs;
+use tokio::sync::Semaphore;
+use tokio::{fs::File, io::AsyncReadExt};
+
+#[derive(Parser, Debug)]
+struct Opt {
+  /// Output directory
+  #[clap(default_value = "generated/doc")]
+  out_dir: PathBuf,
+
+  /// Paths to process
+  paths: Vec<PathBuf>,
+}
+
+#[tokio::main]
+async fn main() -> Result<()> {
+  let opt = Opt::parse();
+
+  fs::create_dir_all(&opt.out_dir).await?;
+
+  let paths = {
+    let mut paths = opt.paths;
+    if paths.is_empty() {
+      paths.push(".".into())
+    }
+    paths
+  };
+
+  let bar = ProgressBar::new(0);
+  bar.set_style(
+    ProgressStyle::with_template(
+      // note that bar size is fixed unlike cargo which is dynamic
+      // and also the truncation in cargo uses trailers (`...`)
+      if Term::stdout().size().1 > 80 {
+        "{prefix:>3.cyan.bold} [{bar:57}] {pos}/{len} {wide_msg}"
+      } else {
+        "{prefix:>3.cyan.bold} [{bar:57}] {pos}/{len}"
+      },
+    )
+    .unwrap()
+    .progress_chars("=> "),
+  );
+  bar.set_prefix("Documenting");
+
+  enum BarAction {
+    Inc,
+    Finish,
+    SetLen(u64),
+  }
+  let (tx, rx) = mpsc::channel::<BarAction>();
+
+  thread::spawn(move || {
+    for msg in rx.into_iter() {
+      match msg {
+        BarAction::Inc => bar.inc(1),
+        BarAction::Finish => bar.finish(),
+        BarAction::SetLen(n) => bar.set_length(n),
+      }
+    }
+  });
+
+  for base_path in paths.into_iter() {
+    let num_files = create_walkdir(&base_path).count().await;
+    tx.send(BarAction::SetLen(num_files as u64));
+    println!("Counted {num_files} files.");
+
+    let entries_stream = create_walkdir(&base_path);
+
+    let semaphore = Semaphore::new(10);
+    let base_path2 = base_path.clone();
+    let processed_stream = entries_stream
+      .map_err(|e| Error::from(e))
+      .and_then(|dir_ent| {
+        let tx = tx.clone();
+        let base_path3 = base_path2.clone();
+        let permit_fut = semaphore.acquire();
+
+        async move {
+          let _permit = permit_fut.await?;
+
+          let path = dir_ent.path();
+          let relative_path = path.strip_prefix(base_path3)?;
+
+          // println!("Path: {relative_path:?}");
+
+          let data = {
+            let mut file = File::open(&path).await?;
+            let mut contents = String::new();
+            file.read_to_string(&mut contents).await?;
+            contents
+          };
+
+          handle_data(relative_path, data).await?;
+
+          tx.send(BarAction::Inc);
+
+          Ok::<_, Error>(false)
+        }
+      })
+      .try_collect::<Vec<_>>()
+      .await;
+  }
+
+  tx.send(BarAction::Finish);
+
+  Ok(())
+}
+
+async fn handle_data(path: impl AsRef<Path>, data: String) -> Result<()> {
+  let cm: Lrc<SourceMap> = Default::default();
+
+  let handler =
+    Handler::with_tty_emitter(ColorConfig::Auto, true, false, Some(cm.clone()));
+
+  let fm = cm.new_source_file(
+    FileName::Custom(path.as_ref().display().to_string()),
+    data,
+  );
+
+  let syntax = Syntax::Es(EsConfig {
+    jsx: true,
+    ..Default::default()
+  });
+
+  let lexer = EcmaLexer::new(
+    // We want to parse ecmascript
+    syntax,
+    // EsVersion defaults to es5
+    Default::default(),
+    StringInput::from(&*fm),
+    None,
+  );
+
+  let mut parser = EcmaParser::new_from(lexer);
+
+  for e in parser.take_errors() {
+    e.into_diagnostic(&handler).emit();
+  }
+
+  let module = parser
+    .parse_module()
+    .map_err(|e| {
+      // Unrecoverable fatal error occurred
+      e.into_diagnostic(&handler).emit()
+    })
+    .expect("failed to parser module");
+
+  // println!("Module: {module:?}");
+
+  Ok(())
+}
+
+fn create_walkdir(path: impl AsRef<Path>) -> WalkDir {
+  WalkDir::new(path.as_ref()).filter(|entry| async move {
+    let path = entry.path();
+
+    if let Some(true) = path
+      .file_name()
+      .map(|f| f == "node_modules" || f.to_string_lossy().starts_with('.'))
+    {
+      return Filtering::IgnoreDir;
+    }
+
+    if path.is_file()
+      && !path
+        .file_name()
+        .and_then(|f| f.to_str())
+        .map_or_else(|| false, |f| f.ends_with(".js"))
+    {
+      return Filtering::Ignore;
+    }
+
+    if path.is_dir() {
+      return Filtering::Ignore;
+    }
+
+    Filtering::Continue
+  })
+}