initial
This commit is contained in:
commit
12c2016003
|
|
@ -0,0 +1,2 @@
|
|||
/target
|
||||
node_modules
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,28 @@
|
|||
[package]
|
||||
name = "rget"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[dependencies]
|
||||
anyhow = "1.0.75"
|
||||
async-compression = { version = "0.4.3", features = ["gzip", "tokio"] }
|
||||
async-tar = "0.4.2"
|
||||
async-trait = "0.1.73"
|
||||
bb8 = "0.8.1"
|
||||
clap = { version = "4.4.3", features = ["derive"] }
|
||||
deadqueue = "0.2.4"
|
||||
flate2 = "1.0.27"
|
||||
futures = "0.3.28"
|
||||
hex-literal = "0.4.1"
|
||||
hyper = { version = "0.14.27", features = ["full"] }
|
||||
nom = "7.1.3"
|
||||
reqwest = { version = "0.11.20", features = ["rustls-tls-webpki-roots", "json", "stream"] }
|
||||
semver = "1.0.18"
|
||||
serde = { version = "1.0.188", features = ["derive"] }
|
||||
sha1 = "0.10.5"
|
||||
simd-json = "0.10.7"
|
||||
tempfile = "3.8.0"
|
||||
tokio = { version = "1.32.0", features = ["full"] }
|
||||
tokio-util = { version = "0.7.8", features = ["full"] }
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
run:
|
||||
cargo run -- install --dir examples/alotta-files
|
||||
|
|
@ -0,0 +1,118 @@
|
|||
{
|
||||
"name": "alotta-modules",
|
||||
"version": "0.0.1",
|
||||
"dependencies": {
|
||||
"animate.less": "^2.2.0",
|
||||
"autoprefixer": "^10.4.8",
|
||||
"babel-core": "^6.26.3",
|
||||
"babel-eslint": "^10.1.0",
|
||||
"babel-loader": "^8.2.5",
|
||||
"babel-plugin-lodash": "^3.3.4",
|
||||
"babel-plugin-module-resolver": "^4.1.0",
|
||||
"babel-plugin-transform-decorators-legacy": "^1.3.5",
|
||||
"babel-plugin-transform-runtime": "^6.23.0",
|
||||
"babel-polyfill": "^6.26.0",
|
||||
"babel-preset-es2015": "^6.24.1",
|
||||
"babel-preset-react": "^6.24.1",
|
||||
"babel-preset-react-hmre": "^1.1.1",
|
||||
"babel-preset-stage-1": "^6.24.1",
|
||||
"babel-runtime": "^6.26.0",
|
||||
"clean-webpack-plugin": "^4.0.0",
|
||||
"core-decorators": "^0.20.0",
|
||||
"css-loader": "^6.7.1",
|
||||
"css-mqpacker": "^7.0.0",
|
||||
"cssnano": "^5.1.13",
|
||||
"custom-event-polyfill": "^1.0.7",
|
||||
"draft-js": "^0.11.7",
|
||||
"ejs": "^3.1.8",
|
||||
"eslint": "^8.23.0",
|
||||
"eslint-config-airbnb": "^19.0.4",
|
||||
"eslint-import-resolver-webpack": "^0.13.2",
|
||||
"eslint-plugin-import": "^2.26.0",
|
||||
"eslint-plugin-jsx-a11y": "^6.6.1",
|
||||
"eslint-plugin-react": "^7.31.6",
|
||||
"express": "^4.18.1",
|
||||
"express-http-proxy": "^1.6.3",
|
||||
"font-awesome": "^4.7.0",
|
||||
"fready": "^1.0.0",
|
||||
"glob": "^8.0.3",
|
||||
"gulp": "^4.0.2",
|
||||
"gulp-concat": "^2.6.1",
|
||||
"gulp-csslint": "^1.0.1",
|
||||
"gulp-cssnano": "^2.1.3",
|
||||
"gulp-eol": "^0.2.0",
|
||||
"gulp-less": "^5.0.0",
|
||||
"gulp-livereload": "^4.0.2",
|
||||
"gulp-minify-css": "^1.2.4",
|
||||
"gulp-postcss": "^9.0.1",
|
||||
"gulp-rename": "^2.0.0",
|
||||
"gulp-util": "^3.0.8",
|
||||
"happypack": "^5.0.1",
|
||||
"highcharts": "^10.2.1",
|
||||
"highcharts-solid-gauge": "^0.1.7",
|
||||
"history": "^5.3.0",
|
||||
"howler": "^2.2.3",
|
||||
"imports-loader": "^4.0.1",
|
||||
"jquery": "^3.6.1",
|
||||
"jquery-ui": "1.13.2",
|
||||
"js-cookie": "^3.0.1",
|
||||
"json-loader": "^0.5.7",
|
||||
"leftpad": "^0.0.1",
|
||||
"less": "^4.1.3",
|
||||
"lesshat": "^4.1.0",
|
||||
"lodash": "^4.17.21",
|
||||
"medium-draft": "^0.5.18",
|
||||
"mobx": "^6.6.1",
|
||||
"mobx-react": "^7.5.2",
|
||||
"moment": "^2.29.4",
|
||||
"moment-range": "^4.0.2",
|
||||
"moment-timezone": "^0.5.37",
|
||||
"password-policy": "0.0.3",
|
||||
"postcss-reporter": "^7.0.5",
|
||||
"progress": "^2.0.3",
|
||||
"qs": "^6.11.0",
|
||||
"raw-loader": "^4.0.2",
|
||||
"rc-slider": "^10.0.1",
|
||||
"react": "^18.2.0",
|
||||
"react-addons-css-transition-group": "^15.6.2",
|
||||
"react-addons-shallow-compare": "^15.6.3",
|
||||
"react-dnd": "^16.0.1",
|
||||
"react-dnd-html5-backend": "^16.0.1",
|
||||
"react-dom": "^18.2.0",
|
||||
"react-draft-wysiwyg": "^1.15.0",
|
||||
"react-dropzone": "^14.2.2",
|
||||
"react-grid-layout": "^1.3.4",
|
||||
"react-highcharts": "^16.1.0",
|
||||
"react-hot-loader": "4.13.0",
|
||||
"react-input-calendar": "^0.5.4",
|
||||
"react-lazyload": "^3.2.0",
|
||||
"react-measure": "^2.5.2",
|
||||
"react-mixin": "^5.0.0",
|
||||
"react-responsive": "9.0.0-beta.10",
|
||||
"react-responsive-tabs": "^4.4.3",
|
||||
"react-router": "^6.3.0",
|
||||
"react-router-dom": "^6.3.0",
|
||||
"react-select-plus": "1.0.0-rc.3.patch12",
|
||||
"react-skylight": "^0.5.1",
|
||||
"react-sortablejs": "^6.1.4",
|
||||
"react-tappable": "^1.0.4",
|
||||
"react-tooltip": "4.2.21",
|
||||
"react-virtualized": "^9.22.3",
|
||||
"react-waypoint": "^10.3.0",
|
||||
"sortablejs": "^1.15.0",
|
||||
"style-loader": "^3.3.1",
|
||||
"stylelint": "^14.11.0",
|
||||
"superagent": "^8.0.0",
|
||||
"uglify-js": "^3.17.0",
|
||||
"uuid": "^8.3.2",
|
||||
"verge": "^1.10.2",
|
||||
"webpack-bundle-analyzer": "^4.6.1",
|
||||
"webpack-hot-middleware": "^2.25.2",
|
||||
"webpack-notifier": "^1.15.0",
|
||||
"webpack-split-by-path": "^2.0.0",
|
||||
"whatwg-fetch": "^3.6.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
"nan-as": "^1.6.1"
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
max_width = 80
|
||||
tab_spaces = 2
|
||||
wrap_comments = true
|
||||
fn_single_line = true
|
||||
|
|
@ -0,0 +1,194 @@
|
|||
use std::{path::PathBuf, sync::Arc};
|
||||
|
||||
use anyhow::{ensure, Context, Result};
|
||||
use async_compression::tokio::bufread::GzipDecoder;
|
||||
use async_tar::Archive;
|
||||
use futures::TryStreamExt;
|
||||
use reqwest::Client;
|
||||
use semver::{Version, VersionReq};
|
||||
|
||||
use sha1::{Digest, Sha1};
|
||||
use tempfile::NamedTempFile;
|
||||
use tokio::{
|
||||
fs::File,
|
||||
io::{AsyncReadExt, AsyncWriteExt, BufReader},
|
||||
};
|
||||
use tokio_util::{compat::TokioAsyncReadCompatExt, io::StreamReader};
|
||||
|
||||
use crate::{
|
||||
parse_package::{parse_package_json, Package, PackageMetadata},
|
||||
queue_utils::add_reqs_to_queue,
|
||||
TaskQueue,
|
||||
};
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum PackageDef {
|
||||
// Look up from registry
|
||||
Name(String, VersionReq),
|
||||
|
||||
// Local file
|
||||
Local(PathBuf),
|
||||
}
|
||||
|
||||
pub async fn install_package(
|
||||
queue: Arc<TaskQueue>,
|
||||
package_def: PackageDef,
|
||||
client: Client,
|
||||
rget_path: PathBuf,
|
||||
) -> Result<()> {
|
||||
match package_def {
|
||||
PackageDef::Name(name, _version_req) => {
|
||||
let uri = format!("https://registry.npmjs.org/{}", name);
|
||||
let res = client.get(uri).send().await?;
|
||||
let mut body = res.text().await?;
|
||||
let res: PackageMetadata =
|
||||
simd_json::from_slice(unsafe { body.as_bytes_mut() })?;
|
||||
// println!("body {res:?}");
|
||||
|
||||
let matching_versions = res
|
||||
.versions
|
||||
.iter()
|
||||
.map(|(version, object)| {
|
||||
(Version::parse(version).expect("wtf"), object)
|
||||
})
|
||||
// .filter(|(version, _)| version_req.matches(&version))
|
||||
.collect::<Vec<_>>();
|
||||
// println!("Matching versions: {matching_versions:?}");
|
||||
|
||||
// TODO: Take the highest version lmao
|
||||
let (used_version, version_object) = matching_versions.first().unwrap();
|
||||
|
||||
let version = used_version.to_string();
|
||||
let base_path = rget_path.join(format!("{name}@{version}"));
|
||||
let tarball_fut = download_tarball(
|
||||
&client,
|
||||
base_path,
|
||||
&version_object.dist.tarball,
|
||||
&version_object.dist.shasum,
|
||||
);
|
||||
let deps_fut = add_deps(&queue, &client, &name, used_version);
|
||||
|
||||
let (tarball_res, deps_res) = futures::join!(tarball_fut, deps_fut,);
|
||||
tarball_res?;
|
||||
deps_res?;
|
||||
|
||||
Ok::<_, anyhow::Error>(())
|
||||
}
|
||||
|
||||
PackageDef::Local(path) => {
|
||||
let package = parse_package_json(path)?;
|
||||
println!("parsed :3");
|
||||
add_reqs_to_queue(&queue, &package).await?;
|
||||
println!("added :3");
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn add_deps(
|
||||
queue: &TaskQueue,
|
||||
client: &Client,
|
||||
name: &str,
|
||||
used_version: &Version,
|
||||
) -> Result<()> {
|
||||
let uri = format!(
|
||||
"https://registry.npmjs.org/{}/{}",
|
||||
name,
|
||||
used_version.to_string()
|
||||
);
|
||||
let res = client.get(uri).send().await?;
|
||||
let mut body = res.text().await?;
|
||||
let package: Package = simd_json::from_slice(unsafe { body.as_bytes_mut() })?;
|
||||
add_reqs_to_queue(&queue, &package).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn download_tarball(
|
||||
client: &Client,
|
||||
base_path: PathBuf,
|
||||
url: &str,
|
||||
sum: &str,
|
||||
) -> Result<()> {
|
||||
let dest = NamedTempFile::new()?;
|
||||
let (file, path) = dest.keep()?;
|
||||
let mut dest_file = File::from_std(file);
|
||||
|
||||
let res = client.get(url).send().await?;
|
||||
let stream = res.bytes_stream();
|
||||
let mut stream_reader = StreamReader::new(stream.map_err(convert_error));
|
||||
|
||||
// Hashing copy
|
||||
let mut buf = [0u8; 4096];
|
||||
let mut hasher = Sha1::new();
|
||||
loop {
|
||||
let bytes = stream_reader.read(&mut buf).await?;
|
||||
if bytes == 0 {
|
||||
break;
|
||||
}
|
||||
|
||||
dest_file.write(&buf[..bytes]).await?;
|
||||
hasher.update(&buf[..bytes]);
|
||||
}
|
||||
// tokio::io::copy(&mut stream_reader, &mut dest_file).await?;
|
||||
let actual_hash = hasher.finalize();
|
||||
let actual_hash_str = format!("{:x}", actual_hash);
|
||||
ensure!(sum == actual_hash_str, "SHIT the hash is wrong.");
|
||||
std::mem::drop(dest_file);
|
||||
|
||||
tokio::fs::remove_dir_all(&base_path).await;
|
||||
tokio::fs::create_dir_all(&base_path).await?;
|
||||
|
||||
fn convert_error(err: reqwest::Error) -> std::io::Error {
|
||||
eprintln!("WTF? {err}");
|
||||
todo!()
|
||||
}
|
||||
|
||||
// let stream = res.bytes_stream();
|
||||
// let stream_reader = StreamReader::new(stream.map_err(convert_error));
|
||||
let file = File::open(path).await?;
|
||||
let file_reader = BufReader::new(file);
|
||||
let decoder = GzipDecoder::new(file_reader);
|
||||
let decoder = TokioAsyncReadCompatExt::compat(decoder);
|
||||
let archive = Archive::new(decoder);
|
||||
|
||||
let temp_dir = tempfile::tempdir()?;
|
||||
let temp_dir = temp_dir.into_path();
|
||||
archive.unpack(&temp_dir).await?;
|
||||
|
||||
let package_dir = temp_dir.join("package");
|
||||
tokio::fs::rename(package_dir, &base_path)
|
||||
.await
|
||||
.context("could not rename")?;
|
||||
base_path.read_dir()?.try_for_each(|x| {
|
||||
let y = x?;
|
||||
// println!("wtf? {}", y.path().display());
|
||||
Ok::<_, anyhow::Error>(())
|
||||
})?;
|
||||
// println!("Unpacked {}", base_path.display());
|
||||
// let files = archive.entries()?;
|
||||
// let files = files.try_collect::<Vec<_>>().await?;
|
||||
// for mut file in files {
|
||||
// let tar_path = file.path()?;
|
||||
// let without_prefix = match tar_path.strip_prefix("package/") {
|
||||
// Ok(v) => v,
|
||||
// Err(_) => continue,
|
||||
// };
|
||||
// let mut new_path = base_path.clone();
|
||||
// new_path.extend(without_prefix);
|
||||
// let parent = new_path.parent().unwrap();
|
||||
// tokio::fs::create_dir_all(parent).await?;
|
||||
// file.unpack(&new_path).await?;
|
||||
// println!("- Unpacked {}", new_path.display());
|
||||
// // println!("new path: {new_path:?}");
|
||||
// }
|
||||
// println!("Installed to {}", base_path.display());
|
||||
// // println!("files: {:?}", files);
|
||||
|
||||
// // let mut content = Cursor::new(res.bytes().await?);
|
||||
// // println!("dest: {dest:?}");
|
||||
// // io::copy(&mut content, &mut dest).await?;
|
||||
|
||||
// // dest.seek(SeekFrom::Start(0)).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -0,0 +1,99 @@
|
|||
#[macro_use]
|
||||
extern crate async_trait;
|
||||
#[macro_use]
|
||||
extern crate hex_literal;
|
||||
#[macro_use]
|
||||
extern crate serde;
|
||||
|
||||
mod install_package;
|
||||
mod parse_package;
|
||||
mod queue_utils;
|
||||
mod semver;
|
||||
|
||||
use std::{path::PathBuf, sync::Arc, time::Duration};
|
||||
|
||||
use anyhow::Result;
|
||||
use clap::{Parser, Subcommand};
|
||||
|
||||
use reqwest::Client;
|
||||
|
||||
use tokio::time::sleep;
|
||||
|
||||
use crate::install_package::{install_package, PackageDef};
|
||||
|
||||
pub type TaskQueue = deadqueue::limited::Queue<PackageDef>;
|
||||
|
||||
#[derive(Debug, Parser)]
|
||||
struct Opt {
|
||||
#[clap(subcommand)]
|
||||
command: Command,
|
||||
}
|
||||
|
||||
#[derive(Debug, Subcommand)]
|
||||
enum Command {
|
||||
Install {
|
||||
#[clap(long = "dir")]
|
||||
dir: PathBuf,
|
||||
},
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<()> {
|
||||
let opt = Opt::parse();
|
||||
println!("Hello, world! {opt:?}");
|
||||
|
||||
let client = Client::builder()
|
||||
.use_rustls_tls()
|
||||
.pool_idle_timeout(Duration::from_secs(30))
|
||||
.build()?;
|
||||
|
||||
match opt.command {
|
||||
Command::Install { dir } => {
|
||||
// Parse package.json
|
||||
let package_json_path = dir.join("package.json");
|
||||
let node_modules_path = dir.join("node_modules");
|
||||
let rget_path = node_modules_path.join(".rget");
|
||||
tokio::fs::create_dir_all(&rget_path).await?;
|
||||
// let parsed = parse_package_json(package_json_path)?;
|
||||
|
||||
let queue = Arc::new(TaskQueue::new(1_000));
|
||||
|
||||
// Handle the first one specifically
|
||||
let task = PackageDef::Local(package_json_path);
|
||||
// queue.push(task).await;
|
||||
install_package(queue.clone(), task, client.clone(), rget_path.clone())
|
||||
.await?;
|
||||
|
||||
for _worker in 0..10 {
|
||||
let queue = queue.clone();
|
||||
let client = client.clone();
|
||||
let rget_path = rget_path.clone();
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
let queue = queue.clone();
|
||||
let client = client.clone();
|
||||
let task = queue.pop().await;
|
||||
// println!("Handling task {task:?}");
|
||||
match install_package(queue, task, client, rget_path.clone()).await
|
||||
{
|
||||
Ok(_) => {}
|
||||
Err(err) => {
|
||||
eprintln!("WTF? {err} {err:?}");
|
||||
}
|
||||
}
|
||||
// println!("worker[{}] processing task[{}] ...", worker, task);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
while queue.len() > 0 {
|
||||
// println!("Waiting for workers to finish...");
|
||||
sleep(Duration::from_millis(1_000)).await;
|
||||
}
|
||||
|
||||
println!("Done!");
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
// https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md
|
||||
|
||||
use std::{collections::HashMap, fs::File, path::Path};
|
||||
|
||||
use anyhow::Result;
|
||||
|
||||
// TODO: Optimize using owning ref?
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct Package {
|
||||
pub name: String,
|
||||
pub version: String,
|
||||
|
||||
#[serde(default)]
|
||||
pub dependencies: HashMap<String, String>,
|
||||
|
||||
#[serde(rename = "devDependencies", default)]
|
||||
pub dev_dependencies: HashMap<String, String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct PackageMetadata {
|
||||
pub name: String,
|
||||
pub versions: HashMap<String, VersionObject>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct VersionObject {
|
||||
pub dist: Dist,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct Dist {
|
||||
pub shasum: String,
|
||||
pub tarball: String,
|
||||
}
|
||||
|
||||
pub fn parse_package_json(path: impl AsRef<Path>) -> Result<Package> {
|
||||
let parsed: Package = {
|
||||
let file = File::open(path)?;
|
||||
simd_json::from_reader(file)?
|
||||
};
|
||||
|
||||
Ok(parsed)
|
||||
}
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
use anyhow::{Context, Result};
|
||||
use semver::VersionReq;
|
||||
|
||||
use crate::{install_package::PackageDef, parse_package::Package, TaskQueue};
|
||||
|
||||
pub async fn add_reqs_to_queue(
|
||||
queue: &TaskQueue,
|
||||
package: &Package,
|
||||
) -> Result<()> {
|
||||
for (package_name, version_req) in package.dependencies.iter() {
|
||||
let version_req = VersionReq::parse(version_req)
|
||||
.with_context(|| format!("could not parse version {version_req:?}"))?;
|
||||
queue
|
||||
.push(PackageDef::Name(package_name.to_owned(), version_req))
|
||||
.await;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
// npm has a kinda fucked version of semver?
|
||||
// https://github.com/npm/node-semver
|
||||
|
||||
use nom::{
|
||||
branch::alt, bytes::complete::take_while1, character::is_alphanumeric,
|
||||
};
|
||||
|
||||
pub fn parse_semver(wtf: &str) {}
|
||||
|
||||
fn is_part(c: u8) -> bool {
|
||||
c == b'-' || is_alphanumeric(c)
|
||||
}
|
||||
|
||||
// fn part() {
|
||||
// alt((nr, take_while1(is_part)))
|
||||
// }
|
||||
Loading…
Reference in New Issue