diff --git a/Cargo.lock b/Cargo.lock index 59dfa9ab380..cc0759679ae 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -238,6 +238,7 @@ version = "0.2.2" dependencies = [ "anyhow", "base64", + "cargo-registry-index", "cargo-registry-markdown", "cargo-registry-s3", "chrono", @@ -263,7 +264,6 @@ dependencies = [ "flate2", "futures-channel", "futures-util", - "git2", "hex", "http", "hyper", @@ -299,6 +299,20 @@ dependencies = [ "url", ] +[[package]] +name = "cargo-registry-index" +version = "0.0.0" +dependencies = [ + "anyhow", + "base64", + "dotenv", + "git2", + "serde", + "serde_json", + "tempfile", + "url", +] + [[package]] name = "cargo-registry-markdown" version = "0.0.0" diff --git a/Cargo.toml b/Cargo.toml index 1777e5a6f94..4e216d4ee02 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,6 +31,7 @@ rustdoc-args = [ [dependencies] anyhow = "=1.0.53" base64 = "=0.13.0" +cargo-registry-index = { path = "cargo-registry-index" } cargo-registry-markdown = { path = "cargo-registry-markdown" } cargo-registry-s3 = { path = "cargo-registry-s3" } chrono = { version = "=0.4.19", features = ["serde"] } @@ -56,7 +57,6 @@ dotenv = "=0.15.0" flate2 = "=1.0.22" futures-channel = { version = "=0.3.19", default-features = false } futures-util = "=0.3.19" -git2 = "=0.13.25" hex = "=0.4.3" http = "=0.2.6" hyper = { version = "=0.14.16", features = ["client", "http1"] } @@ -89,6 +89,7 @@ tracing-subscriber = { version = "=0.3.6", features = ["env-filter"] } url = "=2.2.2" [dev-dependencies] +cargo-registry-index = { path = "cargo-registry-index", features = ["testing"] } claim = "=0.5.0" conduit-test = "=0.10.0" hyper-tls = "=0.5.0" diff --git a/cargo-registry-index/Cargo.toml b/cargo-registry-index/Cargo.toml new file mode 100644 index 00000000000..3601023fb2d --- /dev/null +++ b/cargo-registry-index/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "cargo-registry-index" +version = "0.0.0" +license = "MIT OR Apache-2.0" +repository = "https://github.com/rust-lang/crates.io" +description = "crates.io package index utilities" +edition = "2018" +resolver = "2" + +[lib] +path = "lib.rs" + +[features] +testing = ["serde_json"] + +[dependencies] +anyhow = "=1.0.53" +base64 = "=0.13.0" +dotenv = "=0.15.0" +git2 = "=0.13.25" +serde = { version = "=1.0.135", features = ["derive"] } +tempfile = "=3.3.0" +url = "=2.2.2" +serde_json = { version = "=1.0.78", optional = true } diff --git a/src/git.rs b/cargo-registry-index/lib.rs similarity index 98% rename from src/git.rs rename to cargo-registry-index/lib.rs index 4e9eeaa7622..0e7a8c8acb4 100644 --- a/src/git.rs +++ b/cargo-registry-index/lib.rs @@ -1,3 +1,9 @@ +#[macro_use] +extern crate serde; + +#[cfg(feature = "testing")] +pub mod testing; + use anyhow::{anyhow, Context}; use std::collections::HashMap; use std::io::Write; @@ -7,8 +13,6 @@ use std::process::Command; use tempfile::TempDir; use url::Url; -use crate::models::DependencyKind; - static DEFAULT_GIT_SSH_USERNAME: &str = "git"; #[derive(Clone)] @@ -148,6 +152,14 @@ pub struct Dependency { pub package: Option, } +#[derive(Copy, Clone, Serialize, Deserialize, Debug)] +#[serde(rename_all = "lowercase")] +pub enum DependencyKind { + Normal, + Build, + Dev, +} + pub struct RepositoryConfig { pub index_location: Url, pub credentials: Credentials, diff --git a/src/tests/git.rs b/cargo-registry-index/testing.rs similarity index 92% rename from src/tests/git.rs rename to cargo-registry-index/testing.rs index 39077043ec1..df3665f65cd 100644 --- a/src/tests/git.rs +++ b/cargo-registry-index/testing.rs @@ -25,13 +25,10 @@ impl UpstreamIndex { } /// Obtain a list of crates from the index HEAD - pub fn crates_from_index_head( - &self, - crate_name: &str, - ) -> anyhow::Result> { + pub fn crates_from_index_head(&self, crate_name: &str) -> anyhow::Result> { let repo = &self.repository; - let path = cargo_registry::git::Repository::relative_index_file(crate_name); + let path = crate::Repository::relative_index_file(crate_name); let head = repo.head()?; let tree = head.peel_to_tree()?; diff --git a/src/background_jobs.rs b/src/background_jobs.rs index 97492573e73..caad22c400d 100644 --- a/src/background_jobs.rs +++ b/src/background_jobs.rs @@ -5,8 +5,8 @@ use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; use swirl::PerformError; use crate::db::{DieselPool, DieselPooledConn, PoolError}; -use crate::git::Repository; use crate::uploaders::Uploader; +use cargo_registry_index::Repository; impl<'a> swirl::db::BorrowedConnection<'a> for DieselPool { type Connection = DieselPooledConn<'a>; diff --git a/src/bin/background-worker.rs b/src/bin/background-worker.rs index d8d5387cd9f..80111d66296 100644 --- a/src/bin/background-worker.rs +++ b/src/bin/background-worker.rs @@ -13,8 +13,8 @@ #![warn(clippy::all, rust_2018_idioms)] use cargo_registry::config; -use cargo_registry::git::{Repository, RepositoryConfig}; use cargo_registry::{background_jobs::*, db}; +use cargo_registry_index::{Repository, RepositoryConfig}; use diesel::r2d2; use reqwest::blocking::Client; use std::sync::{Arc, Mutex}; diff --git a/src/controllers/krate/publish.rs b/src/controllers/krate/publish.rs index 9ab5b588d7a..361d808bd17 100644 --- a/src/controllers/krate/publish.rs +++ b/src/controllers/krate/publish.rs @@ -10,7 +10,6 @@ use std::sync::Arc; use swirl::Job; use crate::controllers::cargo_prelude::*; -use crate::git; use crate::models::{ insert_version_owner_action, Badge, Category, Crate, DependencyKind, Keyword, NewCrate, NewVersion, Rights, VersionAction, @@ -230,7 +229,7 @@ pub fn publish(req: &mut dyn RequestExt) -> EndpointResult { }; // Register this crate in our local git repo. - let git_crate = git::Crate { + let git_crate = cargo_registry_index::Crate { name: name.0, vers: vers.to_string(), cksum: hex_cksum, @@ -313,7 +312,7 @@ pub fn add_dependencies( conn: &PgConnection, deps: &[EncodableCrateDependency], target_version_id: i32, -) -> AppResult> { +) -> AppResult> { use self::dependencies::dsl::*; use diesel::insert_into; @@ -348,14 +347,14 @@ pub fn add_dependencies( }; Ok(( - git::Dependency { + cargo_registry_index::Dependency { name, req: dep.version_req.to_string(), features: dep.features.iter().map(|s| s.0.to_string()).collect(), optional: dep.optional, default_features: dep.default_features, target: dep.target.clone(), - kind: dep.kind.or(Some(DependencyKind::Normal)), + kind: dep.kind.or(Some(DependencyKind::Normal)).map(|dk| dk.into()), package, }, ( diff --git a/src/lib.rs b/src/lib.rs index 09e9c8f035d..e2589255251 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -40,7 +40,6 @@ pub mod config; pub mod db; mod downloads_counter; pub mod email; -pub mod git; pub mod github; pub mod metrics; pub mod middleware; diff --git a/src/models/dependency.rs b/src/models/dependency.rs index 37688a5d05f..bd226a0a5ce 100644 --- a/src/models/dependency.rs +++ b/src/models/dependency.rs @@ -4,6 +4,7 @@ use diesel::sql_types::Integer; use crate::models::{Crate, Version}; use crate::schema::*; +use cargo_registry_index::DependencyKind as IndexDependencyKind; #[derive(Identifiable, Associations, Debug, Queryable, QueryableByName)] #[belongs_to(Version)] @@ -42,6 +43,16 @@ pub enum DependencyKind { // if you add a kind here, be sure to update `from_row` below. } +impl From for IndexDependencyKind { + fn from(dk: DependencyKind) -> Self { + match dk { + DependencyKind::Normal => IndexDependencyKind::Normal, + DependencyKind::Build => IndexDependencyKind::Build, + DependencyKind::Dev => IndexDependencyKind::Dev, + } + } +} + impl FromSql for DependencyKind { fn from_sql(bytes: Option<&[u8]>) -> deserialize::Result { match >::from_sql(bytes)? { diff --git a/src/tests/all.rs b/src/tests/all.rs index 4ad6ceaab71..493a1b63a75 100644 --- a/src/tests/all.rs +++ b/src/tests/all.rs @@ -37,7 +37,6 @@ mod builders; mod categories; mod category; mod dump_db; -mod git; mod keyword; mod krate; mod metrics; diff --git a/src/tests/util/test_app.rs b/src/tests/util/test_app.rs index e95752eca53..b6ae2c48469 100644 --- a/src/tests/util/test_app.rs +++ b/src/tests/util/test_app.rs @@ -1,17 +1,12 @@ use super::{MockAnonymousUser, MockCookieUser, MockTokenUser}; -use crate::git::UpstreamIndex; use crate::record; use crate::util::{chaosproxy::ChaosProxy, fresh_schema::FreshSchema}; use cargo_registry::config; -use cargo_registry::{ - background_jobs::Environment, - db::DieselPool, - git::{Credentials, RepositoryConfig}, - App, Emails, -}; +use cargo_registry::{background_jobs::Environment, db::DieselPool, App, Emails}; +use cargo_registry_index::testing::UpstreamIndex; +use cargo_registry_index::{Credentials, Repository as WorkerRepository, RepositoryConfig}; use std::{rc::Rc, sync::Arc, time::Duration}; -use cargo_registry::git::Repository as WorkerRepository; use diesel::PgConnection; use reqwest::{blocking::Client, Proxy}; use std::collections::HashSet; @@ -133,7 +128,7 @@ impl TestApp { } /// Obtain a list of crates from the index HEAD - pub fn crates_from_index_head(&self, crate_name: &str) -> Vec { + pub fn crates_from_index_head(&self, crate_name: &str) -> Vec { self.upstream_index() .crates_from_index_head(crate_name) .unwrap() diff --git a/src/worker/git.rs b/src/worker/git.rs index e8e79bd5217..54319be8be1 100644 --- a/src/worker/git.rs +++ b/src/worker/git.rs @@ -1,7 +1,7 @@ use crate::background_jobs::Environment; -use crate::git::Crate; use crate::schema; use anyhow::Context; +use cargo_registry_index::Crate; use chrono::Utc; use diesel::prelude::*; use std::fs::{self, OpenOptions};