From 69e51b4dcf986c86251b7177bd5db9f9b9d8d604 Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Tue, 11 Aug 2020 22:09:34 +0300 Subject: [PATCH 001/139] Run git gc periodically on the crates.io index #778 --- src/config.rs | 4 ++++ src/utils/daemon.rs | 22 ++++++++++++++++++++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/src/config.rs b/src/config.rs index 936018b6a..497ad575d 100644 --- a/src/config.rs +++ b/src/config.rs @@ -30,6 +30,9 @@ pub struct Config { pub(crate) max_file_size_html: usize, // The most memory that can be used to parse an HTML file pub(crate) max_parse_memory: usize, + + // Time between 'git gc --auto' calls in seconds + pub(crate) registry_gc_interval: u64, } impl Config { @@ -61,6 +64,7 @@ impl Config { // LOL HTML only uses as much memory as the size of the start tag! // https://github.com/rust-lang/docs.rs/pull/930#issuecomment-667729380 max_parse_memory: env("DOCSRS_MAX_PARSE_MEMORY", 5 * 1024 * 1024)?, + registry_gc_interval: env("REGISTRY_GC_INTERVAL", 60 * 60)?, }) } diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs index 990516f5d..2b67dc76e 100644 --- a/src/utils/daemon.rs +++ b/src/utils/daemon.rs @@ -13,18 +13,31 @@ use failure::Error; use log::{debug, error, info}; use std::sync::Arc; use std::thread; -use std::time::Duration; +use std::time::{Duration, Instant}; +use std::process::Command; + + +fn run_git_gc() { + Command::new("git") + .args(&["gc"]) + .output() + .expect("Failed to execute git gc"); +} fn start_registry_watcher( opts: DocBuilderOptions, pool: Pool, build_queue: Arc, + config: Arc, ) -> Result<(), Error> { thread::Builder::new() .name("registry index reader".to_string()) .spawn(move || { // space this out to prevent it from clashing against the queue-builder thread on launch thread::sleep(Duration::from_secs(30)); + let mut last_gc = Instant::now(); + run_git_gc(); + loop { let mut doc_builder = DocBuilder::new(opts.clone(), pool.clone(), build_queue.clone()); @@ -39,6 +52,10 @@ fn start_registry_watcher( } } + if last_gc.elapsed().as_secs() >= config.registry_gc_interval { + run_git_gc(); + last_gc = Instant::now(); + } thread::sleep(Duration::from_secs(60)); } })?; @@ -60,7 +77,8 @@ pub fn start_daemon( if enable_registry_watcher { // check new crates every minute - start_registry_watcher(dbopts.clone(), db.clone(), build_queue.clone())?; + start_registry_watcher(dbopts.clone(), db.clone(), build_queue.clone(), + config.clone())?; } // build new crates every minute From 9cb547f3193be040efaaba5f814c30c9412da6f1 Mon Sep 17 00:00:00 2001 From: ohaddahan Date: Tue, 11 Aug 2020 22:14:34 +0300 Subject: [PATCH 002/139] Update config.rs --- src/config.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config.rs b/src/config.rs index 497ad575d..871002745 100644 --- a/src/config.rs +++ b/src/config.rs @@ -31,7 +31,7 @@ pub struct Config { // The most memory that can be used to parse an HTML file pub(crate) max_parse_memory: usize, - // Time between 'git gc --auto' calls in seconds + // Time between 'git gc' calls in seconds pub(crate) registry_gc_interval: u64, } From 3d0ec4e9e187620cdc839a3356720dfab0a812db Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Tue, 11 Aug 2020 22:20:18 +0300 Subject: [PATCH 003/139] Fixed "cargo fmt" --- .gitignore | 1 + src/utils/daemon.rs | 11 +++++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index d7823cbaf..5b7d9caab 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ target .vagrant .rustwide .rustwide-docker +.idea/ \ No newline at end of file diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs index 2b67dc76e..1a3366207 100644 --- a/src/utils/daemon.rs +++ b/src/utils/daemon.rs @@ -11,11 +11,10 @@ use crate::{ use chrono::{Timelike, Utc}; use failure::Error; use log::{debug, error, info}; +use std::process::Command; use std::sync::Arc; use std::thread; use std::time::{Duration, Instant}; -use std::process::Command; - fn run_git_gc() { Command::new("git") @@ -77,8 +76,12 @@ pub fn start_daemon( if enable_registry_watcher { // check new crates every minute - start_registry_watcher(dbopts.clone(), db.clone(), build_queue.clone(), - config.clone())?; + start_registry_watcher( + dbopts.clone(), + db.clone(), + build_queue.clone(), + config.clone(), + )?; } // build new crates every minute From ce5bbcf5725bedb44c602989ca0590ae50e04a4e Mon Sep 17 00:00:00 2001 From: ohaddahan Date: Tue, 11 Aug 2020 23:22:02 +0300 Subject: [PATCH 004/139] Update src/utils/daemon.rs Co-authored-by: Chase Wilson --- src/utils/daemon.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs index 1a3366207..c1c40b111 100644 --- a/src/utils/daemon.rs +++ b/src/utils/daemon.rs @@ -34,8 +34,8 @@ fn start_registry_watcher( .spawn(move || { // space this out to prevent it from clashing against the queue-builder thread on launch thread::sleep(Duration::from_secs(30)); - let mut last_gc = Instant::now(); run_git_gc(); + let mut last_gc = Instant::now(); loop { let mut doc_builder = From 94f8ac308f523b8f2cf9094a6a8908383a80c08a Mon Sep 17 00:00:00 2001 From: ohaddahan Date: Tue, 11 Aug 2020 23:22:11 +0300 Subject: [PATCH 005/139] Update src/utils/daemon.rs Co-authored-by: Chase Wilson --- src/utils/daemon.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs index c1c40b111..dc0b5835d 100644 --- a/src/utils/daemon.rs +++ b/src/utils/daemon.rs @@ -17,10 +17,13 @@ use std::thread; use std::time::{Duration, Instant}; fn run_git_gc() { - Command::new("git") + let gc = Command::new("git") .args(&["gc"]) - .output() - .expect("Failed to execute git gc"); + .output(); + + if let Err(err) = gc { + log::error!("failed to run `git gc`: {:?}", err); + } } fn start_registry_watcher( From 6335dd56103c6f1cfda3d156105d757f57c3e336 Mon Sep 17 00:00:00 2001 From: ohaddahan Date: Tue, 11 Aug 2020 23:22:27 +0300 Subject: [PATCH 006/139] Update src/utils/daemon.rs Co-authored-by: Chase Wilson --- src/utils/daemon.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs index dc0b5835d..e40de0892 100644 --- a/src/utils/daemon.rs +++ b/src/utils/daemon.rs @@ -18,7 +18,7 @@ use std::time::{Duration, Instant}; fn run_git_gc() { let gc = Command::new("git") - .args(&["gc"]) + .args(&["gc", "--auto"]) .output(); if let Err(err) = gc { From d3e2c80f744ce80d0cb98ae0a20eaf50b32a3e52 Mon Sep 17 00:00:00 2001 From: ohaddahan Date: Tue, 11 Aug 2020 23:22:36 +0300 Subject: [PATCH 007/139] Update src/config.rs Co-authored-by: Chase Wilson --- src/config.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/config.rs b/src/config.rs index 871002745..86fc11db5 100644 --- a/src/config.rs +++ b/src/config.rs @@ -30,7 +30,6 @@ pub struct Config { pub(crate) max_file_size_html: usize, // The most memory that can be used to parse an HTML file pub(crate) max_parse_memory: usize, - // Time between 'git gc' calls in seconds pub(crate) registry_gc_interval: u64, } From 8d06f5140984f30e456aceb262257cf4964155a1 Mon Sep 17 00:00:00 2001 From: ohaddahan Date: Tue, 11 Aug 2020 23:22:43 +0300 Subject: [PATCH 008/139] Update .gitignore Co-authored-by: Chase Wilson --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index 5b7d9caab..d7823cbaf 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,3 @@ target .vagrant .rustwide .rustwide-docker -.idea/ \ No newline at end of file From e323a2f6d1cfa8c9a90433c8cc2a1b5a05da13ba Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Wed, 12 Aug 2020 09:01:44 +0300 Subject: [PATCH 009/139] Added --auto --- src/utils/daemon.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs index 1a3366207..707ed0d99 100644 --- a/src/utils/daemon.rs +++ b/src/utils/daemon.rs @@ -18,7 +18,7 @@ use std::time::{Duration, Instant}; fn run_git_gc() { Command::new("git") - .args(&["gc"]) + .args(&["gc", "--auto"]) .output() .expect("Failed to execute git gc"); } From 29b11130d4091d1ff7a319fdb94abd8d40931d83 Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Wed, 12 Aug 2020 09:11:54 +0300 Subject: [PATCH 010/139] Fix cargo fmt --- src/utils/daemon.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs index e40de0892..4535b5f83 100644 --- a/src/utils/daemon.rs +++ b/src/utils/daemon.rs @@ -17,10 +17,8 @@ use std::thread; use std::time::{Duration, Instant}; fn run_git_gc() { - let gc = Command::new("git") - .args(&["gc", "--auto"]) - .output(); - + let gc = Command::new("git").args(&["gc", "--auto"]).output(); + if let Err(err) = gc { log::error!("failed to run `git gc`: {:?}", err); } From 014338c501917b2deadddd80b29be4467e5bc2cc Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Wed, 12 Aug 2020 09:51:30 +0300 Subject: [PATCH 011/139] Adding pre-commit hook, adding dead-code lint since without it clippy rails with "unused static" --- README.md | 6 ++++++ git_hooks/pre-commit | 19 +++++++++++++++++++ src/web/metrics.rs | 2 ++ 3 files changed, 27 insertions(+) create mode 100755 git_hooks/pre-commit diff --git a/README.md b/README.md index bca9dc2d6..1bee1c15b 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,12 @@ The recommended way to develop docs.rs is a combination of `cargo run` for the main binary and [docker-compose](https://docs.docker.com/compose/) for the external services. This gives you reasonable incremental build times without having to add new users and packages to your host machine. +### Git Hooks +For ease of use, `git_hooks` directory contains useful `git hooks` to make your development easier. +```bash +cp git_hooks/* .git/hooks/ +``` + ### Dependencies Docs.rs requires at least the following native C dependencies. diff --git a/git_hooks/pre-commit b/git_hooks/pre-commit new file mode 100755 index 000000000..8ae45b3f3 --- /dev/null +++ b/git_hooks/pre-commit @@ -0,0 +1,19 @@ +#!/usr/bin/env sh +cargo fmt -- --check +if [ "$?" != "0" ] ; then + echo "\n" + echo "\033[0;31mpre-commit hook failed during:\033[0m" + echo "\033[0;31m\tcargo fmt -- --check\033[0m" + exit 1 +fi +cargo clippy --locked -- -D warnings +if [ "$?" != "0" ] ; then + echo "\n" + echo "\033[0;31mpre-commit hook failed during:\033[0m" + echo "\033[0;31m\tclippy --locked -- -D warning\033[0m" + exit 1 +fi + +echo "\n" +echo "\033[0;32mpre-commit hook succeeded\033[0m" +exit 0 \ No newline at end of file diff --git a/src/web/metrics.rs b/src/web/metrics.rs index 0d0742593..678be49a4 100644 --- a/src/web/metrics.rs +++ b/src/web/metrics.rs @@ -130,6 +130,7 @@ pub static MAX_DB_CONNECTIONS: Lazy = Lazy::new(|| { }); #[cfg(not(windows))] +#[allow(dead_code)] pub static OPEN_FILE_DESCRIPTORS: Lazy = Lazy::new(|| { register_int_gauge!( "docsrs_open_file_descriptors", @@ -139,6 +140,7 @@ pub static OPEN_FILE_DESCRIPTORS: Lazy = Lazy::new(|| { }); #[cfg(not(windows))] +#[allow(dead_code)] pub static CURRENTLY_RUNNING_THREADS: Lazy = Lazy::new(|| { register_int_gauge!( "docsrs_running_threads", From 45892ff0e68bb7920d5311ff0bde896c1bfeb8eb Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Wed, 12 Aug 2020 19:59:52 +0300 Subject: [PATCH 012/139] Changed envar prefix and replace echo with printf for pre-commit hook --- git_hooks/pre-commit | 16 ++++++++-------- src/config.rs | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/git_hooks/pre-commit b/git_hooks/pre-commit index 8ae45b3f3..f6855e88e 100755 --- a/git_hooks/pre-commit +++ b/git_hooks/pre-commit @@ -1,19 +1,19 @@ #!/usr/bin/env sh cargo fmt -- --check if [ "$?" != "0" ] ; then - echo "\n" - echo "\033[0;31mpre-commit hook failed during:\033[0m" - echo "\033[0;31m\tcargo fmt -- --check\033[0m" + printf "\n" + printf "\033[0;31mpre-commit hook failed during:\033[0m\n" + printf "\033[0;31m\tcargo fmt -- --check\033[0m\n" exit 1 fi cargo clippy --locked -- -D warnings if [ "$?" != "0" ] ; then - echo "\n" - echo "\033[0;31mpre-commit hook failed during:\033[0m" - echo "\033[0;31m\tclippy --locked -- -D warning\033[0m" + printf "\n" + printf "\033[0;31mpre-commit hook failed during:\033[0m\n" + printf "\033[0;31m\tclippy --locked -- -D warning\033[0m\n" exit 1 fi -echo "\n" -echo "\033[0;32mpre-commit hook succeeded\033[0m" +printf "\n" +printf "\033[0;32mpre-commit hook succeeded\033[0m\n" exit 0 \ No newline at end of file diff --git a/src/config.rs b/src/config.rs index 86fc11db5..a3ae7dc9f 100644 --- a/src/config.rs +++ b/src/config.rs @@ -63,7 +63,7 @@ impl Config { // LOL HTML only uses as much memory as the size of the start tag! // https://github.com/rust-lang/docs.rs/pull/930#issuecomment-667729380 max_parse_memory: env("DOCSRS_MAX_PARSE_MEMORY", 5 * 1024 * 1024)?, - registry_gc_interval: env("REGISTRY_GC_INTERVAL", 60 * 60)?, + registry_gc_interval: env("DOCSRS_REGISTRY_GC_INTERVAL", 60 * 60)?, }) } From 1e4d83e83fc4c24d755a1317bd1a76b745249f50 Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Wed, 12 Aug 2020 20:33:36 +0300 Subject: [PATCH 013/139] Removed allow(dead_code) and not(windows) , now OPEN_FILE_DESCRIPTORS and CURRENTLY_RUNNING_THREADS declaration match the usage, "target_os = linux" --- src/web/metrics.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/web/metrics.rs b/src/web/metrics.rs index 678be49a4..18a11ed13 100644 --- a/src/web/metrics.rs +++ b/src/web/metrics.rs @@ -129,8 +129,7 @@ pub static MAX_DB_CONNECTIONS: Lazy = Lazy::new(|| { .unwrap() }); -#[cfg(not(windows))] -#[allow(dead_code)] +#[cfg(target_os = "linux")] pub static OPEN_FILE_DESCRIPTORS: Lazy = Lazy::new(|| { register_int_gauge!( "docsrs_open_file_descriptors", @@ -139,8 +138,7 @@ pub static OPEN_FILE_DESCRIPTORS: Lazy = Lazy::new(|| { .unwrap() }); -#[cfg(not(windows))] -#[allow(dead_code)] +#[cfg(target_os = "linux")] pub static CURRENTLY_RUNNING_THREADS: Lazy = Lazy::new(|| { register_int_gauge!( "docsrs_running_threads", From 39e5b26d54b8b85019c190e96ceda77b5438fc1f Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Thu, 13 Aug 2020 15:57:22 +0300 Subject: [PATCH 014/139] Updated pre-commit script and instructions --- {git_hooks => .git_hooks}/pre-commit | 9 ++++----- README.md | 2 +- 2 files changed, 5 insertions(+), 6 deletions(-) rename {git_hooks => .git_hooks}/pre-commit (76%) diff --git a/git_hooks/pre-commit b/.git_hooks/pre-commit similarity index 76% rename from git_hooks/pre-commit rename to .git_hooks/pre-commit index f6855e88e..872c2cdda 100755 --- a/git_hooks/pre-commit +++ b/.git_hooks/pre-commit @@ -1,13 +1,12 @@ #!/usr/bin/env sh -cargo fmt -- --check -if [ "$?" != "0" ] ; then +if ! cargo fmt -- --check ; then printf "\n" printf "\033[0;31mpre-commit hook failed during:\033[0m\n" printf "\033[0;31m\tcargo fmt -- --check\033[0m\n" exit 1 fi -cargo clippy --locked -- -D warnings -if [ "$?" != "0" ] ; then + +if ! cargo clippy --locked -- -D warnings ; then printf "\n" printf "\033[0;31mpre-commit hook failed during:\033[0m\n" printf "\033[0;31m\tclippy --locked -- -D warning\033[0m\n" @@ -16,4 +15,4 @@ fi printf "\n" printf "\033[0;32mpre-commit hook succeeded\033[0m\n" -exit 0 \ No newline at end of file +exit 0 diff --git a/README.md b/README.md index 1bee1c15b..61bf3439b 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ This gives you reasonable incremental build times without having to add new user ### Git Hooks For ease of use, `git_hooks` directory contains useful `git hooks` to make your development easier. ```bash -cp git_hooks/* .git/hooks/ +cd .git/hooks && ln -s ../../.git_hooks/* . && cd ../.. ``` ### Dependencies From 3be80f580fec9d4389b1dcb4d9f8b8e645db7757 Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Thu, 13 Aug 2020 15:58:30 +0300 Subject: [PATCH 015/139] Added newline at end of pre-commit script --- .git_hooks/pre-commit | 1 + 1 file changed, 1 insertion(+) diff --git a/.git_hooks/pre-commit b/.git_hooks/pre-commit index 872c2cdda..105684ef7 100755 --- a/.git_hooks/pre-commit +++ b/.git_hooks/pre-commit @@ -16,3 +16,4 @@ fi printf "\n" printf "\033[0;32mpre-commit hook succeeded\033[0m\n" exit 0 + From 1ba6fd59059243e4f8fe9174e7b05196b3915bf7 Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Thu, 13 Aug 2020 17:54:29 +0300 Subject: [PATCH 016/139] Move run_git_gc to Index --- src/docbuilder/queue.rs | 4 ++++ src/index/mod.rs | 9 +++++++++ src/utils/daemon.rs | 12 +----------- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/src/docbuilder/queue.rs b/src/docbuilder/queue.rs index ae3bfc76e..900c7d67d 100644 --- a/src/docbuilder/queue.rs +++ b/src/docbuilder/queue.rs @@ -7,6 +7,10 @@ use crates_index_diff::ChangeKind; use log::{debug, error}; impl DocBuilder { + pub fn run_git_gc(&self) { + self.index.run_git_gc(); + } + /// Updates registry index repository and adds new crates into build queue. /// Returns the number of crates added pub fn get_new_crates(&mut self) -> Result { diff --git a/src/index/mod.rs b/src/index/mod.rs index acb86a8fc..a4fef6e0e 100644 --- a/src/index/mod.rs +++ b/src/index/mod.rs @@ -1,4 +1,5 @@ use std::path::{Path, PathBuf}; +use std::process::Command; use url::Url; @@ -50,6 +51,14 @@ impl Index { Ok(Self { path, api }) } + pub fn run_git_gc(&self) { + let cmd = format!("cd {} && gc --auto", self.path.to_str().unwrap()); + let gc = Command::new("sh").args(&["-c", cmd.as_str()]).output(); + if let Err(err) = gc { + log::error!("Failed to run `{}`: {:?}", cmd, err); + } + } + pub(crate) fn diff(&self) -> Result { let diff = crates_index_diff::Index::from_path_or_cloned(&self.path) .context("re-opening registry index for diff")?; diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs index 4535b5f83..835af18f2 100644 --- a/src/utils/daemon.rs +++ b/src/utils/daemon.rs @@ -11,19 +11,10 @@ use crate::{ use chrono::{Timelike, Utc}; use failure::Error; use log::{debug, error, info}; -use std::process::Command; use std::sync::Arc; use std::thread; use std::time::{Duration, Instant}; -fn run_git_gc() { - let gc = Command::new("git").args(&["gc", "--auto"]).output(); - - if let Err(err) = gc { - log::error!("failed to run `git gc`: {:?}", err); - } -} - fn start_registry_watcher( opts: DocBuilderOptions, pool: Pool, @@ -35,7 +26,6 @@ fn start_registry_watcher( .spawn(move || { // space this out to prevent it from clashing against the queue-builder thread on launch thread::sleep(Duration::from_secs(30)); - run_git_gc(); let mut last_gc = Instant::now(); loop { @@ -53,7 +43,7 @@ fn start_registry_watcher( } if last_gc.elapsed().as_secs() >= config.registry_gc_interval { - run_git_gc(); + doc_builder.run_git_gc(); last_gc = Instant::now(); } thread::sleep(Duration::from_secs(60)); From f91a2fec8e7fc7b55c491395df8066ed5327b4cc Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Thu, 13 Aug 2020 19:48:32 +0300 Subject: [PATCH 017/139] Fix typo --- src/index/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/index/mod.rs b/src/index/mod.rs index a4fef6e0e..e00c2c357 100644 --- a/src/index/mod.rs +++ b/src/index/mod.rs @@ -52,7 +52,7 @@ impl Index { } pub fn run_git_gc(&self) { - let cmd = format!("cd {} && gc --auto", self.path.to_str().unwrap()); + let cmd = format!("cd {} && git gc --auto", self.path.to_str().unwrap()); let gc = Command::new("sh").args(&["-c", cmd.as_str()]).output(); if let Err(err) = gc { log::error!("Failed to run `{}`: {:?}", cmd, err); From b5ec00ceb4f121058649e7015f7354222f275333 Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Thu, 13 Aug 2020 21:42:57 +0300 Subject: [PATCH 018/139] Rebased --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index d7823cbaf..5b7d9caab 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ target .vagrant .rustwide .rustwide-docker +.idea/ \ No newline at end of file From b1d4a38600012201d2a27444d6a1dae6d2cadcf1 Mon Sep 17 00:00:00 2001 From: ohaddahan Date: Tue, 11 Aug 2020 23:22:43 +0300 Subject: [PATCH 019/139] Update .gitignore Co-authored-by: Chase Wilson --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index 5b7d9caab..d7823cbaf 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,3 @@ target .vagrant .rustwide .rustwide-docker -.idea/ \ No newline at end of file From e6f17341b9db3de5c9e49e78877f336346496f43 Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Thu, 13 Aug 2020 21:48:01 +0300 Subject: [PATCH 020/139] Rebased --- git_hooks/pre-commit | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100755 git_hooks/pre-commit diff --git a/git_hooks/pre-commit b/git_hooks/pre-commit new file mode 100755 index 000000000..8ae45b3f3 --- /dev/null +++ b/git_hooks/pre-commit @@ -0,0 +1,19 @@ +#!/usr/bin/env sh +cargo fmt -- --check +if [ "$?" != "0" ] ; then + echo "\n" + echo "\033[0;31mpre-commit hook failed during:\033[0m" + echo "\033[0;31m\tcargo fmt -- --check\033[0m" + exit 1 +fi +cargo clippy --locked -- -D warnings +if [ "$?" != "0" ] ; then + echo "\n" + echo "\033[0;31mpre-commit hook failed during:\033[0m" + echo "\033[0;31m\tclippy --locked -- -D warning\033[0m" + exit 1 +fi + +echo "\n" +echo "\033[0;32mpre-commit hook succeeded\033[0m" +exit 0 \ No newline at end of file From 7e74a689f208b59003dd3e7e18df2a0e97558285 Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Wed, 12 Aug 2020 19:59:52 +0300 Subject: [PATCH 021/139] Changed envar prefix and replace echo with printf for pre-commit hook --- git_hooks/pre-commit | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/git_hooks/pre-commit b/git_hooks/pre-commit index 8ae45b3f3..f6855e88e 100755 --- a/git_hooks/pre-commit +++ b/git_hooks/pre-commit @@ -1,19 +1,19 @@ #!/usr/bin/env sh cargo fmt -- --check if [ "$?" != "0" ] ; then - echo "\n" - echo "\033[0;31mpre-commit hook failed during:\033[0m" - echo "\033[0;31m\tcargo fmt -- --check\033[0m" + printf "\n" + printf "\033[0;31mpre-commit hook failed during:\033[0m\n" + printf "\033[0;31m\tcargo fmt -- --check\033[0m\n" exit 1 fi cargo clippy --locked -- -D warnings if [ "$?" != "0" ] ; then - echo "\n" - echo "\033[0;31mpre-commit hook failed during:\033[0m" - echo "\033[0;31m\tclippy --locked -- -D warning\033[0m" + printf "\n" + printf "\033[0;31mpre-commit hook failed during:\033[0m\n" + printf "\033[0;31m\tclippy --locked -- -D warning\033[0m\n" exit 1 fi -echo "\n" -echo "\033[0;32mpre-commit hook succeeded\033[0m" +printf "\n" +printf "\033[0;32mpre-commit hook succeeded\033[0m\n" exit 0 \ No newline at end of file From 1fa9a7f90170c80ba92c813e189eec4740549e29 Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Thu, 13 Aug 2020 21:48:48 +0300 Subject: [PATCH 022/139] Rebased --- .git_hooks/pre-commit | 1 - git_hooks/pre-commit | 19 ------------------- 2 files changed, 20 deletions(-) delete mode 100755 git_hooks/pre-commit diff --git a/.git_hooks/pre-commit b/.git_hooks/pre-commit index 105684ef7..872c2cdda 100755 --- a/.git_hooks/pre-commit +++ b/.git_hooks/pre-commit @@ -16,4 +16,3 @@ fi printf "\n" printf "\033[0;32mpre-commit hook succeeded\033[0m\n" exit 0 - diff --git a/git_hooks/pre-commit b/git_hooks/pre-commit deleted file mode 100755 index f6855e88e..000000000 --- a/git_hooks/pre-commit +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env sh -cargo fmt -- --check -if [ "$?" != "0" ] ; then - printf "\n" - printf "\033[0;31mpre-commit hook failed during:\033[0m\n" - printf "\033[0;31m\tcargo fmt -- --check\033[0m\n" - exit 1 -fi -cargo clippy --locked -- -D warnings -if [ "$?" != "0" ] ; then - printf "\n" - printf "\033[0;31mpre-commit hook failed during:\033[0m\n" - printf "\033[0;31m\tclippy --locked -- -D warning\033[0m\n" - exit 1 -fi - -printf "\n" -printf "\033[0;32mpre-commit hook succeeded\033[0m\n" -exit 0 \ No newline at end of file From 98a0a1bd63ca84fd18be878272f871b0b6039bc8 Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Thu, 13 Aug 2020 15:58:30 +0300 Subject: [PATCH 023/139] Added newline at end of pre-commit script --- .git_hooks/pre-commit | 1 + 1 file changed, 1 insertion(+) diff --git a/.git_hooks/pre-commit b/.git_hooks/pre-commit index 872c2cdda..105684ef7 100755 --- a/.git_hooks/pre-commit +++ b/.git_hooks/pre-commit @@ -16,3 +16,4 @@ fi printf "\n" printf "\033[0;32mpre-commit hook succeeded\033[0m\n" exit 0 + From f840c5fa21b5352b24a3e9c93b2d7dbbda9f1ac7 Mon Sep 17 00:00:00 2001 From: Nemo157 Date: Sat, 8 Aug 2020 22:32:08 +0200 Subject: [PATCH 024/139] Update `cargo run` commands Consistently use `--` to separate arguments to `cratesfyi`, remove `web` subcommand that is only relevant for `docker-compose` commands. --- README.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 61bf3439b..c84156f16 100644 --- a/README.md +++ b/README.md @@ -139,7 +139,7 @@ See `cargo run -- --help` for a full list of commands. ```sh # This command will start web interface of docs.rs on http://localhost:3000 -cargo run start-web-server +cargo run -- start-web-server ``` #### `build` subcommand @@ -148,16 +148,16 @@ cargo run start-web-server # Builds and adds it into database # This is the main command to build and add a documentation into docs.rs. # For example, `docker-compose run web build crate regex 1.1.6` -cargo run web build crate +cargo run -- build crate # Builds every crate on crates.io and adds them into database # (beware: this may take months to finish) -cargo run web build world +cargo run -- build world # Builds a local package you have at and adds it to the database. # The package does not have to be on crates.io. # The package must be on the local filesystem, git urls are not allowed. -cargo build crate --local /path/to/source +cargo run -- build crate --local /path/to/source ``` #### `database` subcommand @@ -185,22 +185,22 @@ The database contains a blacklist of crates that should not be built. ```sh # List the crates on the blacklist -cargo run web database blacklist list +cargo run -- database blacklist list # Adds to the blacklist -cargo run web database blacklist add +cargo run -- database blacklist add # Removes from the blacklist -cargo run web database blacklist remove +cargo run -- database blacklist remove ``` #### `daemon` subcommand ```sh # Run a persistent daemon which queues builds and starts a web server. -cargo run daemon --registry-watcher disabled +cargo run -- daemon --registry-watcher=disabled # Add crates to the queue -cargo run queue add +cargo run -- queue add ``` ### Contact From 14f8730e2e5058d04e2ea074c2f1964e6fed428b Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Wed, 5 Aug 2020 13:23:13 +0200 Subject: [PATCH 025/139] Add doc coverage information --- src/db/add_package.rs | 20 +++++ src/db/delete.rs | 1 + src/db/migrate.rs | 17 ++++ src/db/mod.rs | 4 +- src/docbuilder/rustwide_builder.rs | 124 ++++++++++++++++++++++++++++- src/test/fakes.rs | 2 + src/web/crate_details.rs | 12 ++- templates/crate/details.html | 4 + 8 files changed, 181 insertions(+), 3 deletions(-) diff --git a/src/db/add_package.rs b/src/db/add_package.rs index 2bc4062fb..42b2787c1 100644 --- a/src/db/add_package.rs +++ b/src/db/add_package.rs @@ -130,6 +130,26 @@ pub(crate) fn add_package_into_database( Ok(release_id) } +pub(crate) fn add_doc_coverage( + conn: &mut Client, + release_id: i32, + total_items: i32, + documented_items: i32, +) -> Result { + debug!("Adding doc coverage into database"); + let rows = conn.query( + "INSERT INTO doc_coverage (release_id, total_items, documented_items) + VALUES ($1, $2, $3) + ON CONFLICT (release_id) DO UPDATE + SET + total_items = $2, + documented_items = $3 + RETURNING release_id", + &[&release_id, &total_items, &documented_items], + )?; + Ok(rows[0].get(0)) +} + /// Adds a build into database pub(crate) fn add_build_into_database( conn: &mut Client, diff --git a/src/db/delete.rs b/src/db/delete.rs index 0f354d3d3..be73ecfd4 100644 --- a/src/db/delete.rs +++ b/src/db/delete.rs @@ -54,6 +54,7 @@ const METADATA: &[(&str, &str)] = &[ ("keyword_rels", "rid"), ("builds", "rid"), ("compression_rels", "release"), + ("doc_coverage", "release_id"), ]; fn delete_version_from_database(conn: &mut Client, name: &str, version: &str) -> Result<(), Error> { diff --git a/src/db/migrate.rs b/src/db/migrate.rs index ad4361439..d658008b2 100644 --- a/src/db/migrate.rs +++ b/src/db/migrate.rs @@ -381,6 +381,23 @@ pub fn migrate(version: Option, conn: &mut Client) -> CratesfyiResult<( -- Nope, this is a pure database fix, no going back. " ), + migration!( + context, + // version + 16, + // description + "Create new table for doc coverage", + // upgrade query + " + CREATE TABLE doc_coverage ( + release_id INT UNIQUE REFERENCES releases(id), + total_items INT, + documented_items INT + ); + ", + // downgrade query + "DROP TABLE doc_coverage;" + ), ]; for migration in migrations { diff --git a/src/db/mod.rs b/src/db/mod.rs index e1f8db097..c506ada11 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -1,7 +1,9 @@ //! Database operations pub use self::add_package::update_crate_data_in_database; -pub(crate) use self::add_package::{add_build_into_database, add_package_into_database}; +pub(crate) use self::add_package::{ + add_build_into_database, add_doc_coverage, add_package_into_database, +}; pub use self::delete::{delete_crate, delete_version}; pub use self::file::add_path_into_database; pub use self::migrate::migrate; diff --git a/src/docbuilder/rustwide_builder.rs b/src/docbuilder/rustwide_builder.rs index c975238c9..0780044bc 100644 --- a/src/docbuilder/rustwide_builder.rs +++ b/src/docbuilder/rustwide_builder.rs @@ -3,7 +3,8 @@ use super::Metadata; use crate::db::blacklist::is_blacklisted; use crate::db::file::add_path_into_database; use crate::db::{ - add_build_into_database, add_package_into_database, update_crate_data_in_database, Pool, + add_build_into_database, add_doc_coverage, add_package_into_database, + update_crate_data_in_database, Pool, }; use crate::docbuilder::{crates::crates_from_path, Limits}; use crate::error::Result; @@ -427,6 +428,12 @@ impl RustwideBuilder { algs, )?; + if let (Some(total), Some(documented)) = + (res.result.total_items, res.result.documented_items) + { + add_doc_coverage(&mut conn, release_id, total, documented)?; + } + add_build_into_database(&mut conn, release_id, &res.result)?; // Some crates.io crate data is mutable, so we proactively update it during a release @@ -468,6 +475,106 @@ impl RustwideBuilder { Ok(()) } + fn get_coverage( + &self, + target: &str, + build: &Build, + metadata: &Metadata, + limits: &Limits, + ) -> Option<(i32, i32)> { + let rustdoc_flags: Vec = vec![ + "-Z".to_string(), + "unstable-options".to_string(), + "--static-root-path".to_string(), + "/".to_string(), + "--cap-lints".to_string(), + "warn".to_string(), + "--output-format".to_string(), + "json".to_string(), + "--show-coverage".to_string(), + ]; + + let mut cargo_args = vec!["doc", "--lib", "--no-deps"]; + if target != HOST_TARGET { + // If the explicit target is not a tier one target, we need to install it. + if !TARGETS.contains(&target) { + // This is a no-op if the target is already installed. + self.toolchain.add_target(&self.workspace, target).ok()?; + } + cargo_args.push("--target"); + cargo_args.push(target); + }; + + let tmp_jobs; + if let Some(cpu_limit) = self.cpu_limit { + tmp_jobs = format!("-j{}", cpu_limit); + cargo_args.push(&tmp_jobs); + } + + let tmp; + if let Some(features) = &metadata.features { + cargo_args.push("--features"); + tmp = features.join(" "); + cargo_args.push(&tmp); + } + if metadata.all_features { + cargo_args.push("--all-features"); + } + if metadata.no_default_features { + cargo_args.push("--no-default-features"); + } + + let mut storage = LogStorage::new(LevelFilter::Info); + storage.set_max_size(limits.max_log_size()); + + let mut json = String::new(); + if build + .cargo() + .timeout(Some(limits.timeout())) + .no_output_timeout(None) + .env( + "RUSTFLAGS", + metadata + .rustc_args + .as_ref() + .map(|args| args.join(" ")) + .unwrap_or_default(), + ) + .env("RUSTDOCFLAGS", rustdoc_flags.join(" ")) + // For docs.rs detection from build script: + // https://github.com/rust-lang/docs.rs/issues/147 + .env("DOCS_RS", "1") + .args(&cargo_args) + .log_output(false) + .process_lines(&mut |line, _| { + if line.starts_with('{') && line.ends_with('}') { + json = line.to_owned(); + } + }) + .run() + .is_ok() + { + match serde_json::from_str(&json).expect("conversion failed...") { + Value::Object(m) => { + let mut total = 0; + let mut documented = 0; + for entry in m.values() { + if let Some(Value::Number(n)) = entry.get("total") { + total += n.as_i64().unwrap_or(0) as i32; + } + if let Some(Value::Number(n)) = entry.get("with_docs") { + documented += n.as_i64().unwrap_or(0) as i32; + } + } + Some((total, documented)) + } + _ => None, + } + } else { + None + } + } + fn execute_build( &self, target: &str, @@ -555,6 +662,14 @@ impl RustwideBuilder { .run() .is_ok() }); + let mut total_items = None; + let mut documented_items = None; + if successful { + if let Some((total, documented)) = self.get_coverage(target, build, metadata, limits) { + total_items = Some(total); + documented_items = Some(documented); + } + } // If we're passed a default_target which requires a cross-compile, // cargo will put the output in `target//doc`. // However, if this is the default build, we don't want it there, @@ -574,6 +689,8 @@ impl RustwideBuilder { rustc_version: self.rustc_version.clone(), docsrs_version: format!("docsrs {}", crate::BUILD_VERSION), successful, + total_items, + documented_items, }, cargo_metadata, target: target.to_string(), @@ -630,4 +747,9 @@ pub(crate) struct BuildResult { pub(crate) docsrs_version: String, pub(crate) build_log: String, pub(crate) successful: bool, + /// The total items that could be documented in the current crate, used to calculate + /// documentation coverage. + pub(crate) total_items: Option, + /// The items of the crate that are documented, used to calculate documentation coverage. + pub(crate) documented_items: Option, } diff --git a/src/test/fakes.rs b/src/test/fakes.rs index 5cc10ca91..61cb8664e 100644 --- a/src/test/fakes.rs +++ b/src/test/fakes.rs @@ -59,6 +59,8 @@ impl<'a> FakeRelease<'a> { docsrs_version: "docs.rs 1.0.0 (000000000 1970-01-01)".into(), build_log: "It works!".into(), successful: true, + total_items: None, + documented_items: None, }, source_files: Vec::new(), rustdoc_files: Vec::new(), diff --git a/src/web/crate_details.rs b/src/web/crate_details.rs index 032ae5614..fb02b173a 100644 --- a/src/web/crate_details.rs +++ b/src/web/crate_details.rs @@ -44,6 +44,8 @@ pub struct CrateDetails { pub(crate) doc_targets: Vec, license: Option, documentation_url: Option, + total_items: Option, + documented_items: Option, } fn optional_markdown(markdown: &Option, serializer: S) -> Result @@ -97,9 +99,12 @@ impl CrateDetails { releases.doc_targets, releases.license, releases.documentation_url, - releases.default_target + releases.default_target, + doc_coverage.total_items, + doc_coverage.documented_items FROM releases INNER JOIN crates ON releases.crate_id = crates.id + LEFT JOIN doc_coverage ON doc_coverage.release_id = releases.id WHERE crates.name = $1 AND releases.version = $2;"; let rows = conn.query(query, &[&name, &version]).unwrap(); @@ -150,6 +155,9 @@ impl CrateDetails { .unwrap_or_else(Vec::new) }; + let documented_items: Option = krate.get("documented_items"); + let total_items: Option = krate.get("total_items"); + let mut crate_details = CrateDetails { name: krate.get("name"), version: krate.get("version"), @@ -180,6 +188,8 @@ impl CrateDetails { doc_targets, license: krate.get("license"), documentation_url: krate.get("documentation_url"), + documented_items: documented_items.map(|v| v as f32), + total_items: total_items.map(|v| v as f32), }; if let Some(repository_url) = crate_details.repository_url.clone() { diff --git a/templates/crate/details.html b/templates/crate/details.html index 61d581a72..a299d2dad 100644 --- a/templates/crate/details.html +++ b/templates/crate/details.html @@ -16,6 +16,10 @@
    + {%- if details.doc_coverage -%} +
  • Documentation
    coverage
    • +
  • {{ details.doc_coverage | round(precision=2) }} %
    • + {%- endif -%} {# List the release author's names and a link to their docs.rs profile #}
  • Authors
    • {%- for author in details.authors -%} From 87d6fe408d25e59d92724977db783eb7e4999a8d Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Wed, 5 Aug 2020 16:37:57 +0200 Subject: [PATCH 026/139] Clean code a bit --- src/docbuilder/rustwide_builder.rs | 321 ++++++++++++++--------------- 1 file changed, 155 insertions(+), 166 deletions(-) diff --git a/src/docbuilder/rustwide_builder.rs b/src/docbuilder/rustwide_builder.rs index 0780044bc..7c12f0a39 100644 --- a/src/docbuilder/rustwide_builder.rs +++ b/src/docbuilder/rustwide_builder.rs @@ -475,31 +475,19 @@ impl RustwideBuilder { Ok(()) } - fn get_coverage( + fn make_build_object, Vec, LogStorage) -> Result>( &self, target: &str, - build: &Build, metadata: &Metadata, limits: &Limits, - ) -> Option<(i32, i32)> { - let rustdoc_flags: Vec = vec![ - "-Z".to_string(), - "unstable-options".to_string(), - "--static-root-path".to_string(), - "/".to_string(), - "--cap-lints".to_string(), - "warn".to_string(), - "--output-format".to_string(), - "json".to_string(), - "--show-coverage".to_string(), - ]; - + f: F, + ) -> Result { let mut cargo_args = vec!["doc", "--lib", "--no-deps"]; if target != HOST_TARGET { // If the explicit target is not a tier one target, we need to install it. if !TARGETS.contains(&target) { // This is a no-op if the target is already installed. - self.toolchain.add_target(&self.workspace, target).ok()?; + self.toolchain.add_target(&self.workspace, target)?; } cargo_args.push("--target"); cargo_args.push(target); @@ -527,52 +515,78 @@ impl RustwideBuilder { let mut storage = LogStorage::new(LevelFilter::Info); storage.set_max_size(limits.max_log_size()); - let mut json = String::new(); - if build - .cargo() - .timeout(Some(limits.timeout())) - .no_output_timeout(None) - .env( - "RUSTFLAGS", - metadata - .rustc_args - .as_ref() - .map(|args| args.join(" ")) - .unwrap_or_default(), - ) - .env("RUSTDOCFLAGS", rustdoc_flags.join(" ")) - // For docs.rs detection from build script: - // https://github.com/rust-lang/docs.rs/issues/147 - .env("DOCS_RS", "1") - .args(&cargo_args) - .log_output(false) - .process_lines(&mut |line, _| { - if line.starts_with('{') && line.ends_with('}') { - json = line.to_owned(); - } - }) - .run() - .is_ok() - { - match serde_json::from_str(&json).expect("conversion failed...") { - Value::Object(m) => { - let mut total = 0; - let mut documented = 0; - for entry in m.values() { - if let Some(Value::Number(n)) = entry.get("total") { - total += n.as_i64().unwrap_or(0) as i32; + let rustdoc_flags: Vec = vec![ + "-Z".to_string(), + "unstable-options".to_string(), + "--static-root-path".to_string(), + "/".to_string(), + "--cap-lints".to_string(), + "warn".to_string(), + ]; + + f(cargo_args, rustdoc_flags, storage) + } + + fn get_coverage( + &self, + target: &str, + build: &Build, + metadata: &Metadata, + limits: &Limits, + ) -> Result> { + self.make_build_object( + target, + metadata, + limits, + |cargo_args, mut rustdoc_flags, _| { + rustdoc_flags.extend(vec![ + "--output-format".to_string(), + "json".to_string(), + "--show-coverage".to_string(), + ]); + let mut doc_coverage_json = None; + build + .cargo() + .timeout(Some(limits.timeout())) + .no_output_timeout(None) + .env( + "RUSTFLAGS", + metadata + .rustc_args + .as_ref() + .map(|args| args.join(" ")) + .unwrap_or_default(), + ) + .env("RUSTDOCFLAGS", rustdoc_flags.join(" ")) + // For docs.rs detection from build script: + // https://github.com/rust-lang/docs.rs/issues/147 + .env("DOCS_RS", "1") + .args(&cargo_args) + .log_output(false) + .process_lines(&mut |line, _| { + if line.starts_with('{') && line.ends_with('}') { + doc_coverage_json = Some(line.to_owned()); } - if let Some(Value::Number(n)) = entry.get("with_docs") { - documented += n.as_i64().unwrap_or(0) as i32; + }) + .run() + .ok(); + if let Some(json) = doc_coverage_json { + if let Ok(Value::Object(m)) = serde_json::from_str(&json) { + let (mut total, mut documented) = (0, 0); + for entry in m.values() { + if let Some(Value::Number(n)) = entry.get("total") { + total += n.as_i64().unwrap_or(0) as i32; + } + if let Some(Value::Number(n)) = entry.get("with_docs") { + documented += n.as_i64().unwrap_or(0) as i32; + } } + return Ok(Some((total, documented))); } - Some((total, documented)) } - _ => None, - } - } else { - None - } + Ok(None) + }, + ) } fn execute_build( @@ -583,118 +597,93 @@ impl RustwideBuilder { limits: &Limits, metadata: &Metadata, ) -> Result { - let cargo_metadata = - CargoMetadata::load(&self.workspace, &self.toolchain, &build.host_source_dir())?; - - let mut rustdoc_flags: Vec = vec![ - "-Z".to_string(), - "unstable-options".to_string(), - "--resource-suffix".to_string(), - format!("-{}", parse_rustc_version(&self.rustc_version)?), - "--static-root-path".to_string(), - "/".to_string(), - "--cap-lints".to_string(), - "warn".to_string(), - ]; - for dep in &cargo_metadata.root_dependencies() { - rustdoc_flags.push("--extern-html-root-url".to_string()); - rustdoc_flags.push(format!( - "{}=https://docs.rs/{}/{}", - dep.name.replace("-", "_"), - dep.name, - dep.version - )); - } - if let Some(package_rustdoc_args) = &metadata.rustdoc_args { - rustdoc_flags.append(&mut package_rustdoc_args.iter().map(|s| s.to_owned()).collect()); - } - let mut cargo_args = vec!["doc", "--lib", "--no-deps"]; - if target != HOST_TARGET { - // If the explicit target is not a tier one target, we need to install it. - if !TARGETS.contains(&target) { - // This is a no-op if the target is already installed. - self.toolchain.add_target(&self.workspace, target)?; - } - cargo_args.push("--target"); - cargo_args.push(target); - }; - - let tmp_jobs; - if let Some(cpu_limit) = self.cpu_limit { - tmp_jobs = format!("-j{}", cpu_limit); - cargo_args.push(&tmp_jobs); - } - - let tmp; - if let Some(features) = &metadata.features { - cargo_args.push("--features"); - tmp = features.join(" "); - cargo_args.push(&tmp); - } - if metadata.all_features { - cargo_args.push("--all-features"); - } - if metadata.no_default_features { - cargo_args.push("--no-default-features"); - } + self.make_build_object( + target, + metadata, + limits, + |cargo_args, mut rustdoc_flags, storage| { + rustdoc_flags.extend(vec![ + "--resource-suffix".to_string(), + format!("-{}", parse_rustc_version(&self.rustc_version)?), + ]); + let cargo_metadata = CargoMetadata::load( + &self.workspace, + &self.toolchain, + &build.host_source_dir(), + )?; - let mut storage = LogStorage::new(LevelFilter::Info); - storage.set_max_size(limits.max_log_size()); + for dep in &cargo_metadata.root_dependencies() { + rustdoc_flags.push("--extern-html-root-url".to_string()); + rustdoc_flags.push(format!( + "{}=https://docs.rs/{}/{}", + dep.name.replace("-", "_"), + dep.name, + dep.version + )); + } + if let Some(package_rustdoc_args) = &metadata.rustdoc_args { + rustdoc_flags + .append(&mut package_rustdoc_args.iter().map(|s| s.to_owned()).collect()); + } - let successful = logging::capture(&storage, || { - build - .cargo() - .timeout(Some(limits.timeout())) - .no_output_timeout(None) - .env( - "RUSTFLAGS", - metadata - .rustc_args - .as_ref() - .map(|args| args.join(" ")) - .unwrap_or_default(), - ) - .env("RUSTDOCFLAGS", rustdoc_flags.join(" ")) - // For docs.rs detection from build script: - // https://github.com/rust-lang/docs.rs/issues/147 - .env("DOCS_RS", "1") - .args(&cargo_args) - .run() - .is_ok() - }); - let mut total_items = None; - let mut documented_items = None; - if successful { - if let Some((total, documented)) = self.get_coverage(target, build, metadata, limits) { - total_items = Some(total); - documented_items = Some(documented); - } - } - // If we're passed a default_target which requires a cross-compile, - // cargo will put the output in `target//doc`. - // However, if this is the default build, we don't want it there, - // we want it in `target/doc`. - if target != HOST_TARGET && is_default_target { - // mv target/$target/doc target/doc - let target_dir = build.host_target_dir(); - let old_dir = target_dir.join(target).join("doc"); - let new_dir = target_dir.join("doc"); - debug!("rename {} to {}", old_dir.display(), new_dir.display()); - std::fs::rename(old_dir, new_dir)?; - } + let successful = logging::capture(&storage, || { + build + .cargo() + .timeout(Some(limits.timeout())) + .no_output_timeout(None) + .env( + "RUSTFLAGS", + metadata + .rustc_args + .as_ref() + .map(|args| args.join(" ")) + .unwrap_or_default(), + ) + .env("RUSTDOCFLAGS", rustdoc_flags.join(" ")) + // For docs.rs detection from build script: + // https://github.com/rust-lang/docs.rs/issues/147 + .env("DOCS_RS", "1") + .args(&cargo_args) + .run() + .is_ok() + }); + let mut total_items = None; + let mut documented_items = None; + if successful { + if let Some((total, documented)) = + self.get_coverage(target, build, metadata, limits)? + { + total_items = Some(total); + documented_items = Some(documented); + } + } + // If we're passed a default_target which requires a cross-compile, + // cargo will put the output in `target//doc`. + // However, if this is the default build, we don't want it there, + // we want it in `target/doc`. + if target != HOST_TARGET && is_default_target { + // mv target/$target/doc target/doc + let target_dir = build.host_target_dir(); + let old_dir = target_dir.join(target).join("doc"); + let new_dir = target_dir.join("doc"); + debug!("rename {} to {}", old_dir.display(), new_dir.display()); + std::fs::rename(old_dir, new_dir)?; + } - Ok(FullBuildResult { - result: BuildResult { - build_log: storage.to_string(), - rustc_version: self.rustc_version.clone(), - docsrs_version: format!("docsrs {}", crate::BUILD_VERSION), - successful, - total_items, - documented_items, + Ok(FullBuildResult { + result: BuildResult { + build_log: storage.to_string(), + rustc_version: self.rustc_version.clone(), + docsrs_version: format!("docsrs {}", crate::BUILD_VERSION), + successful, + total_items, + documented_items, + }, + cargo_metadata, + target: target.to_string(), + }) }, - cargo_metadata, - target: target.to_string(), - }) + ) } fn copy_docs( From 5c45d4bdd1bebf5660ec8bacccf2d56cef40b49d Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Thu, 6 Aug 2020 14:32:57 +0200 Subject: [PATCH 027/139] Improve rendering --- templates/crate/details.html | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/templates/crate/details.html b/templates/crate/details.html index a299d2dad..508c5b19b 100644 --- a/templates/crate/details.html +++ b/templates/crate/details.html @@ -16,9 +16,12 @@
      - {%- if details.doc_coverage -%} + {%- if details.documented_items and details.total_items -%} + {% set percent = details.documented_items * 100 / details.total_items %}
    • Documentation
      coverage
      • -
    • {{ details.doc_coverage | round(precision=2) }} %
      • +
    • {{ percent | round(precision=2) }} %
      + {{ details.documented_items }} out of {{ details.total_items }} items documented +
      • {%- endif -%} {# List the release author's names and a link to their docs.rs profile #}
    • Authors
      • From 7fe62252cdf1f2529de2debc0b2e5ba04e8d7a90 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Thu, 6 Aug 2020 20:51:04 +0200 Subject: [PATCH 028/139] Put doc coverage data into a new DocCoverage struct --- src/db/add_package.rs | 7 ++-- src/docbuilder/mod.rs | 2 +- src/docbuilder/rustwide_builder.rs | 52 +++++++++++++++--------------- src/test/fakes.rs | 3 +- 4 files changed, 31 insertions(+), 33 deletions(-) diff --git a/src/db/add_package.rs b/src/db/add_package.rs index 42b2787c1..63ddaaed4 100644 --- a/src/db/add_package.rs +++ b/src/db/add_package.rs @@ -5,7 +5,7 @@ use std::{ }; use crate::{ - docbuilder::BuildResult, + docbuilder::{BuildResult, DocCoverage}, error::Result, index::api::{CrateData, CrateOwner, ReleaseData}, storage::CompressionAlgorithm, @@ -133,8 +133,7 @@ pub(crate) fn add_package_into_database( pub(crate) fn add_doc_coverage( conn: &mut Client, release_id: i32, - total_items: i32, - documented_items: i32, + doc_coverage: DocCoverage, ) -> Result { debug!("Adding doc coverage into database"); let rows = conn.query( @@ -145,7 +144,7 @@ pub(crate) fn add_doc_coverage( total_items = $2, documented_items = $3 RETURNING release_id", - &[&release_id, &total_items, &documented_items], + &[&release_id, &doc_coverage.total_items, &doc_coverage.documented_items], )?; Ok(rows[0].get(0)) } diff --git a/src/docbuilder/mod.rs b/src/docbuilder/mod.rs index ede7d2ac8..571854ff8 100644 --- a/src/docbuilder/mod.rs +++ b/src/docbuilder/mod.rs @@ -7,7 +7,7 @@ mod rustwide_builder; pub(crate) use self::limits::Limits; pub(self) use self::metadata::Metadata; -pub(crate) use self::rustwide_builder::BuildResult; +pub(crate) use self::rustwide_builder::{BuildResult, DocCoverage}; pub use self::rustwide_builder::RustwideBuilder; use crate::db::Pool; diff --git a/src/docbuilder/rustwide_builder.rs b/src/docbuilder/rustwide_builder.rs index 7c12f0a39..366727d0e 100644 --- a/src/docbuilder/rustwide_builder.rs +++ b/src/docbuilder/rustwide_builder.rs @@ -428,10 +428,8 @@ impl RustwideBuilder { algs, )?; - if let (Some(total), Some(documented)) = - (res.result.total_items, res.result.documented_items) - { - add_doc_coverage(&mut conn, release_id, total, documented)?; + if let Some(doc_coverage) = res.result.doc_coverage { + add_doc_coverage(&mut conn, release_id, doc_coverage)?; } add_build_into_database(&mut conn, release_id, &res.result)?; @@ -533,7 +531,7 @@ impl RustwideBuilder { build: &Build, metadata: &Metadata, limits: &Limits, - ) -> Result> { + ) -> Result> { self.make_build_object( target, metadata, @@ -572,16 +570,19 @@ impl RustwideBuilder { .ok(); if let Some(json) = doc_coverage_json { if let Ok(Value::Object(m)) = serde_json::from_str(&json) { - let (mut total, mut documented) = (0, 0); + let (mut total_items, mut documented_items) = (0, 0); for entry in m.values() { if let Some(Value::Number(n)) = entry.get("total") { - total += n.as_i64().unwrap_or(0) as i32; + total_items += n.as_i64().unwrap_or(0) as i32; } if let Some(Value::Number(n)) = entry.get("with_docs") { - documented += n.as_i64().unwrap_or(0) as i32; + documented_items += n.as_i64().unwrap_or(0) as i32; } } - return Ok(Some((total, documented))); + return Ok(Some(DocCoverage { + total_items, + documented_items, + })); } } Ok(None) @@ -647,16 +648,11 @@ impl RustwideBuilder { .run() .is_ok() }); - let mut total_items = None; - let mut documented_items = None; - if successful { - if let Some((total, documented)) = - self.get_coverage(target, build, metadata, limits)? - { - total_items = Some(total); - documented_items = Some(documented); - } - } + let doc_coverage = if successful { + self.get_coverage(target, build, metadata, limits)? + } else { + None + }; // If we're passed a default_target which requires a cross-compile, // cargo will put the output in `target//doc`. // However, if this is the default build, we don't want it there, @@ -676,8 +672,7 @@ impl RustwideBuilder { rustc_version: self.rustc_version.clone(), docsrs_version: format!("docsrs {}", crate::BUILD_VERSION), successful, - total_items, - documented_items, + doc_coverage }, cargo_metadata, target: target.to_string(), @@ -731,14 +726,19 @@ struct FullBuildResult { cargo_metadata: CargoMetadata, } +#[derive(Clone, Copy)] +pub(crate) struct DocCoverage { + /// The total items that could be documented in the current crate, used to calculate + /// documentation coverage. + pub(crate) total_items: i32, + /// The items of the crate that are documented, used to calculate documentation coverage. + pub(crate) documented_items: i32, +} + pub(crate) struct BuildResult { pub(crate) rustc_version: String, pub(crate) docsrs_version: String, pub(crate) build_log: String, pub(crate) successful: bool, - /// The total items that could be documented in the current crate, used to calculate - /// documentation coverage. - pub(crate) total_items: Option, - /// The items of the crate that are documented, used to calculate documentation coverage. - pub(crate) documented_items: Option, + pub(crate) doc_coverage: Option, } diff --git a/src/test/fakes.rs b/src/test/fakes.rs index 61cb8664e..3a3ad52a4 100644 --- a/src/test/fakes.rs +++ b/src/test/fakes.rs @@ -59,8 +59,7 @@ impl<'a> FakeRelease<'a> { docsrs_version: "docs.rs 1.0.0 (000000000 1970-01-01)".into(), build_log: "It works!".into(), successful: true, - total_items: None, - documented_items: None, + doc_coverage: None, }, source_files: Vec::new(), rustdoc_files: Vec::new(), From f914f1b2a6297388a838b319a050c748929d0346 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Thu, 6 Aug 2020 23:21:19 +0200 Subject: [PATCH 029/139] Move common code into one macro to make the maintenance easier --- src/db/add_package.rs | 6 +- src/docbuilder/mod.rs | 2 +- src/docbuilder/rustwide_builder.rs | 338 +++++++++++++---------------- 3 files changed, 161 insertions(+), 185 deletions(-) diff --git a/src/db/add_package.rs b/src/db/add_package.rs index 63ddaaed4..ed5e1bfb9 100644 --- a/src/db/add_package.rs +++ b/src/db/add_package.rs @@ -144,7 +144,11 @@ pub(crate) fn add_doc_coverage( total_items = $2, documented_items = $3 RETURNING release_id", - &[&release_id, &doc_coverage.total_items, &doc_coverage.documented_items], + &[ + &release_id, + &doc_coverage.total_items, + &doc_coverage.documented_items, + ], )?; Ok(rows[0].get(0)) } diff --git a/src/docbuilder/mod.rs b/src/docbuilder/mod.rs index 571854ff8..fb1277be0 100644 --- a/src/docbuilder/mod.rs +++ b/src/docbuilder/mod.rs @@ -7,8 +7,8 @@ mod rustwide_builder; pub(crate) use self::limits::Limits; pub(self) use self::metadata::Metadata; -pub(crate) use self::rustwide_builder::{BuildResult, DocCoverage}; pub use self::rustwide_builder::RustwideBuilder; +pub(crate) use self::rustwide_builder::{BuildResult, DocCoverage}; use crate::db::Pool; use crate::error::Result; diff --git a/src/docbuilder/rustwide_builder.rs b/src/docbuilder/rustwide_builder.rs index 366727d0e..d66e9eced 100644 --- a/src/docbuilder/rustwide_builder.rs +++ b/src/docbuilder/rustwide_builder.rs @@ -69,6 +69,69 @@ const ESSENTIAL_FILES_UNVERSIONED: &[&str] = &[ const DUMMY_CRATE_NAME: &str = "empty-library"; const DUMMY_CRATE_VERSION: &str = "1.0.0"; +macro_rules! config_command { + ($obj:ident, $build:expr, $target:expr, $metadata:expr, $limits:expr, $rustdoc_flags_extras:expr, $($extra:tt)+) => {{ + let mut cargo_args = vec!["doc", "--lib", "--no-deps"]; + if $target != HOST_TARGET { + // If the explicit target is not a tier one target, we need to install it. + if !TARGETS.contains(&$target) { + // This is a no-op if the target is already installed. + $obj.toolchain.add_target(&$obj.workspace, $target)?; + } + cargo_args.push("--target"); + cargo_args.push($target); + }; + + let tmp; + if let Some(cpu_limit) = $obj.cpu_limit { + tmp = format!("-j{}", cpu_limit); + cargo_args.push(&tmp); + } + + let tmp; + if let Some(features) = &$metadata.features { + cargo_args.push("--features"); + tmp = features.join(" "); + cargo_args.push(&tmp); + } + if $metadata.all_features { + cargo_args.push("--all-features"); + } + if $metadata.no_default_features { + cargo_args.push("--no-default-features"); + } + + let mut rustdoc_flags = vec![ + "-Z".to_string(), + "unstable-options".to_string(), + "--static-root-path".to_string(), + "/".to_string(), + "--cap-lints".to_string(), + "warn".to_string(), + ]; + rustdoc_flags.extend($rustdoc_flags_extras); + + $build + .cargo() + .timeout(Some($limits.timeout())) + .no_output_timeout(None) + .env( + "RUSTFLAGS", + $metadata + .rustc_args + .as_ref() + .map(|args| args.join(" ")) + .unwrap_or_default(), + ) + .env("RUSTDOCFLAGS", rustdoc_flags.join(" ")) + // For docs.rs detection from build script: + // https://github.com/rust-lang/docs.rs/issues/147 + .env("DOCS_RS", "1") + .args(&cargo_args) + $($extra)+ + }} +} + pub struct RustwideBuilder { workspace: Workspace, toolchain: Toolchain, @@ -473,58 +536,6 @@ impl RustwideBuilder { Ok(()) } - fn make_build_object, Vec, LogStorage) -> Result>( - &self, - target: &str, - metadata: &Metadata, - limits: &Limits, - f: F, - ) -> Result { - let mut cargo_args = vec!["doc", "--lib", "--no-deps"]; - if target != HOST_TARGET { - // If the explicit target is not a tier one target, we need to install it. - if !TARGETS.contains(&target) { - // This is a no-op if the target is already installed. - self.toolchain.add_target(&self.workspace, target)?; - } - cargo_args.push("--target"); - cargo_args.push(target); - }; - - let tmp_jobs; - if let Some(cpu_limit) = self.cpu_limit { - tmp_jobs = format!("-j{}", cpu_limit); - cargo_args.push(&tmp_jobs); - } - - let tmp; - if let Some(features) = &metadata.features { - cargo_args.push("--features"); - tmp = features.join(" "); - cargo_args.push(&tmp); - } - if metadata.all_features { - cargo_args.push("--all-features"); - } - if metadata.no_default_features { - cargo_args.push("--no-default-features"); - } - - let mut storage = LogStorage::new(LevelFilter::Info); - storage.set_max_size(limits.max_log_size()); - - let rustdoc_flags: Vec = vec![ - "-Z".to_string(), - "unstable-options".to_string(), - "--static-root-path".to_string(), - "/".to_string(), - "--cap-lints".to_string(), - "warn".to_string(), - ]; - - f(cargo_args, rustdoc_flags, storage) - } - fn get_coverage( &self, target: &str, @@ -532,62 +543,41 @@ impl RustwideBuilder { metadata: &Metadata, limits: &Limits, ) -> Result> { - self.make_build_object( - target, - metadata, - limits, - |cargo_args, mut rustdoc_flags, _| { - rustdoc_flags.extend(vec![ - "--output-format".to_string(), - "json".to_string(), - "--show-coverage".to_string(), - ]); - let mut doc_coverage_json = None; - build - .cargo() - .timeout(Some(limits.timeout())) - .no_output_timeout(None) - .env( - "RUSTFLAGS", - metadata - .rustc_args - .as_ref() - .map(|args| args.join(" ")) - .unwrap_or_default(), - ) - .env("RUSTDOCFLAGS", rustdoc_flags.join(" ")) - // For docs.rs detection from build script: - // https://github.com/rust-lang/docs.rs/issues/147 - .env("DOCS_RS", "1") - .args(&cargo_args) - .log_output(false) - .process_lines(&mut |line, _| { - if line.starts_with('{') && line.ends_with('}') { - doc_coverage_json = Some(line.to_owned()); - } - }) - .run() - .ok(); - if let Some(json) = doc_coverage_json { - if let Ok(Value::Object(m)) = serde_json::from_str(&json) { - let (mut total_items, mut documented_items) = (0, 0); - for entry in m.values() { - if let Some(Value::Number(n)) = entry.get("total") { - total_items += n.as_i64().unwrap_or(0) as i32; - } - if let Some(Value::Number(n)) = entry.get("with_docs") { - documented_items += n.as_i64().unwrap_or(0) as i32; - } - } - return Ok(Some(DocCoverage { - total_items, - documented_items, - })); + let mut doc_coverage_json = None; + + let rustdoc_flags = vec![ + "--output-format".to_string(), + "json".to_string(), + "--show-coverage".to_string(), + ]; + config_command!(self, build, target, metadata, limits, rustdoc_flags, + .process_lines(&mut |line, _| { + if line.starts_with('{') && line.ends_with('}') { + doc_coverage_json = Some(line.to_owned()); + } + }) + .log_output(false) + .run()?; + ); + + if let Some(json) = doc_coverage_json { + if let Ok(Value::Object(m)) = serde_json::from_str(&json) { + let (mut total_items, mut documented_items) = (0, 0); + for entry in m.values() { + if let Some(Value::Number(n)) = entry.get("total") { + total_items += n.as_i64().unwrap_or(0) as i32; + } + if let Some(Value::Number(n)) = entry.get("with_docs") { + documented_items += n.as_i64().unwrap_or(0) as i32; } } - Ok(None) - }, - ) + return Ok(Some(DocCoverage { + total_items, + documented_items, + })); + } + } + Ok(None) } fn execute_build( @@ -598,87 +588,69 @@ impl RustwideBuilder { limits: &Limits, metadata: &Metadata, ) -> Result { - self.make_build_object( - target, - metadata, - limits, - |cargo_args, mut rustdoc_flags, storage| { - rustdoc_flags.extend(vec![ - "--resource-suffix".to_string(), - format!("-{}", parse_rustc_version(&self.rustc_version)?), - ]); - let cargo_metadata = CargoMetadata::load( - &self.workspace, - &self.toolchain, - &build.host_source_dir(), - )?; + let cargo_metadata = + CargoMetadata::load(&self.workspace, &self.toolchain, &build.host_source_dir())?; + + let mut rustdoc_flags = Vec::new(); + + for dep in &cargo_metadata.root_dependencies() { + rustdoc_flags.push("--extern-html-root-url".to_string()); + rustdoc_flags.push(format!( + "{}=https://docs.rs/{}/{}", + dep.name.replace("-", "_"), + dep.name, + dep.version + )); + } + if let Some(package_rustdoc_args) = &metadata.rustdoc_args { + rustdoc_flags.append(&mut package_rustdoc_args.iter().map(|s| s.to_owned()).collect()); + } - for dep in &cargo_metadata.root_dependencies() { - rustdoc_flags.push("--extern-html-root-url".to_string()); - rustdoc_flags.push(format!( - "{}=https://docs.rs/{}/{}", - dep.name.replace("-", "_"), - dep.name, - dep.version - )); - } - if let Some(package_rustdoc_args) = &metadata.rustdoc_args { - rustdoc_flags - .append(&mut package_rustdoc_args.iter().map(|s| s.to_owned()).collect()); - } + rustdoc_flags.extend(vec![ + "--resource-suffix".to_string(), + format!("-{}", parse_rustc_version(&self.rustc_version)?), + ]); - let successful = logging::capture(&storage, || { - build - .cargo() - .timeout(Some(limits.timeout())) - .no_output_timeout(None) - .env( - "RUSTFLAGS", - metadata - .rustc_args - .as_ref() - .map(|args| args.join(" ")) - .unwrap_or_default(), - ) - .env("RUSTDOCFLAGS", rustdoc_flags.join(" ")) - // For docs.rs detection from build script: - // https://github.com/rust-lang/docs.rs/issues/147 - .env("DOCS_RS", "1") - .args(&cargo_args) - .run() - .is_ok() - }); - let doc_coverage = if successful { - self.get_coverage(target, build, metadata, limits)? - } else { - None - }; - // If we're passed a default_target which requires a cross-compile, - // cargo will put the output in `target//doc`. - // However, if this is the default build, we don't want it there, - // we want it in `target/doc`. - if target != HOST_TARGET && is_default_target { - // mv target/$target/doc target/doc - let target_dir = build.host_target_dir(); - let old_dir = target_dir.join(target).join("doc"); - let new_dir = target_dir.join("doc"); - debug!("rename {} to {}", old_dir.display(), new_dir.display()); - std::fs::rename(old_dir, new_dir)?; - } + let mut storage = LogStorage::new(LevelFilter::Info); + storage.set_max_size(limits.max_log_size()); + + let successful = logging::capture(&storage, || { + let wrap = || { + config_command!(self, build, target, metadata, limits, rustdoc_flags, + .run() + ) + }; + wrap().is_ok() + }); + let doc_coverage = if successful { + self.get_coverage(target, build, metadata, limits)? + } else { + None + }; + // If we're passed a default_target which requires a cross-compile, + // cargo will put the output in `target//doc`. + // However, if this is the default build, we don't want it there, + // we want it in `target/doc`. + if target != HOST_TARGET && is_default_target { + // mv target/$target/doc target/doc + let target_dir = build.host_target_dir(); + let old_dir = target_dir.join(target).join("doc"); + let new_dir = target_dir.join("doc"); + debug!("rename {} to {}", old_dir.display(), new_dir.display()); + std::fs::rename(old_dir, new_dir)?; + } - Ok(FullBuildResult { - result: BuildResult { - build_log: storage.to_string(), - rustc_version: self.rustc_version.clone(), - docsrs_version: format!("docsrs {}", crate::BUILD_VERSION), - successful, - doc_coverage - }, - cargo_metadata, - target: target.to_string(), - }) + Ok(FullBuildResult { + result: BuildResult { + build_log: storage.to_string(), + rustc_version: self.rustc_version.clone(), + docsrs_version: format!("docsrs {}", crate::BUILD_VERSION), + successful, + doc_coverage, }, - ) + cargo_metadata, + target: target.to_string(), + }) } fn copy_docs( From b5ab71243821fa693155f809c554b287e60c3cdd Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Thu, 6 Aug 2020 23:28:04 +0200 Subject: [PATCH 030/139] Improve templates for doc coverage --- templates/crate/details.html | 2 +- templates/rustdoc/body.html | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/templates/crate/details.html b/templates/crate/details.html index 508c5b19b..d83b65ceb 100644 --- a/templates/crate/details.html +++ b/templates/crate/details.html @@ -18,7 +18,7 @@
        {%- if details.documented_items and details.total_items -%} {% set percent = details.documented_items * 100 / details.total_items %} -
      • Documentation
        coverage
        • +
      • Coverage
      • {{ percent | round(precision=2) }} %
        {{ details.documented_items }} out of {{ details.total_items }} items documented
        • diff --git a/templates/rustdoc/body.html b/templates/rustdoc/body.html index 7e95972d4..eb5b87b62 100644 --- a/templates/rustdoc/body.html +++ b/templates/rustdoc/body.html @@ -117,6 +117,20 @@
    + {%- if krate.documented_items and krate.total_items -%} + {% set percent = krate.documented_items * 100 / krate.total_items %} +
    +
    +
      +
    • Coverage
      • +
    • {{ percent | round(precision=2) }} %
      + {{ krate.documented_items }} out of {{ krate.total_items }} items documented +
      • +
    +
    +
    + {%- endif -%} +
      From 12c88286abbd261c7ce4e7d2be102dfaf80e0db1 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Fri, 7 Aug 2020 00:37:10 +0200 Subject: [PATCH 031/139] Add explanations about the "config_command" macro and append "rustdoc_args" from "metadata" inside the macro too --- src/docbuilder/rustwide_builder.rs | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/docbuilder/rustwide_builder.rs b/src/docbuilder/rustwide_builder.rs index d66e9eced..78b684c51 100644 --- a/src/docbuilder/rustwide_builder.rs +++ b/src/docbuilder/rustwide_builder.rs @@ -69,6 +69,16 @@ const ESSENTIAL_FILES_UNVERSIONED: &[&str] = &[ const DUMMY_CRATE_NAME: &str = "empty-library"; const DUMMY_CRATE_VERSION: &str = "1.0.0"; +// This macro exists because of lifetimes issues surrounding the `Command::process_lines` method: +// it expects a mutable reference to a closure, in which we capture a variable in order to store +// the JSON output. Unfortunately, we *need* to create the command in the same scope level otherwise +// rustc becomes very grumpy about the fact that the variable needs to be static because it is +// captured in a mutably referenced closure. +// +// So either you create a function with a callback in which you pass the `Command` as an argument, +// or you create a function returning a `Command`, it's very unhappy in both cases. +// +// TODO: make `Command::process_lines` take the closure by value rather than a mutable reference. macro_rules! config_command { ($obj:ident, $build:expr, $target:expr, $metadata:expr, $limits:expr, $rustdoc_flags_extras:expr, $($extra:tt)+) => {{ let mut cargo_args = vec!["doc", "--lib", "--no-deps"]; @@ -109,6 +119,11 @@ macro_rules! config_command { "--cap-lints".to_string(), "warn".to_string(), ]; + + if let Some(package_rustdoc_args) = &$metadata.rustdoc_args { + rustdoc_flags.append(&mut package_rustdoc_args.clone()); + } + rustdoc_flags.extend($rustdoc_flags_extras); $build @@ -602,9 +617,6 @@ impl RustwideBuilder { dep.version )); } - if let Some(package_rustdoc_args) = &metadata.rustdoc_args { - rustdoc_flags.append(&mut package_rustdoc_args.iter().map(|s| s.to_owned()).collect()); - } rustdoc_flags.extend(vec![ "--resource-suffix".to_string(), From 200e3400f24e6a4f4b8b1b209051d51c27f324ee Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Fri, 7 Aug 2020 16:59:14 +0200 Subject: [PATCH 032/139] Add test to ensure that doc coverage is present --- src/test/fakes.rs | 12 ++++++++++-- src/web/mod.rs | 30 ++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/src/test/fakes.rs b/src/test/fakes.rs index 3a3ad52a4..a3372a68d 100644 --- a/src/test/fakes.rs +++ b/src/test/fakes.rs @@ -1,5 +1,5 @@ use super::TestDatabase; -use crate::docbuilder::BuildResult; +use crate::docbuilder::{BuildResult, DocCoverage}; use crate::index::api::{CrateData, CrateOwner, ReleaseData}; use crate::storage::Storage; use crate::utils::{Dependency, MetadataPackage, Target}; @@ -59,7 +59,10 @@ impl<'a> FakeRelease<'a> { docsrs_version: "docs.rs 1.0.0 (000000000 1970-01-01)".into(), build_log: "It works!".into(), successful: true, - doc_coverage: None, + doc_coverage: Some(DocCoverage { + total_items: 10, + documented_items: 6, + }), }, source_files: Vec::new(), rustdoc_files: Vec::new(), @@ -274,6 +277,11 @@ impl<'a> FakeRelease<'a> { &self.registry_crate_data, )?; crate::db::add_build_into_database(&mut db.conn(), release_id, &self.build_result)?; + crate::db::add_doc_coverage( + &mut db.conn(), + release_id, + self.build_result.doc_coverage.clone().unwrap(), + )?; Ok(release_id) } diff --git a/src/web/mod.rs b/src/web/mod.rs index 2afa558ef..23dc8912b 100644 --- a/src/web/mod.rs +++ b/src/web/mod.rs @@ -668,6 +668,14 @@ mod test { node.select("#clipboard").unwrap().count() == 1 } + fn check_doc_coverage_is_present_for_path(path: &str, web: &TestFrontend) -> bool { + let data = web.get(path).send().unwrap().text().unwrap(); + let node = kuchiki::parse_html().one(data); + node.select(".pure-menu-heading") + .unwrap() + .any(|e| e.text_contents().contains("Coverage")) + } + #[test] fn test_index_returns_success() { wrapper(|env| { @@ -677,6 +685,28 @@ mod test { }); } + #[test] + fn test_doc_coverage_for_crate_pages() { + wrapper(|env| { + env.fake_release() + .name("fake_crate") + .version("0.0.1") + .source_file("test.rs", &[]) + .create() + .unwrap(); + let web = env.frontend(); + assert!(check_doc_coverage_is_present_for_path( + "/crate/fake_crate/0.0.1", + web + )); + assert!(check_doc_coverage_is_present_for_path( + "/fake_crate/0.0.1/fake_crate", + web + )); + Ok(()) + }); + } + #[test] fn test_show_clipboard_for_crate_pages() { wrapper(|env| { From 35584bb3ea9e016193db183e087217746bc686d6 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Sat, 8 Aug 2020 16:55:59 +0200 Subject: [PATCH 033/139] bump rustwide to 0.10.0 --- Cargo.lock | 9 +++++---- Cargo.toml | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0515be845..b1095bbb4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -368,7 +368,7 @@ dependencies = [ "rusoto_core 0.45.0 (registry+https://github.com/rust-lang/crates.io-index)", "rusoto_credential 0.45.0 (registry+https://github.com/rust-lang/crates.io-index)", "rusoto_s3 0.45.0 (registry+https://github.com/rust-lang/crates.io-index)", - "rustwide 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rustwide 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", "sass-rs 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "schemamama 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "schemamama_postgres 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2516,10 +2516,10 @@ dependencies = [ [[package]] name = "rustwide" -version = "0.7.1" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "base64 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", + "base64 0.12.1 (registry+https://github.com/rust-lang/crates.io-index)", "failure 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", "flate2 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)", "fs2 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2536,6 +2536,7 @@ dependencies = [ "serde_json 1.0.53 (registry+https://github.com/rust-lang/crates.io-index)", "tar 0.4.26 (registry+https://github.com/rust-lang/crates.io-index)", "tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "thiserror 1.0.20 (registry+https://github.com/rust-lang/crates.io-index)", "tokio 0.2.22 (registry+https://github.com/rust-lang/crates.io-index)", "toml 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)", "walkdir 2.3.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -3971,7 +3972,7 @@ dependencies = [ "checksum rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)" = "dcf128d1287d2ea9d80910b5f1120d0b8eede3fbf1abe91c40d39ea7d51e6fda" "checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" "checksum rusttype 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)" = "dc7c727aded0be18c5b80c1640eae0ac8e396abf6fa8477d96cb37d18ee5ec59" -"checksum rustwide 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "6e97a51dfbb3333ed70d6373d7980165d20dbd84a8173ab184c2c4d8f27bd122" +"checksum rustwide 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "417d578ebc7fa963bcd06f365f7987c091abeba70eac22dba94b7fd922a95c09" "checksum ryu 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "ed3d612bc64430efeb3f7ee6ef26d590dce0c43249217bddc62112540c7941e1" "checksum safemem 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ef703b7cb59335eae2eb93ceb664c0eb7ea6bf567079d843e09420219668e072" "checksum same-file 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" diff --git a/Cargo.toml b/Cargo.toml index d62891cfb..60fa94494 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,7 +31,7 @@ schemamama = "0.3" schemamama_postgres = "0.3" systemstat = "0.1.4" prometheus = { version = "0.7.0", default-features = false } -rustwide = "0.7.1" +rustwide = "0.10.0" mime_guess = "2" dotenv = "0.15" zstd = "0.5" From 2d23d4c3b8ca1bbf6003e5ff78ad9888822b7092 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Fri, 7 Aug 2020 18:18:50 +0200 Subject: [PATCH 034/139] remove the macro and improve the coverage extraction code --- src/docbuilder/rustwide_builder.rs | 216 ++++++++++++++--------------- 1 file changed, 107 insertions(+), 109 deletions(-) diff --git a/src/docbuilder/rustwide_builder.rs b/src/docbuilder/rustwide_builder.rs index 78b684c51..19fe02d6f 100644 --- a/src/docbuilder/rustwide_builder.rs +++ b/src/docbuilder/rustwide_builder.rs @@ -20,7 +20,7 @@ use rustwide::toolchain::ToolchainError; use rustwide::{Build, Crate, Toolchain, Workspace, WorkspaceBuilder}; use serde_json::Value; use std::borrow::Cow; -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::path::Path; use std::sync::Arc; @@ -69,84 +69,6 @@ const ESSENTIAL_FILES_UNVERSIONED: &[&str] = &[ const DUMMY_CRATE_NAME: &str = "empty-library"; const DUMMY_CRATE_VERSION: &str = "1.0.0"; -// This macro exists because of lifetimes issues surrounding the `Command::process_lines` method: -// it expects a mutable reference to a closure, in which we capture a variable in order to store -// the JSON output. Unfortunately, we *need* to create the command in the same scope level otherwise -// rustc becomes very grumpy about the fact that the variable needs to be static because it is -// captured in a mutably referenced closure. -// -// So either you create a function with a callback in which you pass the `Command` as an argument, -// or you create a function returning a `Command`, it's very unhappy in both cases. -// -// TODO: make `Command::process_lines` take the closure by value rather than a mutable reference. -macro_rules! config_command { - ($obj:ident, $build:expr, $target:expr, $metadata:expr, $limits:expr, $rustdoc_flags_extras:expr, $($extra:tt)+) => {{ - let mut cargo_args = vec!["doc", "--lib", "--no-deps"]; - if $target != HOST_TARGET { - // If the explicit target is not a tier one target, we need to install it. - if !TARGETS.contains(&$target) { - // This is a no-op if the target is already installed. - $obj.toolchain.add_target(&$obj.workspace, $target)?; - } - cargo_args.push("--target"); - cargo_args.push($target); - }; - - let tmp; - if let Some(cpu_limit) = $obj.cpu_limit { - tmp = format!("-j{}", cpu_limit); - cargo_args.push(&tmp); - } - - let tmp; - if let Some(features) = &$metadata.features { - cargo_args.push("--features"); - tmp = features.join(" "); - cargo_args.push(&tmp); - } - if $metadata.all_features { - cargo_args.push("--all-features"); - } - if $metadata.no_default_features { - cargo_args.push("--no-default-features"); - } - - let mut rustdoc_flags = vec![ - "-Z".to_string(), - "unstable-options".to_string(), - "--static-root-path".to_string(), - "/".to_string(), - "--cap-lints".to_string(), - "warn".to_string(), - ]; - - if let Some(package_rustdoc_args) = &$metadata.rustdoc_args { - rustdoc_flags.append(&mut package_rustdoc_args.clone()); - } - - rustdoc_flags.extend($rustdoc_flags_extras); - - $build - .cargo() - .timeout(Some($limits.timeout())) - .no_output_timeout(None) - .env( - "RUSTFLAGS", - $metadata - .rustc_args - .as_ref() - .map(|args| args.join(" ")) - .unwrap_or_default(), - ) - .env("RUSTDOCFLAGS", rustdoc_flags.join(" ")) - // For docs.rs detection from build script: - // https://github.com/rust-lang/docs.rs/issues/147 - .env("DOCS_RS", "1") - .args(&cargo_args) - $($extra)+ - }} -} - pub struct RustwideBuilder { workspace: Workspace, toolchain: Toolchain, @@ -558,41 +480,46 @@ impl RustwideBuilder { metadata: &Metadata, limits: &Limits, ) -> Result> { - let mut doc_coverage_json = None; - let rustdoc_flags = vec![ "--output-format".to_string(), "json".to_string(), "--show-coverage".to_string(), ]; - config_command!(self, build, target, metadata, limits, rustdoc_flags, + + #[derive(serde::Deserialize)] + struct FileCoverage { + total: i32, + with_docs: i32, + } + + let mut coverage = DocCoverage { + total_items: 0, + documented_items: 0, + }; + + self.prepare_command(build, target, metadata, limits, rustdoc_flags)? .process_lines(&mut |line, _| { if line.starts_with('{') && line.ends_with('}') { - doc_coverage_json = Some(line.to_owned()); + let parsed = match serde_json::from_str::>(line) { + Ok(parsed) => parsed, + Err(_) => return, + }; + for file in parsed.values() { + coverage.total_items += file.total; + coverage.documented_items += file.with_docs; + } } }) .log_output(false) .run()?; - ); - - if let Some(json) = doc_coverage_json { - if let Ok(Value::Object(m)) = serde_json::from_str(&json) { - let (mut total_items, mut documented_items) = (0, 0); - for entry in m.values() { - if let Some(Value::Number(n)) = entry.get("total") { - total_items += n.as_i64().unwrap_or(0) as i32; - } - if let Some(Value::Number(n)) = entry.get("with_docs") { - documented_items += n.as_i64().unwrap_or(0) as i32; - } - } - return Ok(Some(DocCoverage { - total_items, - documented_items, - })); - } - } - Ok(None) + + Ok( + if coverage.total_items == 0 && coverage.documented_items == 0 { + None + } else { + Some(coverage) + }, + ) } fn execute_build( @@ -627,12 +554,9 @@ impl RustwideBuilder { storage.set_max_size(limits.max_log_size()); let successful = logging::capture(&storage, || { - let wrap = || { - config_command!(self, build, target, metadata, limits, rustdoc_flags, - .run() - ) - }; - wrap().is_ok() + self.prepare_command(build, target, metadata, limits, rustdoc_flags) + .and_then(|command| command.run().map_err(failure::Error::from)) + .is_ok() }); let doc_coverage = if successful { self.get_coverage(target, build, metadata, limits)? @@ -665,6 +589,80 @@ impl RustwideBuilder { }) } + fn prepare_command<'ws, 'pl>( + &self, + build: &'ws Build, + target: &str, + metadata: &Metadata, + limits: &Limits, + rustdoc_flags_extras: Vec, + ) -> Result> { + let mut cargo_args = vec!["doc", "--lib", "--no-deps"]; + if target != HOST_TARGET { + // If the explicit target is not a tier one target, we need to install it. + if !TARGETS.contains(&target) { + // This is a no-op if the target is already installed. + self.toolchain.add_target(&self.workspace, target)?; + } + cargo_args.push("--target"); + cargo_args.push(target); + }; + + let tmp; + if let Some(cpu_limit) = self.cpu_limit { + tmp = format!("-j{}", cpu_limit); + cargo_args.push(&tmp); + } + + let tmp; + if let Some(features) = &metadata.features { + cargo_args.push("--features"); + tmp = features.join(" "); + cargo_args.push(&tmp); + } + if metadata.all_features { + cargo_args.push("--all-features"); + } + if metadata.no_default_features { + cargo_args.push("--no-default-features"); + } + + let mut rustdoc_flags = vec![ + "-Z".to_string(), + "unstable-options".to_string(), + "--static-root-path".to_string(), + "/".to_string(), + "--cap-lints".to_string(), + "warn".to_string(), + ]; + + if let Some(package_rustdoc_args) = &metadata.rustdoc_args { + rustdoc_flags.append(&mut package_rustdoc_args.clone()); + } + + rustdoc_flags.extend(rustdoc_flags_extras); + + let command = build + .cargo() + .timeout(Some(limits.timeout())) + .no_output_timeout(None) + .env( + "RUSTFLAGS", + metadata + .rustc_args + .as_ref() + .map(|args| args.join(" ")) + .unwrap_or_default(), + ) + .env("RUSTDOCFLAGS", rustdoc_flags.join(" ")) + // For docs.rs detection from build script: + // https://github.com/rust-lang/docs.rs/issues/147 + .env("DOCS_RS", "1") + .args(&cargo_args); + + Ok(command) + } + fn copy_docs( &self, target_dir: &Path, From d250251e5e5f5c4ff5369aeff2bd49bbe6a083cf Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Sat, 8 Aug 2020 21:57:21 +0200 Subject: [PATCH 035/139] context: create the Context trait --- src/bin/cratesfyi.rs | 40 +++++++++++++++++++++------------------- src/context.rs | 11 +++++++++++ src/lib.rs | 2 ++ src/test/mod.rs | 21 +++++++++++++++++++-- 4 files changed, 53 insertions(+), 21 deletions(-) create mode 100644 src/context.rs diff --git a/src/bin/cratesfyi.rs b/src/bin/cratesfyi.rs index bc6201bc7..348bb40eb 100644 --- a/src/bin/cratesfyi.rs +++ b/src/bin/cratesfyi.rs @@ -7,7 +7,7 @@ use cratesfyi::db::{self, add_path_into_database, Pool}; use cratesfyi::index::Index; use cratesfyi::utils::{remove_crate_priority, set_crate_priority}; use cratesfyi::{ - BuildQueue, Config, DocBuilder, DocBuilderOptions, RustwideBuilder, Server, Storage, + BuildQueue, Config, Context, DocBuilder, DocBuilderOptions, RustwideBuilder, Server, Storage, }; use failure::{err_msg, Error, ResultExt}; use once_cell::sync::OnceCell; @@ -108,7 +108,7 @@ enum CommandLine { impl CommandLine { pub fn handle_args(self) -> Result<(), Error> { - let ctx = Context::new(); + let ctx = BinContext::new(); match self { Self::Build(build) => build.handle_args(ctx)?, @@ -177,7 +177,7 @@ enum QueueSubcommand { } impl QueueSubcommand { - pub fn handle_args(self, ctx: Context) -> Result<(), Error> { + pub fn handle_args(self, ctx: BinContext) -> Result<(), Error> { match self { Self::Add { crate_name, @@ -213,7 +213,7 @@ enum PrioritySubcommand { } impl PrioritySubcommand { - pub fn handle_args(self, ctx: Context) -> Result<(), Error> { + pub fn handle_args(self, ctx: BinContext) -> Result<(), Error> { match self { Self::Set { pattern, priority } => { set_crate_priority(&mut *ctx.conn()?, &pattern, priority) @@ -259,7 +259,7 @@ struct Build { } impl Build { - pub fn handle_args(self, ctx: Context) -> Result<(), Error> { + pub fn handle_args(self, ctx: BinContext) -> Result<(), Error> { let docbuilder = { let config = ctx.config()?; let mut doc_options = @@ -324,7 +324,7 @@ enum BuildSubcommand { } impl BuildSubcommand { - pub fn handle_args(self, ctx: Context, mut docbuilder: DocBuilder) -> Result<(), Error> { + pub fn handle_args(self, ctx: BinContext, mut docbuilder: DocBuilder) -> Result<(), Error> { match self { Self::World => { docbuilder.load_cache().context("Failed to load cache")?; @@ -446,7 +446,7 @@ enum DatabaseSubcommand { } impl DatabaseSubcommand { - pub fn handle_args(self, ctx: Context) -> Result<(), Error> { + pub fn handle_args(self, ctx: BinContext) -> Result<(), Error> { match self { Self::Migrate { version } => { db::migrate(version, &mut *ctx.conn()?) @@ -514,7 +514,7 @@ enum BlacklistSubcommand { } impl BlacklistSubcommand { - fn handle_args(self, ctx: Context) -> Result<(), Error> { + fn handle_args(self, ctx: BinContext) -> Result<(), Error> { let mut conn = &mut *ctx.conn()?; match self { Self::List => { @@ -554,14 +554,14 @@ enum DeleteSubcommand { }, } -struct Context { +struct BinContext { build_queue: OnceCell>, storage: OnceCell>, config: OnceCell>, pool: OnceCell, } -impl Context { +impl BinContext { fn new() -> Self { Self { build_queue: OnceCell::new(), @@ -571,6 +571,17 @@ impl Context { } } + fn conn( + &self, + ) -> Result< + r2d2::PooledConnection>, + Error, + > { + Ok(self.pool()?.get()?) + } +} + +impl Context for BinContext { fn build_queue(&self) -> Result, Error> { Ok(self .build_queue @@ -602,13 +613,4 @@ impl Context { .get_or_try_init::<_, Error>(|| Ok(Pool::new(&*self.config()?)?))? .clone()) } - - fn conn( - &self, - ) -> Result< - r2d2::PooledConnection>, - Error, - > { - Ok(self.pool()?.get()?) - } } diff --git a/src/context.rs b/src/context.rs new file mode 100644 index 000000000..4edbf863a --- /dev/null +++ b/src/context.rs @@ -0,0 +1,11 @@ +use crate::db::Pool; +use crate::{BuildQueue, Config, Storage}; +use failure::Error; +use std::sync::Arc; + +pub trait Context { + fn config(&self) -> Result, Error>; + fn build_queue(&self) -> Result, Error>; + fn storage(&self) -> Result, Error>; + fn pool(&self) -> Result; +} diff --git a/src/lib.rs b/src/lib.rs index d25e79c5a..245bdabe2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,6 +4,7 @@ pub use self::build_queue::BuildQueue; pub use self::config::Config; +pub use self::context::Context; pub use self::docbuilder::options::DocBuilderOptions; pub use self::docbuilder::DocBuilder; pub use self::docbuilder::RustwideBuilder; @@ -12,6 +13,7 @@ pub use self::web::Server; mod build_queue; mod config; +mod context; pub mod db; mod docbuilder; mod error; diff --git a/src/test/mod.rs b/src/test/mod.rs index 5b0e4ffdf..d97f3d2b5 100644 --- a/src/test/mod.rs +++ b/src/test/mod.rs @@ -3,8 +3,7 @@ mod fakes; use crate::db::{Pool, PoolClient}; use crate::storage::Storage; use crate::web::Server; -use crate::BuildQueue; -use crate::Config; +use crate::{BuildQueue, Config, Context}; use failure::Error; use log::error; use once_cell::unsync::OnceCell; @@ -218,6 +217,24 @@ impl TestEnvironment { } } +impl Context for TestEnvironment { + fn config(&self) -> Result, Error> { + Ok(TestEnvironment::config(self)) + } + + fn build_queue(&self) -> Result, Error> { + Ok(TestEnvironment::build_queue(self)) + } + + fn storage(&self) -> Result, Error> { + Ok(TestEnvironment::storage(self)) + } + + fn pool(&self) -> Result { + Ok(self.db().pool()) + } +} + pub(crate) struct TestDatabase { pool: Pool, schema: String, From 154cb1ebe333d5f77384d7928669b1382dd74a6b Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Sat, 8 Aug 2020 22:20:29 +0200 Subject: [PATCH 036/139] web: use Context to initialize the web server --- src/bin/cratesfyi.rs | 10 ++------- src/test/mod.rs | 22 ++++--------------- src/utils/daemon.rs | 5 +++-- src/web/extensions.rs | 31 +++++++++++++++++++-------- src/web/mod.rs | 50 ++++++++++++------------------------------- 5 files changed, 45 insertions(+), 73 deletions(-) diff --git a/src/bin/cratesfyi.rs b/src/bin/cratesfyi.rs index 348bb40eb..1f204ccd3 100644 --- a/src/bin/cratesfyi.rs +++ b/src/bin/cratesfyi.rs @@ -116,14 +116,7 @@ impl CommandLine { socket_addr, reload_templates, } => { - Server::start( - Some(&socket_addr), - reload_templates, - ctx.pool()?, - ctx.config()?, - ctx.build_queue()?, - ctx.storage()?, - )?; + Server::start(Some(&socket_addr), reload_templates, &ctx)?; } Self::Daemon { foreground, @@ -134,6 +127,7 @@ impl CommandLine { } cratesfyi::utils::start_daemon( + &ctx, ctx.config()?, ctx.pool()?, ctx.build_queue()?, diff --git a/src/test/mod.rs b/src/test/mod.rs index d97f3d2b5..ac0ac99ec 100644 --- a/src/test/mod.rs +++ b/src/test/mod.rs @@ -185,9 +185,7 @@ impl TestEnvironment { } pub(crate) fn frontend(&self) -> &TestFrontend { - self.frontend.get_or_init(|| { - TestFrontend::new(self.db(), self.config(), self.build_queue(), self.storage()) - }) + self.frontend.get_or_init(|| TestFrontend::new(&*self)) } pub(crate) fn s3(&self) -> Arc { @@ -314,22 +312,10 @@ pub(crate) struct TestFrontend { } impl TestFrontend { - fn new( - db: &TestDatabase, - config: Arc, - build_queue: Arc, - storage: Arc, - ) -> Self { + fn new(context: &dyn Context) -> Self { Self { - server: Server::start( - Some("127.0.0.1:0"), - false, - db.pool.clone(), - config, - build_queue, - storage, - ) - .expect("failed to start the web server"), + server: Server::start(Some("127.0.0.1:0"), false, context) + .expect("failed to start the web server"), client: Client::new(), } } diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs index 835af18f2..c066caa6d 100644 --- a/src/utils/daemon.rs +++ b/src/utils/daemon.rs @@ -6,7 +6,7 @@ use crate::{ db::Pool, storage::Storage, utils::{queue_builder, update_release_activity, GithubUpdater}, - BuildQueue, Config, DocBuilder, DocBuilderOptions, + Context, BuildQueue, Config, DocBuilder, DocBuilderOptions, }; use chrono::{Timelike, Utc}; use failure::Error; @@ -54,6 +54,7 @@ fn start_registry_watcher( } pub fn start_daemon( + context: &dyn Context, config: Arc, db: Pool, build_queue: Arc, @@ -122,7 +123,7 @@ pub fn start_daemon( // at least start web server info!("Starting web server"); - crate::Server::start(None, false, db, config, build_queue, storage)?; + crate::Server::start(None, false, context)?; Ok(()) } diff --git a/src/web/extensions.rs b/src/web/extensions.rs index a8957f650..fabc285c4 100644 --- a/src/web/extensions.rs +++ b/src/web/extensions.rs @@ -1,18 +1,31 @@ -use crate::config::Config; -use crate::db::Pool; -use crate::storage::Storage; use crate::web::page::TemplateData; -use crate::BuildQueue; +use crate::{db::Pool, BuildQueue, Config, Context, Storage}; +use failure::Error; use iron::{BeforeMiddleware, IronResult, Request}; use std::sync::Arc; #[derive(Debug, Clone)] pub(super) struct InjectExtensions { - pub(super) build_queue: Arc, - pub(super) pool: Pool, - pub(super) config: Arc, - pub(super) storage: Arc, - pub(super) template_data: Arc, + build_queue: Arc, + pool: Pool, + config: Arc, + storage: Arc, + template_data: Arc, +} + +impl InjectExtensions { + pub(super) fn new( + context: &dyn Context, + template_data: Arc, + ) -> Result { + Ok(Self { + build_queue: context.build_queue()?, + pool: context.pool()?, + config: context.config()?, + storage: context.storage()?, + template_data, + }) + } } impl BeforeMiddleware for InjectExtensions { diff --git a/src/web/mod.rs b/src/web/mod.rs index 23dc8912b..3b31b3d37 100644 --- a/src/web/mod.rs +++ b/src/web/mod.rs @@ -77,7 +77,7 @@ mod rustdoc; mod sitemap; mod source; -use crate::{config::Config, db::Pool, impl_webpage, BuildQueue, Storage}; +use crate::{impl_webpage, Context}; use chrono::{DateTime, Utc}; use extensions::InjectExtensions; use failure::Error; @@ -123,19 +123,10 @@ impl CratesfyiHandler { } fn new( - pool: Pool, - config: Arc, template_data: Arc, - build_queue: Arc, - storage: Arc, - ) -> CratesfyiHandler { - let inject_extensions = InjectExtensions { - build_queue, - pool, - config, - storage, - template_data, - }; + context: &dyn Context, + ) -> Result { + let inject_extensions = InjectExtensions::new(context, template_data)?; let routes = routes::build_routes(); let blacklisted_prefixes = routes.page_prefixes(); @@ -151,7 +142,7 @@ impl CratesfyiHandler { let static_handler = Static::new(prefix).cache(Duration::from_secs(STATIC_FILE_CACHE_DURATION)); - CratesfyiHandler { + Ok(CratesfyiHandler { shared_resource_handler: Box::new(shared_resources), router_handler: Box::new(router_chain), database_file_handler: Box::new(routes::BlockBlacklistedPrefixes::new( @@ -160,7 +151,7 @@ impl CratesfyiHandler { )), static_handler: Box::new(static_handler), inject_extensions, - } + }) } } @@ -394,37 +385,24 @@ impl Server { pub fn start( addr: Option<&str>, reload_templates: bool, - db: Pool, - config: Arc, - build_queue: Arc, - storage: Arc, + context: &dyn Context, ) -> Result { // Initialize templates - let template_data = Arc::new(TemplateData::new(&mut *db.get()?)?); + let template_data = Arc::new(TemplateData::new(&mut *context.pool()?.get()?)?); if reload_templates { - TemplateData::start_template_reloading(template_data.clone(), db.clone()); + TemplateData::start_template_reloading(template_data.clone(), context.pool()?); } - let server = Self::start_inner( - addr.unwrap_or(DEFAULT_BIND), - db, - config, - template_data, - build_queue, - storage, - ); + let server = Self::start_inner(addr.unwrap_or(DEFAULT_BIND), template_data, context)?; info!("Running docs.rs web server on http://{}", server.addr()); Ok(server) } fn start_inner( addr: &str, - pool: Pool, - config: Arc, template_data: Arc, - build_queue: Arc, - storage: Arc, - ) -> Self { + context: &dyn Context, + ) -> Result { // poke all the metrics counters to instantiate and register them metrics::TOTAL_BUILDS.inc_by(0); metrics::SUCCESSFUL_BUILDS.inc_by(0); @@ -433,12 +411,12 @@ impl Server { metrics::UPLOADED_FILES_TOTAL.inc_by(0); metrics::FAILED_DB_CONNECTIONS.inc_by(0); - let cratesfyi = CratesfyiHandler::new(pool, config, template_data, build_queue, storage); + let cratesfyi = CratesfyiHandler::new(template_data, context)?; let inner = Iron::new(cratesfyi) .http(addr) .unwrap_or_else(|_| panic!("Failed to bind to socket on {}", addr)); - Server { inner } + Ok(Server { inner }) } pub(crate) fn addr(&self) -> SocketAddr { From 793bd58b0a3eb3ee7bc022e2428b82d73a954a16 Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Thu, 13 Aug 2020 22:16:15 +0300 Subject: [PATCH 037/139] Rebased --- src/bin/cratesfyi.rs | 9 +------- src/utils/daemon.rs | 54 +++++++++++++------------------------------- 2 files changed, 17 insertions(+), 46 deletions(-) diff --git a/src/bin/cratesfyi.rs b/src/bin/cratesfyi.rs index 1f204ccd3..8339628e1 100644 --- a/src/bin/cratesfyi.rs +++ b/src/bin/cratesfyi.rs @@ -126,14 +126,7 @@ impl CommandLine { log::warn!("--foreground was passed, but there is no need for it anymore"); } - cratesfyi::utils::start_daemon( - &ctx, - ctx.config()?, - ctx.pool()?, - ctx.build_queue()?, - ctx.storage()?, - registry_watcher == Toggle::Enabled, - )?; + cratesfyi::utils::start_daemon(&ctx, registry_watcher == Toggle::Enabled)?; } Self::Database { subcommand } => subcommand.handle_args(ctx)?, Self::Queue { subcommand } => subcommand.handle_args(ctx)?, diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs index c066caa6d..d142fa544 100644 --- a/src/utils/daemon.rs +++ b/src/utils/daemon.rs @@ -3,24 +3,19 @@ //! This daemon will start web server, track new packages and build them use crate::{ - db::Pool, - storage::Storage, utils::{queue_builder, update_release_activity, GithubUpdater}, - Context, BuildQueue, Config, DocBuilder, DocBuilderOptions, + Context, DocBuilder, DocBuilderOptions, }; use chrono::{Timelike, Utc}; use failure::Error; use log::{debug, error, info}; -use std::sync::Arc; use std::thread; use std::time::{Duration, Instant}; -fn start_registry_watcher( - opts: DocBuilderOptions, - pool: Pool, - build_queue: Arc, - config: Arc, -) -> Result<(), Error> { +fn start_registry_watcher(opts: DocBuilderOptions, context: &dyn Context) -> Result<(), Error> { + let pool = context.pool()?; + let build_queue = context.build_queue()?; + let config = context.config()?; thread::Builder::new() .name("registry index reader".to_string()) .spawn(move || { @@ -53,14 +48,8 @@ fn start_registry_watcher( Ok(()) } -pub fn start_daemon( - context: &dyn Context, - config: Arc, - db: Pool, - build_queue: Arc, - storage: Arc, - enable_registry_watcher: bool, -) -> Result<(), Error> { +pub fn start_daemon(context: &dyn Context, enable_registry_watcher: bool) -> Result<(), Error> { + let config = context.config()?; let dbopts = DocBuilderOptions::new(config.prefix.clone(), config.registry_index_path.clone()); // check paths once @@ -68,32 +57,23 @@ pub fn start_daemon( if enable_registry_watcher { // check new crates every minute - start_registry_watcher( - dbopts.clone(), - db.clone(), - build_queue.clone(), - config.clone(), - )?; + start_registry_watcher(dbopts.clone(), context)?; } // build new crates every minute - let cloned_db = db.clone(); - let cloned_build_queue = build_queue.clone(); - let cloned_storage = storage.clone(); + let pool = context.pool()?; + let build_queue = context.build_queue()?; + let storage = context.storage()?; thread::Builder::new() .name("build queue reader".to_string()) .spawn(move || { - let doc_builder = DocBuilder::new( - dbopts.clone(), - cloned_db.clone(), - cloned_build_queue.clone(), - ); - queue_builder(doc_builder, cloned_db, cloned_build_queue, cloned_storage).unwrap(); + let doc_builder = DocBuilder::new(dbopts.clone(), pool.clone(), build_queue.clone()); + queue_builder(doc_builder, pool, build_queue, storage).unwrap(); }) .unwrap(); // update release activity everyday at 23:55 - let cloned_db = db.clone(); + let pool = context.pool()?; cron( "release activity updater", Duration::from_secs(60), @@ -101,14 +81,14 @@ pub fn start_daemon( let now = Utc::now(); if now.hour() == 23 && now.minute() == 55 { info!("Updating release activity"); - update_release_activity(&mut *cloned_db.get()?)?; + update_release_activity(&mut *pool.get()?)?; } Ok(()) }, )?; // update github stats every hour - let github_updater = GithubUpdater::new(&config, db.clone())?; + let github_updater = GithubUpdater::new(&config, context.pool()?)?; cron( "github stats updater", Duration::from_secs(60 * 60), @@ -118,8 +98,6 @@ pub fn start_daemon( }, )?; - // TODO: update ssl certificate every 3 months - // at least start web server info!("Starting web server"); From 9436391684b62405361dfb03babdcd89aa9d48db Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Sun, 9 Aug 2020 12:00:04 +0200 Subject: [PATCH 038/139] bump prometheus to 0.9.0 --- Cargo.lock | 8 ++++---- Cargo.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b1095bbb4..2239f0ec8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -358,7 +358,7 @@ dependencies = [ "path-slash 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", "postgres 0.17.5 (registry+https://github.com/rust-lang/crates.io-index)", "procfs 0.7.9 (registry+https://github.com/rust-lang/crates.io-index)", - "prometheus 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "prometheus 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "r2d2 0.8.8 (registry+https://github.com/rust-lang/crates.io-index)", "r2d2_postgres 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2079,14 +2079,14 @@ dependencies = [ [[package]] name = "prometheus" -version = "0.7.0" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", "fnv 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "quick-error 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)", "spin 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", + "thiserror 1.0.20 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -3930,7 +3930,7 @@ dependencies = [ "checksum proc-macro2 1.0.13 (registry+https://github.com/rust-lang/crates.io-index)" = "53f5ffe53a6b28e37c9c1ce74893477864d64f74778a93a4beb43c8fa167f639" "checksum procedural-masquerade 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "8f1383dff4092fe903ac180e391a8d4121cc48f08ccf850614b0290c6673b69d" "checksum procfs 0.7.9 (registry+https://github.com/rust-lang/crates.io-index)" = "c434e93ef69c216e68e4f417c927b4f31502c3560b72cfdb6827e2321c5c6b3e" -"checksum prometheus 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5567486d5778e2c6455b1b90ff1c558f29e751fc018130fa182e15828e728af1" +"checksum prometheus 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "dd0ced56dee39a6e960c15c74dc48849d614586db2eaada6497477af7c7811cd" "checksum quick-error 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" "checksum quote 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "54a21852a652ad6f610c9510194f398ff6f8692e334fd1145fed931f7fbe44ea" "checksum r2d2 0.8.8 (registry+https://github.com/rust-lang/crates.io-index)" = "1497e40855348e4a8a40767d8e55174bce1e445a3ac9254ad44ad468ee0485af" diff --git a/Cargo.toml b/Cargo.toml index 60fa94494..9b224d88c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,7 +30,7 @@ toml = "0.5" schemamama = "0.3" schemamama_postgres = "0.3" systemstat = "0.1.4" -prometheus = { version = "0.7.0", default-features = false } +prometheus = { version = "0.9.0", default-features = false } rustwide = "0.10.0" mime_guess = "2" dotenv = "0.15" From c9b7fe1dd67609e0496080c5110549a4cb8de23e Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Thu, 13 Aug 2020 22:23:40 +0300 Subject: [PATCH 039/139] Rebased --- src/bin/cratesfyi.rs | 40 +++- src/build_queue.rs | 16 +- src/context.rs | 3 +- src/db/pool.rs | 32 ++- src/docbuilder/rustwide_builder.rs | 12 +- src/lib.rs | 2 + src/metrics/macros.rs | 65 ++++++ src/metrics/mod.rs | 97 +++++++++ src/storage/mod.rs | 10 +- src/storage/s3.rs | 14 +- src/test/mod.rs | 35 +++- src/utils/daemon.rs | 3 +- src/utils/queue_builder.rs | 6 +- src/web/extensions.rs | 6 +- src/web/metrics.rs | 310 +++++++---------------------- src/web/mod.rs | 8 - src/web/rustdoc.rs | 17 +- 17 files changed, 381 insertions(+), 295 deletions(-) create mode 100644 src/metrics/macros.rs create mode 100644 src/metrics/mod.rs diff --git a/src/bin/cratesfyi.rs b/src/bin/cratesfyi.rs index 8339628e1..63578b354 100644 --- a/src/bin/cratesfyi.rs +++ b/src/bin/cratesfyi.rs @@ -7,7 +7,8 @@ use cratesfyi::db::{self, add_path_into_database, Pool}; use cratesfyi::index::Index; use cratesfyi::utils::{remove_crate_priority, set_crate_priority}; use cratesfyi::{ - BuildQueue, Config, Context, DocBuilder, DocBuilderOptions, RustwideBuilder, Server, Storage, + BuildQueue, Config, Context, DocBuilder, DocBuilderOptions, Metrics, RustwideBuilder, Server, + Storage, }; use failure::{err_msg, Error, ResultExt}; use once_cell::sync::OnceCell; @@ -316,7 +317,8 @@ impl BuildSubcommand { Self::World => { docbuilder.load_cache().context("Failed to load cache")?; - let mut builder = RustwideBuilder::init(ctx.pool()?, ctx.storage()?)?; + let mut builder = + RustwideBuilder::init(ctx.pool()?, ctx.metrics()?, ctx.storage()?)?; builder .build_world(&mut docbuilder) .context("Failed to build world")?; @@ -330,8 +332,9 @@ impl BuildSubcommand { local, } => { docbuilder.load_cache().context("Failed to load cache")?; - let mut builder = RustwideBuilder::init(ctx.pool()?, ctx.storage()?) - .context("failed to initialize rustwide")?; + let mut builder = + RustwideBuilder::init(ctx.pool()?, ctx.metrics()?, ctx.storage()?) + .context("failed to initialize rustwide")?; if let Some(path) = local { builder @@ -367,14 +370,16 @@ impl BuildSubcommand { } } - let mut builder = RustwideBuilder::init(ctx.pool()?, ctx.storage()?)?; + let mut builder = + RustwideBuilder::init(ctx.pool()?, ctx.metrics()?, ctx.storage()?)?; builder .update_toolchain() .context("failed to update toolchain")?; } Self::AddEssentialFiles => { - let mut builder = RustwideBuilder::init(ctx.pool()?, ctx.storage()?)?; + let mut builder = + RustwideBuilder::init(ctx.pool()?, ctx.metrics()?, ctx.storage()?)?; builder .add_essential_files() .context("failed to add essential files")?; @@ -546,6 +551,7 @@ struct BinContext { storage: OnceCell>, config: OnceCell>, pool: OnceCell, + metrics: OnceCell>, } impl BinContext { @@ -555,6 +561,7 @@ impl BinContext { storage: OnceCell::new(), config: OnceCell::new(), pool: OnceCell::new(), + metrics: OnceCell::new(), } } @@ -573,7 +580,11 @@ impl Context for BinContext { Ok(self .build_queue .get_or_try_init::<_, Error>(|| { - Ok(Arc::new(BuildQueue::new(self.pool()?, &*self.config()?))) + Ok(Arc::new(BuildQueue::new( + self.pool()?, + self.metrics()?, + &*self.config()?, + ))) })? .clone()) } @@ -582,7 +593,11 @@ impl Context for BinContext { Ok(self .storage .get_or_try_init::<_, Error>(|| { - Ok(Arc::new(Storage::new(self.pool()?, &*self.config()?)?)) + Ok(Arc::new(Storage::new( + self.pool()?, + self.metrics()?, + &*self.config()?, + )?)) })? .clone()) } @@ -597,7 +612,14 @@ impl Context for BinContext { fn pool(&self) -> Result { Ok(self .pool - .get_or_try_init::<_, Error>(|| Ok(Pool::new(&*self.config()?)?))? + .get_or_try_init::<_, Error>(|| Ok(Pool::new(&*self.config()?, self.metrics()?)?))? + .clone()) + } + + fn metrics(&self) -> Result, Error> { + Ok(self + .metrics + .get_or_try_init::<_, Error>(|| Ok(Arc::new(Metrics::new()?)))? .clone()) } } diff --git a/src/build_queue.rs b/src/build_queue.rs index a5490abc6..869aa9b38 100644 --- a/src/build_queue.rs +++ b/src/build_queue.rs @@ -1,7 +1,8 @@ -use crate::config::Config; use crate::db::Pool; use crate::error::Result; +use crate::{Config, Metrics}; use log::error; +use std::sync::Arc; #[derive(Debug, Clone, Eq, PartialEq, serde::Serialize)] pub(crate) struct QueuedCrate { @@ -15,13 +16,15 @@ pub(crate) struct QueuedCrate { #[derive(Debug)] pub struct BuildQueue { db: Pool, + metrics: Arc, max_attempts: i32, } impl BuildQueue { - pub fn new(db: Pool, config: &Config) -> Self { + pub fn new(db: Pool, metrics: Arc, config: &Config) -> Self { BuildQueue { db, + metrics, max_attempts: config.build_attempts.into(), } } @@ -91,7 +94,7 @@ impl BuildQueue { }; let res = f(&to_process); - crate::web::metrics::TOTAL_BUILDS.inc(); + self.metrics.total_builds.inc(); match res { Ok(()) => { conn.execute("DELETE FROM queue WHERE id = $1;", &[&to_process.id])?; @@ -105,7 +108,7 @@ impl BuildQueue { let attempt: i32 = rows[0].get(0); if attempt >= self.max_attempts { - crate::web::metrics::FAILED_BUILDS.inc(); + self.metrics.failed_builds.inc(); } error!( @@ -196,6 +199,11 @@ mod tests { })?; assert!(!called, "there were still items in the queue"); + // Ensure metrics were recorded correctly + let metrics = env.metrics(); + assert_eq!(metrics.total_builds.get(), 9); + assert_eq!(metrics.failed_builds.get(), 1); + Ok(()) }) } diff --git a/src/context.rs b/src/context.rs index 4edbf863a..cc18b1430 100644 --- a/src/context.rs +++ b/src/context.rs @@ -1,5 +1,5 @@ use crate::db::Pool; -use crate::{BuildQueue, Config, Storage}; +use crate::{BuildQueue, Config, Metrics, Storage}; use failure::Error; use std::sync::Arc; @@ -8,4 +8,5 @@ pub trait Context { fn build_queue(&self) -> Result, Error>; fn storage(&self) -> Result, Error>; fn pool(&self) -> Result; + fn metrics(&self) -> Result, Error>; } diff --git a/src/db/pool.rs b/src/db/pool.rs index db9b3d235..acbc8bf5a 100644 --- a/src/db/pool.rs +++ b/src/db/pool.rs @@ -1,6 +1,8 @@ +use crate::metrics::Metrics; use crate::Config; use postgres::{Client, NoTls}; use r2d2_postgres::PostgresConnectionManager; +use std::sync::Arc; pub(crate) type PoolClient = r2d2::PooledConnection>; @@ -9,21 +11,25 @@ const DEFAULT_SCHEMA: &str = "public"; #[derive(Debug, Clone)] pub struct Pool { pool: r2d2::Pool>, + metrics: Arc, + max_size: u32, } impl Pool { - pub fn new(config: &Config) -> Result { - Self::new_inner(config, DEFAULT_SCHEMA) + pub fn new(config: &Config, metrics: Arc) -> Result { + Self::new_inner(config, metrics, DEFAULT_SCHEMA) } #[cfg(test)] - pub(crate) fn new_with_schema(config: &Config, schema: &str) -> Result { - Self::new_inner(config, schema) + pub(crate) fn new_with_schema( + config: &Config, + metrics: Arc, + schema: &str, + ) -> Result { + Self::new_inner(config, metrics, schema) } - fn new_inner(config: &Config, schema: &str) -> Result { - crate::web::metrics::MAX_DB_CONNECTIONS.set(config.max_pool_size as i64); - + fn new_inner(config: &Config, metrics: Arc, schema: &str) -> Result { let url = config .database_url .parse() @@ -36,14 +42,18 @@ impl Pool { .build(manager) .map_err(PoolError::PoolCreationFailed)?; - Ok(Pool { pool }) + Ok(Pool { + pool, + metrics, + max_size: config.max_pool_size, + }) } pub fn get(&self) -> Result { match self.pool.get() { Ok(conn) => Ok(conn), Err(err) => { - crate::web::metrics::FAILED_DB_CONNECTIONS.inc(); + self.metrics.failed_db_connections.inc(); Err(PoolError::ClientError(err)) } } @@ -56,6 +66,10 @@ impl Pool { pub(crate) fn idle_connections(&self) -> u32 { self.pool.state().idle_connections } + + pub(crate) fn max_size(&self) -> u32 { + self.max_size + } } #[derive(Debug)] diff --git a/src/docbuilder/rustwide_builder.rs b/src/docbuilder/rustwide_builder.rs index 19fe02d6f..474d37675 100644 --- a/src/docbuilder/rustwide_builder.rs +++ b/src/docbuilder/rustwide_builder.rs @@ -10,8 +10,8 @@ use crate::docbuilder::{crates::crates_from_path, Limits}; use crate::error::Result; use crate::index::api::ReleaseData; use crate::storage::CompressionAlgorithms; -use crate::storage::Storage; use crate::utils::{copy_doc_dir, parse_rustc_version, CargoMetadata}; +use crate::{Metrics, Storage}; use failure::ResultExt; use log::{debug, info, warn, LevelFilter}; use rustwide::cmd::{Command, SandboxBuilder}; @@ -74,12 +74,13 @@ pub struct RustwideBuilder { toolchain: Toolchain, db: Pool, storage: Arc, + metrics: Arc, rustc_version: String, cpu_limit: Option, } impl RustwideBuilder { - pub fn init(db: Pool, storage: Arc) -> Result { + pub fn init(db: Pool, metrics: Arc, storage: Arc) -> Result { use rustwide::cmd::SandboxImage; let env_workspace_path = ::std::env::var("CRATESFYI_RUSTWIDE_WORKSPACE"); let workspace_path = env_workspace_path @@ -115,6 +116,7 @@ impl RustwideBuilder { toolchain, db, storage, + metrics, rustc_version: String::new(), cpu_limit, }) @@ -399,11 +401,11 @@ impl RustwideBuilder { let has_examples = build.host_source_dir().join("examples").is_dir(); if res.result.successful { - crate::web::metrics::SUCCESSFUL_BUILDS.inc(); + self.metrics.successful_builds.inc(); } else if res.cargo_metadata.root().is_library() { - crate::web::metrics::FAILED_BUILDS.inc(); + self.metrics.failed_builds.inc(); } else { - crate::web::metrics::NON_LIBRARY_BUILDS.inc(); + self.metrics.non_library_builds.inc(); } let release_data = match doc_builder.index.api().get_release_data(name, version) { diff --git a/src/lib.rs b/src/lib.rs index 245bdabe2..63eaa9ac7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -8,6 +8,7 @@ pub use self::context::Context; pub use self::docbuilder::options::DocBuilderOptions; pub use self::docbuilder::DocBuilder; pub use self::docbuilder::RustwideBuilder; +pub use self::metrics::Metrics; pub use self::storage::Storage; pub use self::web::Server; @@ -18,6 +19,7 @@ pub mod db; mod docbuilder; mod error; pub mod index; +mod metrics; pub mod storage; #[cfg(test)] mod test; diff --git a/src/metrics/macros.rs b/src/metrics/macros.rs new file mode 100644 index 000000000..c79bd60df --- /dev/null +++ b/src/metrics/macros.rs @@ -0,0 +1,65 @@ +pub(super) trait MetricFromOpts: Sized { + fn from_opts(opts: prometheus::Opts) -> Result; +} + +#[macro_export] +macro_rules! metrics { + ( + $vis:vis struct $name:ident { + $(#[doc = $help:expr] $metric:ident: $ty:ty $([$($label:expr),*])?,)* + } + metrics visibility: $metric_vis:vis, + namespace: $namespace:expr, + ) => { + $vis struct $name { + registry: Registry, + $($metric_vis $metric: $ty,)* + } + impl $name { + $vis fn new() -> Result { + let registry = Registry::new(); + $( + let $metric = <$ty>::from_opts( + Opts::new(stringify!($metric), $help) + .namespace($namespace) + $(.variable_labels(vec![$($label.into()),*]))? + )?; + registry.register(Box::new($metric.clone()))?; + )* + Ok(Self { registry, $($metric,)* }) + } + } + impl std::fmt::Debug for $name { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "{}", stringify!($name)) + } + } + }; +} + +#[macro_export] +macro_rules! load_metric_type { + ($name:ident as single) => { + use prometheus::$name; + impl MetricFromOpts for $name { + fn from_opts(opts: Opts) -> Result { + $name::with_opts(opts) + } + } + }; + ($name:ident as vec) => { + use prometheus::$name; + impl MetricFromOpts for $name { + fn from_opts(opts: Opts) -> Result { + $name::new( + opts.clone().into(), + opts.variable_labels + .iter() + .map(|s| s.as_str()) + .collect::>() + .as_slice(), + ) + } + } + }; +} diff --git a/src/metrics/mod.rs b/src/metrics/mod.rs new file mode 100644 index 000000000..870109a7e --- /dev/null +++ b/src/metrics/mod.rs @@ -0,0 +1,97 @@ +#[macro_use] +mod macros; + +use self::macros::MetricFromOpts; +use crate::db::Pool; +use crate::BuildQueue; +use failure::Error; +use prometheus::{proto::MetricFamily, Opts, Registry}; + +load_metric_type!(IntGauge as single); +load_metric_type!(IntCounter as single); +load_metric_type!(IntCounterVec as vec); +load_metric_type!(HistogramVec as vec); + +metrics! { + pub struct Metrics { + /// Number of crates in the build queue + queued_crates_count: IntGauge, + /// Number of crates in the build queue that have a positive priority + prioritized_crates_count: IntGauge, + /// Number of crates that failed to build + failed_crates_count: IntGauge, + + /// The number of idle database connections + idle_db_connections: IntGauge, + /// The number of used database connections + used_db_connections: IntGauge, + /// The maximum number of database connections + max_db_connections: IntGauge, + /// Number of attempted and failed connections to the database + failed_db_connections: IntCounter, + + /// The number of currently opened file descriptors + open_file_descriptors: IntGauge, + /// The number of threads being used by docs.rs + running_threads: IntGauge, + + /// The traffic of various docs.rs routes + routes_visited: IntCounterVec["route"], + /// The response times of various docs.rs routes + response_time: HistogramVec["route"], + /// The time it takes to render a rustdoc page + rustdoc_rendering_times: HistogramVec["step"], + + /// Number of crates built + total_builds: IntCounter, + /// Number of builds that successfully generated docs + successful_builds: IntCounter, + /// Number of builds that generated a compiler error + failed_builds: IntCounter, + /// Number of builds that did not complete due to not being a library + non_library_builds: IntCounter, + + /// Number of files uploaded to the storage backend + uploaded_files_total: IntCounter, + + /// The number of attempted files that failed due to a memory limit + html_rewrite_ooms: IntCounter, + } + + metrics visibility: pub(crate), + namespace: "docsrs", +} + +impl Metrics { + pub(crate) fn gather( + &self, + pool: &Pool, + queue: &BuildQueue, + ) -> Result, Error> { + self.idle_db_connections.set(pool.idle_connections() as i64); + self.used_db_connections.set(pool.used_connections() as i64); + self.max_db_connections.set(pool.max_size() as i64); + + self.queued_crates_count.set(queue.pending_count()? as i64); + self.prioritized_crates_count + .set(queue.prioritized_count()? as i64); + self.failed_crates_count.set(queue.failed_count()? as i64); + + self.gather_system_performance(); + Ok(self.registry.gather()) + } + + #[cfg(not(linux))] + fn gather_system_performance(&self) {} + + #[cfg(target_os = "linux")] + fn gather_system_performance(&self) { + use procfs::process::Process; + + let process = Process::myself().unwrap(); + self.open_file_descriptors + .set(process.fd().unwrap().len() as i64); + self.running_threads + .set(process.stat().unwrap().num_threads as i64); + } +} diff --git a/src/storage/mod.rs b/src/storage/mod.rs index bdb643ffe..5ded2d9fa 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -5,7 +5,7 @@ mod s3; pub use self::compression::{compress, decompress, CompressionAlgorithm, CompressionAlgorithms}; use self::database::DatabaseBackend; use self::s3::S3Backend; -use crate::{db::Pool, Config}; +use crate::{db::Pool, Config, Metrics}; use chrono::{DateTime, Utc}; use failure::{err_msg, Error}; use path_slash::PathExt; @@ -14,6 +14,7 @@ use std::{ ffi::OsStr, fmt, fs, path::{Path, PathBuf}, + sync::Arc, }; const MAX_CONCURRENT_UPLOADS: usize = 1000; @@ -76,9 +77,9 @@ pub struct Storage { } impl Storage { - pub fn new(pool: Pool, config: &Config) -> Result { + pub fn new(pool: Pool, metrics: Arc, config: &Config) -> Result { let backend = if let Some(c) = s3::s3_client() { - StorageBackend::S3(Box::new(S3Backend::new(c, config)?)) + StorageBackend::S3(Box::new(S3Backend::new(c, metrics, config)?)) } else { StorageBackend::Database(DatabaseBackend::new(pool)) }; @@ -86,10 +87,11 @@ impl Storage { } #[cfg(test)] - pub(crate) fn temp_new_s3(config: &Config) -> Result { + pub(crate) fn temp_new_s3(metrics: Arc, config: &Config) -> Result { Ok(Storage { backend: StorageBackend::S3(Box::new(S3Backend::new( s3::s3_client().unwrap(), + metrics, config, )?)), }) diff --git a/src/storage/s3.rs b/src/storage/s3.rs index b45813664..e8b541c1c 100644 --- a/src/storage/s3.rs +++ b/src/storage/s3.rs @@ -1,5 +1,5 @@ use super::{Blob, StorageTransaction}; -use crate::Config; +use crate::{Config, Metrics}; use chrono::{DateTime, NaiveDateTime, Utc}; use failure::Error; use futures_util::{ @@ -13,19 +13,24 @@ use rusoto_s3::{ DeleteObjectsRequest, GetObjectError, GetObjectRequest, HeadObjectError, HeadObjectRequest, ListObjectsV2Request, ObjectIdentifier, PutObjectRequest, S3Client, S3, }; -use std::{convert::TryInto, io::Write}; +use std::{convert::TryInto, io::Write, sync::Arc}; use tokio::runtime::Runtime; pub(super) struct S3Backend { client: S3Client, runtime: Runtime, bucket: String, + metrics: Arc, #[cfg(test)] temporary: bool, } impl S3Backend { - pub(super) fn new(client: S3Client, config: &Config) -> Result { + pub(super) fn new( + client: S3Client, + metrics: Arc, + config: &Config, + ) -> Result { let runtime = Runtime::new()?; #[cfg(test)] @@ -48,6 +53,7 @@ impl S3Backend { Ok(Self { client, runtime, + metrics, bucket: config.s3_bucket.clone(), #[cfg(test)] temporary: config.s3_bucket_is_temporary, @@ -173,7 +179,7 @@ impl<'a> StorageTransaction for S3StorageTransaction<'a> { ..Default::default() }) .map_ok(|_| { - crate::web::metrics::UPLOADED_FILES_TOTAL.inc_by(1); + self.s3.metrics.uploaded_files_total.inc(); }) .map_err(|err| { log::error!("Failed to upload blob to S3: {:?}", err); diff --git a/src/test/mod.rs b/src/test/mod.rs index ac0ac99ec..c1b9d5462 100644 --- a/src/test/mod.rs +++ b/src/test/mod.rs @@ -3,7 +3,7 @@ mod fakes; use crate::db::{Pool, PoolClient}; use crate::storage::Storage; use crate::web::Server; -use crate::{BuildQueue, Config, Context}; +use crate::{BuildQueue, Config, Context, Metrics}; use failure::Error; use log::error; use once_cell::unsync::OnceCell; @@ -96,6 +96,7 @@ pub(crate) struct TestEnvironment { config: OnceCell>, db: OnceCell, storage: OnceCell>, + metrics: OnceCell>, frontend: OnceCell, s3: OnceCell>, storage_db: OnceCell>, @@ -116,6 +117,7 @@ impl TestEnvironment { config: OnceCell::new(), db: OnceCell::new(), storage: OnceCell::new(), + metrics: OnceCell::new(), frontend: OnceCell::new(), s3: OnceCell::new(), storage_db: OnceCell::new(), @@ -158,7 +160,13 @@ impl TestEnvironment { pub(crate) fn build_queue(&self) -> Arc { self.build_queue - .get_or_init(|| Arc::new(BuildQueue::new(self.db().pool(), &self.config()))) + .get_or_init(|| { + Arc::new(BuildQueue::new( + self.db().pool(), + self.metrics(), + &self.config(), + )) + }) .clone() } @@ -172,16 +180,23 @@ impl TestEnvironment { self.storage .get_or_init(|| { Arc::new( - Storage::new(self.db().pool(), &*self.config()) + Storage::new(self.db().pool(), self.metrics(), &*self.config()) .expect("failed to initialize the storage"), ) }) .clone() } + pub(crate) fn metrics(&self) -> Arc { + self.metrics + .get_or_init(|| Arc::new(Metrics::new().expect("failed to initialize the metrics"))) + .clone() + } + pub(crate) fn db(&self) -> &TestDatabase { - self.db - .get_or_init(|| TestDatabase::new(&self.config()).expect("failed to initialize the db")) + self.db.get_or_init(|| { + TestDatabase::new(&self.config(), self.metrics()).expect("failed to initialize the db") + }) } pub(crate) fn frontend(&self) -> &TestFrontend { @@ -192,7 +207,7 @@ impl TestEnvironment { self.s3 .get_or_init(|| { Arc::new( - Storage::temp_new_s3(&*self.config()) + Storage::temp_new_s3(self.metrics(), &*self.config()) .expect("failed to initialize the storage"), ) }) @@ -231,6 +246,10 @@ impl Context for TestEnvironment { fn pool(&self) -> Result { Ok(self.db().pool()) } + + fn metrics(&self) -> Result, Error> { + Ok(self.metrics()) + } } pub(crate) struct TestDatabase { @@ -239,7 +258,7 @@ pub(crate) struct TestDatabase { } impl TestDatabase { - fn new(config: &Config) -> Result { + fn new(config: &Config, metrics: Arc) -> Result { // A random schema name is generated and used for the current connection. This allows each // test to create a fresh instance of the database to run within. let schema = format!("docs_rs_test_schema_{}", rand::random::()); @@ -278,7 +297,7 @@ impl TestDatabase { conn.batch_execute(&query)?; Ok(TestDatabase { - pool: Pool::new_with_schema(config, &schema)?, + pool: Pool::new_with_schema(config, metrics, &schema)?, schema, }) } diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs index d142fa544..673f05204 100644 --- a/src/utils/daemon.rs +++ b/src/utils/daemon.rs @@ -64,11 +64,12 @@ pub fn start_daemon(context: &dyn Context, enable_registry_watcher: bool) -> Res let pool = context.pool()?; let build_queue = context.build_queue()?; let storage = context.storage()?; + let metrics = context.metrics()?; thread::Builder::new() .name("build queue reader".to_string()) .spawn(move || { let doc_builder = DocBuilder::new(dbopts.clone(), pool.clone(), build_queue.clone()); - queue_builder(doc_builder, pool, build_queue, storage).unwrap(); + queue_builder(doc_builder, pool, build_queue, metrics, storage).unwrap(); }) .unwrap(); diff --git a/src/utils/queue_builder.rs b/src/utils/queue_builder.rs index 1eaef5478..95f5285cf 100644 --- a/src/utils/queue_builder.rs +++ b/src/utils/queue_builder.rs @@ -1,5 +1,6 @@ use crate::{ - db::Pool, docbuilder::RustwideBuilder, utils::pubsubhubbub, BuildQueue, DocBuilder, Storage, + db::Pool, docbuilder::RustwideBuilder, utils::pubsubhubbub, BuildQueue, DocBuilder, Metrics, + Storage, }; use failure::Error; use log::{debug, error, info, warn}; @@ -13,6 +14,7 @@ pub fn queue_builder( mut doc_builder: DocBuilder, db: Pool, build_queue: Arc, + metrics: Arc, storage: Arc, ) -> Result<(), Error> { /// Represents the current state of the builder thread. @@ -28,7 +30,7 @@ pub fn queue_builder( QueueInProgress(usize), } - let mut builder = RustwideBuilder::init(db, storage)?; + let mut builder = RustwideBuilder::init(db, metrics, storage)?; let mut status = BuilderState::Fresh; diff --git a/src/web/extensions.rs b/src/web/extensions.rs index fabc285c4..468e67b66 100644 --- a/src/web/extensions.rs +++ b/src/web/extensions.rs @@ -1,5 +1,5 @@ use crate::web::page::TemplateData; -use crate::{db::Pool, BuildQueue, Config, Context, Storage}; +use crate::{db::Pool, BuildQueue, Config, Context, Metrics, Storage}; use failure::Error; use iron::{BeforeMiddleware, IronResult, Request}; use std::sync::Arc; @@ -10,6 +10,7 @@ pub(super) struct InjectExtensions { pool: Pool, config: Arc, storage: Arc, + metrics: Arc, template_data: Arc, } @@ -23,6 +24,7 @@ impl InjectExtensions { pool: context.pool()?, config: context.config()?, storage: context.storage()?, + metrics: context.metrics()?, template_data, }) } @@ -35,6 +37,7 @@ impl BeforeMiddleware for InjectExtensions { req.extensions.insert::(self.pool.clone()); req.extensions.insert::(self.config.clone()); req.extensions.insert::(self.storage.clone()); + req.extensions.insert::(self.metrics.clone()); req.extensions .insert::(self.template_data.clone()); @@ -54,4 +57,5 @@ key!(BuildQueue => Arc); key!(Pool => Pool); key!(Config => Arc); key!(Storage => Arc); +key!(Metrics => Arc); key!(TemplateData => Arc); diff --git a/src/web/metrics.rs b/src/web/metrics.rs index 18a11ed13..14a1c54f1 100644 --- a/src/web/metrics.rs +++ b/src/web/metrics.rs @@ -1,182 +1,19 @@ use crate::db::Pool; use crate::BuildQueue; +use crate::Metrics; use iron::headers::ContentType; use iron::prelude::*; use iron::status::Status; -use once_cell::sync::Lazy; -use prometheus::{ - opts, register_counter, register_int_counter, register_int_gauge, Encoder, IntCounter, - IntGauge, TextEncoder, __register_gauge, register_int_counter_vec, IntCounterVec, - __register_counter_vec, histogram_opts, register_histogram_vec, HistogramVec, -}; +use prometheus::{Encoder, HistogramVec, TextEncoder}; use std::time::{Duration, Instant}; -static QUEUED_CRATES_COUNT: Lazy = Lazy::new(|| { - register_int_gauge!( - "docsrs_queued_crates_count", - "Number of crates in the build queue" - ) - .unwrap() -}); - -pub static PRIORITIZED_CRATES_COUNT: Lazy = Lazy::new(|| { - register_int_gauge!( - "docsrs_prioritized_crates_count", - "Number of crates in the build queue that have a positive priority" - ) - .unwrap() -}); - -static FAILED_CRATES_COUNT: Lazy = Lazy::new(|| { - register_int_gauge!( - "docsrs_failed_crates_count", - "Number of crates that failed to build" - ) - .unwrap() -}); - -pub static TOTAL_BUILDS: Lazy = - Lazy::new(|| register_int_counter!("docsrs_total_builds", "Number of crates built").unwrap()); - -pub static SUCCESSFUL_BUILDS: Lazy = Lazy::new(|| { - register_int_counter!( - "docsrs_successful_builds", - "Number of builds that successfully generated docs" - ) - .unwrap() -}); - -pub static FAILED_BUILDS: Lazy = Lazy::new(|| { - register_int_counter!( - "docsrs_failed_builds", - "Number of builds that generated a compile error" - ) - .unwrap() -}); - -pub static NON_LIBRARY_BUILDS: Lazy = Lazy::new(|| { - register_int_counter!( - "docsrs_non_library_builds", - "Number of builds that did not complete due to not being a library" - ) - .unwrap() -}); - -pub static UPLOADED_FILES_TOTAL: Lazy = Lazy::new(|| { - register_int_counter!( - "docsrs_uploaded_files_total", - "Number of files uploaded to S3 or stored in the database" - ) - .unwrap() -}); - -pub static ROUTES_VISITED: Lazy = Lazy::new(|| { - register_int_counter_vec!( - "docsrs_routes_visited", - "The traffic of various docs.rs routes", - &["route"] - ) - .unwrap() -}); - -pub static RESPONSE_TIMES: Lazy = Lazy::new(|| { - register_histogram_vec!( - "docsrs_response_time", - "The response times of various docs.rs routes", - &["route"] - ) - .unwrap() -}); - -pub static RUSTDOC_RENDERING_TIMES: Lazy = Lazy::new(|| { - register_histogram_vec!( - "docsrs_rustdoc_rendering_time", - "The time it takes to render a rustdoc page", - &["step"] - ) - .unwrap() -}); - -pub static FAILED_DB_CONNECTIONS: Lazy = Lazy::new(|| { - register_int_counter!( - "docsrs_failed_db_connections", - "Number of attempted and failed connections to the database" - ) - .unwrap() -}); - -pub static USED_DB_CONNECTIONS: Lazy = Lazy::new(|| { - register_int_gauge!( - "docsrs_used_db_connections", - "The number of used database connections" - ) - .unwrap() -}); - -pub static IDLE_DB_CONNECTIONS: Lazy = Lazy::new(|| { - register_int_gauge!( - "docsrs_idle_db_connections", - "The number of idle database connections" - ) - .unwrap() -}); - -pub static MAX_DB_CONNECTIONS: Lazy = Lazy::new(|| { - register_int_gauge!( - "docsrs_max_db_connections", - "The maximum database connections" - ) - .unwrap() -}); - -#[cfg(target_os = "linux")] -pub static OPEN_FILE_DESCRIPTORS: Lazy = Lazy::new(|| { - register_int_gauge!( - "docsrs_open_file_descriptors", - "The number of currently opened file descriptors" - ) - .unwrap() -}); - -#[cfg(target_os = "linux")] -pub static CURRENTLY_RUNNING_THREADS: Lazy = Lazy::new(|| { - register_int_gauge!( - "docsrs_running_threads", - "The number of threads being used by docs.rs" - ) - .unwrap() -}); - -pub static HTML_REWRITE_OOMS: Lazy = Lazy::new(|| { - register_int_gauge!( - "docsrs_html_rewrite_ooms", - "The number of attempted files that failed due to a memory limit" - ) - .unwrap() -}); - pub fn metrics_handler(req: &mut Request) -> IronResult { + let metrics = extension!(req, Metrics); let pool = extension!(req, Pool); let queue = extension!(req, BuildQueue); - USED_DB_CONNECTIONS.set(pool.used_connections() as i64); - IDLE_DB_CONNECTIONS.set(pool.idle_connections() as i64); - - QUEUED_CRATES_COUNT.set(ctry!(req, queue.pending_count()) as i64); - PRIORITIZED_CRATES_COUNT.set(ctry!(req, queue.prioritized_count()) as i64); - FAILED_CRATES_COUNT.set(ctry!(req, queue.failed_count()) as i64); - - #[cfg(target_os = "linux")] - { - use procfs::process::Process; - - let process = Process::myself().unwrap(); - OPEN_FILE_DESCRIPTORS.set(process.fd().unwrap().len() as i64); - CURRENTLY_RUNNING_THREADS.set(process.stat().unwrap().num_threads as i64); - } - let mut buffer = Vec::new(); - let families = prometheus::gather(); + let families = ctry!(req, metrics.gather(pool, &*queue)); ctry!(req, TextEncoder::new().encode(&families, &mut buffer)); let mut resp = Response::with(buffer); @@ -213,14 +50,16 @@ impl iron::Handler for RequestRecorder { let result = self.handler.handle(request); let resp_time = duration_to_seconds(start.elapsed()); - ROUTES_VISITED.with_label_values(&[&self.route_name]).inc(); - RESPONSE_TIMES + let metrics = extension!(request, Metrics); + metrics + .routes_visited + .with_label_values(&[&self.route_name]) + .inc(); + metrics + .response_time .with_label_values(&[&self.route_name]) .observe(resp_time); - #[cfg(test)] - tests::record_tests(&self.route_name); - result } } @@ -230,13 +69,13 @@ struct RenderingTime { step: &'static str, } -pub(crate) struct RenderingTimesRecorder { - metric: &'static HistogramVec, +pub(crate) struct RenderingTimesRecorder<'a> { + metric: &'a HistogramVec, current: Option, } -impl RenderingTimesRecorder { - pub(crate) fn new(metric: &'static HistogramVec) -> Self { +impl<'a> RenderingTimesRecorder<'a> { + pub(crate) fn new(metric: &'a HistogramVec) -> Self { Self { metric, current: None, @@ -260,7 +99,7 @@ impl RenderingTimesRecorder { } } -impl Drop for RenderingTimesRecorder { +impl Drop for RenderingTimesRecorder<'_> { fn drop(&mut self) { self.record_current(); } @@ -269,53 +108,34 @@ impl Drop for RenderingTimesRecorder { #[cfg(test)] mod tests { use crate::test::{assert_success, wrapper}; - use once_cell::sync::Lazy; - use std::{ - collections::HashMap, - sync::{ - atomic::{AtomicUsize, Ordering}, - Mutex, - }, - }; - - static ROUTES_VISITED: AtomicUsize = AtomicUsize::new(0); - static RESPONSE_TIMES: Lazy>> = - Lazy::new(|| Mutex::new(HashMap::new())); - - pub fn record_tests(route: &str) { - ROUTES_VISITED.fetch_add(1, Ordering::SeqCst); - - let mut times = RESPONSE_TIMES.lock().unwrap(); - if let Some(requests) = times.get_mut(route) { - *requests += 1; - } else { - times.insert(route.to_owned(), 1); - } - } - - fn reset_records() { - ROUTES_VISITED.store(0, Ordering::SeqCst); - RESPONSE_TIMES.lock().unwrap().clear(); - } #[test] fn home_page() { wrapper(|env| { let frontend = env.frontend(); - - reset_records(); + let metrics = env.metrics(); frontend.get("/").send()?; frontend.get("/").send()?; - assert_eq!(ROUTES_VISITED.load(Ordering::SeqCst), 2); - assert_eq!(RESPONSE_TIMES.lock().unwrap().get("/"), Some(&2)); - - reset_records(); + assert_eq!(metrics.routes_visited.with_label_values(&["/"]).get(), 2); + assert_eq!( + metrics + .response_time + .with_label_values(&["/"]) + .get_sample_count(), + 2 + ); frontend.get("").send()?; frontend.get("").send()?; - assert_eq!(ROUTES_VISITED.load(Ordering::SeqCst), 2); - assert_eq!(RESPONSE_TIMES.lock().unwrap().get("/"), Some(&2)); + assert_eq!(metrics.routes_visited.with_label_values(&["/"]).get(), 4); + assert_eq!( + metrics + .response_time + .with_label_values(&["/"]) + .get_sample_count(), + 4 + ); Ok(()) }) @@ -325,6 +145,7 @@ mod tests { fn resources() { wrapper(|env| { let frontend = env.frontend(); + let metrics = env.metrics(); let routes = [ "/style.css", @@ -336,18 +157,25 @@ mod tests { ]; for route in routes.iter() { - reset_records(); - frontend.get(route).send()?; frontend.get(route).send()?; - - assert_eq!(ROUTES_VISITED.load(Ordering::SeqCst), 2); - assert_eq!( - RESPONSE_TIMES.lock().unwrap().get("static resource"), - Some(&2) - ); } + assert_eq!( + metrics + .routes_visited + .with_label_values(&["static resource"]) + .get(), + 12 + ); + assert_eq!( + metrics + .response_time + .with_label_values(&["static resource"]) + .get_sample_count(), + 12 + ); + Ok(()) }) } @@ -367,6 +195,7 @@ mod tests { .create()?; let frontend = env.frontend(); + let metrics = env.metrics(); let routes = [ ("/releases", "/releases"), @@ -381,13 +210,20 @@ mod tests { ]; for (route, correct) in routes.iter() { - reset_records(); - frontend.get(route).send()?; frontend.get(route).send()?; - assert_eq!(ROUTES_VISITED.load(Ordering::SeqCst), 2); - assert_eq!(RESPONSE_TIMES.lock().unwrap().get(*correct), Some(&2)); + assert_eq!( + metrics.routes_visited.with_label_values(&[*correct]).get(), + 2 + ); + assert_eq!( + metrics + .response_time + .with_label_values(&[*correct]) + .get_sample_count(), + 2 + ); } Ok(()) @@ -404,22 +240,30 @@ mod tests { .create()?; let frontend = env.frontend(); + let metrics = env.metrics(); let routes = ["/crate/rcc/0.0.0", "/crate/hexponent/0.2.0"]; for route in routes.iter() { - reset_records(); - frontend.get(route).send()?; frontend.get(route).send()?; - - assert_eq!(ROUTES_VISITED.load(Ordering::SeqCst), 2); - assert_eq!( - RESPONSE_TIMES.lock().unwrap().get("/crate/:name/:version"), - Some(&2) - ); } + assert_eq!( + metrics + .routes_visited + .with_label_values(&["/crate/:name/:version"]) + .get(), + 4 + ); + assert_eq!( + metrics + .response_time + .with_label_values(&["/crate/:name/:version"]) + .get_sample_count(), + 4 + ); + Ok(()) }) } diff --git a/src/web/mod.rs b/src/web/mod.rs index 3b31b3d37..a46867cd7 100644 --- a/src/web/mod.rs +++ b/src/web/mod.rs @@ -403,14 +403,6 @@ impl Server { template_data: Arc, context: &dyn Context, ) -> Result { - // poke all the metrics counters to instantiate and register them - metrics::TOTAL_BUILDS.inc_by(0); - metrics::SUCCESSFUL_BUILDS.inc_by(0); - metrics::FAILED_BUILDS.inc_by(0); - metrics::NON_LIBRARY_BUILDS.inc_by(0); - metrics::UPLOADED_FILES_TOTAL.inc_by(0); - metrics::FAILED_DB_CONNECTIONS.inc_by(0); - let cratesfyi = CratesfyiHandler::new(template_data, context)?; let inner = Iron::new(cratesfyi) .http(addr) diff --git a/src/web/rustdoc.rs b/src/web/rustdoc.rs index 666a64089..d5cb0de22 100644 --- a/src/web/rustdoc.rs +++ b/src/web/rustdoc.rs @@ -4,10 +4,10 @@ use crate::{ db::Pool, utils, web::{ - crate_details::CrateDetails, error::Nope, file::File, match_version, metrics, - redirect_base, MatchSemver, + crate_details::CrateDetails, error::Nope, file::File, match_version, + metrics::RenderingTimesRecorder, redirect_base, MatchSemver, }, - Config, Storage, + Config, Metrics, Storage, }; use iron::{ headers::{CacheControl, CacheDirective, Expires, HttpDate}, @@ -207,13 +207,18 @@ impl RustdocPage { .get::() .expect("missing TemplateData from the request extensions"); + let metrics = req + .extensions + .get::() + .expect("missing Metrics from the request extensions"); + // Build the page of documentation let ctx = ctry!(req, tera::Context::from_serialize(self)); // Extract the head and body of the rustdoc file so that we can insert it into our own html // while logging OOM errors from html rewriting let html = match utils::rewrite_lol(rustdoc_html, max_parse_memory, ctx, templates) { Err(RewritingError::MemoryLimitExceeded(..)) => { - crate::web::metrics::HTML_REWRITE_OOMS.inc(); + metrics.html_rewrite_ooms.inc(); let config = extension!(req, Config); let err = failure::err_msg(format!( @@ -238,8 +243,8 @@ impl RustdocPage { /// This includes all HTML files for an individual crate, as well as the `search-index.js`, which is /// also crate-specific. pub fn rustdoc_html_server_handler(req: &mut Request) -> IronResult { - let mut rendering_time = - metrics::RenderingTimesRecorder::new(&metrics::RUSTDOC_RENDERING_TIMES); + let metrics = extension!(req, Metrics).clone(); + let mut rendering_time = RenderingTimesRecorder::new(&metrics.rustdoc_rendering_times); // Get the request parameters let router = extension!(req, Router); From f23e41c9d14977071fc70363a0a2fc283d6803a7 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Sun, 9 Aug 2020 13:00:29 +0200 Subject: [PATCH 040/139] storage: ensure uploaded_files_total metric is recorded --- src/storage/database.rs | 12 ++++++++++-- src/storage/mod.rs | 38 ++++++++++++++++++++++++++++++-------- src/test/mod.rs | 2 +- 3 files changed, 41 insertions(+), 11 deletions(-) diff --git a/src/storage/database.rs b/src/storage/database.rs index 9d6343607..6084f4306 100644 --- a/src/storage/database.rs +++ b/src/storage/database.rs @@ -1,16 +1,19 @@ use super::{Blob, StorageTransaction}; use crate::db::Pool; +use crate::Metrics; use chrono::{DateTime, NaiveDateTime, Utc}; use failure::Error; use postgres::Transaction; +use std::sync::Arc; pub(crate) struct DatabaseBackend { pool: Pool, + metrics: Arc, } impl DatabaseBackend { - pub(crate) fn new(pool: Pool) -> Self { - Self { pool } + pub(crate) fn new(pool: Pool, metrics: Arc) -> Self { + Self { pool, metrics } } pub(super) fn exists(&self, path: &str) -> Result { @@ -68,12 +71,14 @@ impl DatabaseBackend { pub(super) fn start_connection(&self) -> Result { Ok(DatabaseClient { conn: self.pool.get()?, + metrics: self.metrics.clone(), }) } } pub(super) struct DatabaseClient { conn: crate::db::PoolClient, + metrics: Arc, } impl DatabaseClient { @@ -82,12 +87,14 @@ impl DatabaseClient { ) -> Result, Error> { Ok(DatabaseStorageTransaction { transaction: self.conn.transaction()?, + metrics: &self.metrics, }) } } pub(super) struct DatabaseStorageTransaction<'a> { transaction: Transaction<'a>, + metrics: &'a Metrics, } impl<'a> StorageTransaction for DatabaseStorageTransaction<'a> { @@ -101,6 +108,7 @@ impl<'a> StorageTransaction for DatabaseStorageTransaction<'a> { SET mime = EXCLUDED.mime, content = EXCLUDED.content, compression = EXCLUDED.compression", &[&blob.path, &blob.mime, &blob.content, &compression], )?; + self.metrics.uploaded_files_total.inc(); } Ok(()) } diff --git a/src/storage/mod.rs b/src/storage/mod.rs index 5ded2d9fa..8e9939ec6 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -81,7 +81,7 @@ impl Storage { let backend = if let Some(c) = s3::s3_client() { StorageBackend::S3(Box::new(S3Backend::new(c, metrics, config)?)) } else { - StorageBackend::Database(DatabaseBackend::new(pool)) + StorageBackend::Database(DatabaseBackend::new(pool, metrics)) }; Ok(Storage { backend }) } @@ -98,9 +98,9 @@ impl Storage { } #[cfg(test)] - pub(crate) fn temp_new_db(pool: Pool) -> Result { + pub(crate) fn temp_new_db(pool: Pool, metrics: Arc) -> Result { Ok(Storage { - backend: StorageBackend::Database(DatabaseBackend::new(pool)), + backend: StorageBackend::Database(DatabaseBackend::new(pool, metrics)), }) } @@ -391,7 +391,7 @@ mod backend_tests { Ok(()) } - fn test_store_blobs(storage: &Storage) -> Result<(), Error> { + fn test_store_blobs(storage: &Storage, metrics: &Metrics) -> Result<(), Error> { const NAMES: &[&str] = &[ "a", "b", @@ -419,10 +419,12 @@ mod backend_tests { assert_eq!(blob.mime, actual.mime); } + assert_eq!(NAMES.len(), metrics.uploaded_files_total.get() as usize); + Ok(()) } - fn test_store_all(storage: &Storage) -> Result<(), Error> { + fn test_store_all(storage: &Storage, metrics: &Metrics) -> Result<(), Error> { let dir = tempfile::Builder::new() .prefix("docs.rs-upload-test") .tempdir()?; @@ -464,6 +466,8 @@ mod backend_tests { expected_algs.insert(CompressionAlgorithm::default()); assert_eq!(algs, expected_algs); + assert_eq!(2, metrics.uploaded_files_total.get()); + Ok(()) } @@ -559,7 +563,11 @@ mod backend_tests { // Remember to add the test name to the macro below when adding a new one. macro_rules! backend_tests { - (backends($env:ident) { $($backend:ident => $create:expr,)* } tests $tests:tt ) => { + ( + backends($env:ident) { $($backend:ident => $create:expr,)* } + tests $tests:tt + tests_with_metrics $tests_with_metrics:tt + ) => { $( mod $backend { use crate::test::TestEnvironment; @@ -571,6 +579,7 @@ mod backend_tests { } backend_tests!(@tests $tests); + backend_tests!(@tests_with_metrics $tests_with_metrics); } )* }; @@ -584,6 +593,16 @@ mod backend_tests { } )* }; + (@tests_with_metrics { $($test:ident,)* }) => { + $( + #[test] + fn $test() { + crate::test::wrapper(|env| { + super::$test(&*get_storage(env), &*env.metrics()) + }); + } + )* + }; } backend_tests! { @@ -597,10 +616,13 @@ mod backend_tests { test_exists, test_get_object, test_get_too_big, - test_store_blobs, - test_store_all, test_delete_prefix, test_delete_percent, } + + tests_with_metrics { + test_store_blobs, + test_store_all, + } } } diff --git a/src/test/mod.rs b/src/test/mod.rs index c1b9d5462..2a6d7cdfb 100644 --- a/src/test/mod.rs +++ b/src/test/mod.rs @@ -218,7 +218,7 @@ impl TestEnvironment { self.storage_db .get_or_init(|| { Arc::new( - Storage::temp_new_db(self.db().pool()) + Storage::temp_new_db(self.db().pool(), self.metrics()) .expect("failed to initialize the storage"), ) }) From 13e68cafa3c426b1636d101e9aca72248db7160a Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Sun, 9 Aug 2020 13:09:11 +0200 Subject: [PATCH 041/139] web: merge metrics tests --- src/web/metrics.rs | 168 +++++++++++---------------------------------- 1 file changed, 41 insertions(+), 127 deletions(-) diff --git a/src/web/metrics.rs b/src/web/metrics.rs index 14a1c54f1..44c26358d 100644 --- a/src/web/metrics.rs +++ b/src/web/metrics.rs @@ -108,80 +108,32 @@ impl Drop for RenderingTimesRecorder<'_> { #[cfg(test)] mod tests { use crate::test::{assert_success, wrapper}; + use std::collections::HashMap; #[test] - fn home_page() { - wrapper(|env| { - let frontend = env.frontend(); - let metrics = env.metrics(); - - frontend.get("/").send()?; - frontend.get("/").send()?; - assert_eq!(metrics.routes_visited.with_label_values(&["/"]).get(), 2); - assert_eq!( - metrics - .response_time - .with_label_values(&["/"]) - .get_sample_count(), - 2 - ); - - frontend.get("").send()?; - frontend.get("").send()?; - assert_eq!(metrics.routes_visited.with_label_values(&["/"]).get(), 4); - assert_eq!( - metrics - .response_time - .with_label_values(&["/"]) - .get_sample_count(), - 4 - ); - - Ok(()) - }) - } + fn test_response_times_count_being_collected() { + const ROUTES: &[(&str, &str)] = &[ + ("", "/"), + ("/", "/"), + ("/crate/hexponent/0.2.0", "/crate/:name/:version"), + ("/crate/rcc/0.0.0", "/crate/:name/:version"), + ("/index.js", "static resource"), + ("/menu.js", "static resource"), + ("/opensearch.xml", "static resource"), + ("/releases", "/releases"), + ("/releases/feed", "static resource"), + ("/releases/queue", "/releases/queue"), + ("/releases/recent-failures", "/releases/recent-failures"), + ( + "/releases/recent-failures/1", + "/releases/recent-failures/:page", + ), + ("/releases/recent/1", "/releases/recent/:page"), + ("/robots.txt", "static resource"), + ("/sitemap.xml", "static resource"), + ("/style.css", "static resource"), + ]; - #[test] - fn resources() { - wrapper(|env| { - let frontend = env.frontend(); - let metrics = env.metrics(); - - let routes = [ - "/style.css", - "/index.js", - "/menu.js", - "/sitemap.xml", - "/opensearch.xml", - "/robots.txt", - ]; - - for route in routes.iter() { - frontend.get(route).send()?; - frontend.get(route).send()?; - } - - assert_eq!( - metrics - .routes_visited - .with_label_values(&["static resource"]) - .get(), - 12 - ); - assert_eq!( - metrics - .response_time - .with_label_values(&["static resource"]) - .get_sample_count(), - 12 - ); - - Ok(()) - }) - } - - #[test] - fn releases() { wrapper(|env| { env.fake_release() .name("rcc") @@ -193,36 +145,36 @@ mod tests { .version("1.0.0") .build_result_successful(false) .create()?; + env.fake_release() + .name("hexponent") + .version("0.2.0") + .create()?; let frontend = env.frontend(); let metrics = env.metrics(); - let routes = [ - ("/releases", "/releases"), - ("/releases/recent/1", "/releases/recent/:page"), - ("/releases/feed", "static resource"), - ("/releases/queue", "/releases/queue"), - ("/releases/recent-failures", "/releases/recent-failures"), - ( - "/releases/recent-failures/1", - "/releases/recent-failures/:page", - ), - ]; - - for (route, correct) in routes.iter() { + for (route, _) in ROUTES.iter() { frontend.get(route).send()?; frontend.get(route).send()?; + } + let mut expected = HashMap::new(); + for (_, correct) in ROUTES.iter() { + let entry = expected.entry(*correct).or_insert(0); + *entry += 2; + } + + for (label, count) in expected.iter() { assert_eq!( - metrics.routes_visited.with_label_values(&[*correct]).get(), - 2 + metrics.routes_visited.with_label_values(&[*label]).get(), + *count ); assert_eq!( metrics .response_time - .with_label_values(&[*correct]) + .with_label_values(&[*label]) .get_sample_count(), - 2 + *count as u64 ); } @@ -231,45 +183,7 @@ mod tests { } #[test] - fn crates() { - wrapper(|env| { - env.fake_release().name("rcc").version("0.0.0").create()?; - env.fake_release() - .name("hexponent") - .version("0.2.0") - .create()?; - - let frontend = env.frontend(); - let metrics = env.metrics(); - - let routes = ["/crate/rcc/0.0.0", "/crate/hexponent/0.2.0"]; - - for route in routes.iter() { - frontend.get(route).send()?; - frontend.get(route).send()?; - } - - assert_eq!( - metrics - .routes_visited - .with_label_values(&["/crate/:name/:version"]) - .get(), - 4 - ); - assert_eq!( - metrics - .response_time - .with_label_values(&["/crate/:name/:version"]) - .get_sample_count(), - 4 - ); - - Ok(()) - }) - } - - #[test] - fn metrics() { + fn test_metrics_page_success() { wrapper(|env| { let web = env.frontend(); assert_success("/about/metrics", web) From 083f5d35096144851edb3adb0b0dabbe2b6bab56 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Sun, 9 Aug 2020 13:12:51 +0200 Subject: [PATCH 042/139] ci: run tests in parallel --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d0c2ce804..20f544924 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,7 +61,7 @@ jobs: run: cargo build --locked - name: Test - run: cargo test --locked -- --test-threads=1 + run: cargo test --locked - name: Clean up the database run: docker-compose down --volumes From 9da332f7af7e484253331409c2ce28990af22f82 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Sun, 9 Aug 2020 20:57:56 +0200 Subject: [PATCH 043/139] metrics: address review comments --- src/bin/cratesfyi.rs | 9 ++------- src/db/mod.rs | 3 +-- src/db/pool.rs | 2 +- src/metrics/macros.rs | 17 ++++++++++------- src/metrics/mod.rs | 2 +- 5 files changed, 15 insertions(+), 18 deletions(-) diff --git a/src/bin/cratesfyi.rs b/src/bin/cratesfyi.rs index 63578b354..a22d122c0 100644 --- a/src/bin/cratesfyi.rs +++ b/src/bin/cratesfyi.rs @@ -3,7 +3,7 @@ use std::fmt::Write; use std::path::PathBuf; use std::sync::Arc; -use cratesfyi::db::{self, add_path_into_database, Pool}; +use cratesfyi::db::{self, add_path_into_database, Pool, PoolClient}; use cratesfyi::index::Index; use cratesfyi::utils::{remove_crate_priority, set_crate_priority}; use cratesfyi::{ @@ -565,12 +565,7 @@ impl BinContext { } } - fn conn( - &self, - ) -> Result< - r2d2::PooledConnection>, - Error, - > { + fn conn(&self) -> Result { Ok(self.pool()?.get()?) } } diff --git a/src/db/mod.rs b/src/db/mod.rs index c506ada11..ecb326404 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -7,8 +7,7 @@ pub(crate) use self::add_package::{ pub use self::delete::{delete_crate, delete_version}; pub use self::file::add_path_into_database; pub use self::migrate::migrate; -pub(crate) use self::pool::PoolClient; -pub use self::pool::{Pool, PoolError}; +pub use self::pool::{Pool, PoolClient, PoolError}; mod add_package; pub mod blacklist; diff --git a/src/db/pool.rs b/src/db/pool.rs index acbc8bf5a..69abd697c 100644 --- a/src/db/pool.rs +++ b/src/db/pool.rs @@ -4,7 +4,7 @@ use postgres::{Client, NoTls}; use r2d2_postgres::PostgresConnectionManager; use std::sync::Arc; -pub(crate) type PoolClient = r2d2::PooledConnection>; +pub type PoolClient = r2d2::PooledConnection>; const DEFAULT_SCHEMA: &str = "public"; diff --git a/src/metrics/macros.rs b/src/metrics/macros.rs index c79bd60df..24d9055a5 100644 --- a/src/metrics/macros.rs +++ b/src/metrics/macros.rs @@ -6,21 +6,24 @@ pub(super) trait MetricFromOpts: Sized { macro_rules! metrics { ( $vis:vis struct $name:ident { - $(#[doc = $help:expr] $metric:ident: $ty:ty $([$($label:expr),*])?,)* + $( + #[doc = $help:expr] + $metric:ident: $ty:ty $([$($label:expr),* $(,)?])? + ),* $(,)? } metrics visibility: $metric_vis:vis, namespace: $namespace:expr, ) => { $vis struct $name { - registry: Registry, + registry: prometheus::Registry, $($metric_vis $metric: $ty,)* } impl $name { - $vis fn new() -> Result { - let registry = Registry::new(); + $vis fn new() -> Result { + let registry = prometheus::Registry::new(); $( let $metric = <$ty>::from_opts( - Opts::new(stringify!($metric), $help) + prometheus::Opts::new(stringify!($metric), $help) .namespace($namespace) $(.variable_labels(vec![$($label.into()),*]))? )?; @@ -42,7 +45,7 @@ macro_rules! load_metric_type { ($name:ident as single) => { use prometheus::$name; impl MetricFromOpts for $name { - fn from_opts(opts: Opts) -> Result { + fn from_opts(opts: prometheus::Opts) -> Result { $name::with_opts(opts) } } @@ -50,7 +53,7 @@ macro_rules! load_metric_type { ($name:ident as vec) => { use prometheus::$name; impl MetricFromOpts for $name { - fn from_opts(opts: Opts) -> Result { + fn from_opts(opts: prometheus::Opts) -> Result { $name::new( opts.clone().into(), opts.variable_labels diff --git a/src/metrics/mod.rs b/src/metrics/mod.rs index 870109a7e..9994ae329 100644 --- a/src/metrics/mod.rs +++ b/src/metrics/mod.rs @@ -5,7 +5,7 @@ use self::macros::MetricFromOpts; use crate::db::Pool; use crate::BuildQueue; use failure::Error; -use prometheus::{proto::MetricFamily, Opts, Registry}; +use prometheus::proto::MetricFamily; load_metric_type!(IntGauge as single); load_metric_type!(IntCounter as single); From 39f7db6657cf81a6380834fb079b17a0a587fb8c Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Sun, 9 Aug 2020 20:59:59 +0200 Subject: [PATCH 044/139] metrics: make visibility explicit --- src/metrics/macros.rs | 3 +-- src/metrics/mod.rs | 21 ++++++++++----------- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/src/metrics/macros.rs b/src/metrics/macros.rs index 24d9055a5..1ed94b1fe 100644 --- a/src/metrics/macros.rs +++ b/src/metrics/macros.rs @@ -8,10 +8,9 @@ macro_rules! metrics { $vis:vis struct $name:ident { $( #[doc = $help:expr] - $metric:ident: $ty:ty $([$($label:expr),* $(,)?])? + $metric_vis:vis $metric:ident: $ty:ty $([$($label:expr),* $(,)?])? ),* $(,)? } - metrics visibility: $metric_vis:vis, namespace: $namespace:expr, ) => { $vis struct $name { diff --git a/src/metrics/mod.rs b/src/metrics/mod.rs index 9994ae329..b41d98dda 100644 --- a/src/metrics/mod.rs +++ b/src/metrics/mod.rs @@ -28,7 +28,7 @@ metrics! { /// The maximum number of database connections max_db_connections: IntGauge, /// Number of attempted and failed connections to the database - failed_db_connections: IntCounter, + pub(crate) failed_db_connections: IntCounter, /// The number of currently opened file descriptors open_file_descriptors: IntGauge, @@ -36,29 +36,28 @@ metrics! { running_threads: IntGauge, /// The traffic of various docs.rs routes - routes_visited: IntCounterVec["route"], + pub(crate) routes_visited: IntCounterVec["route"], /// The response times of various docs.rs routes - response_time: HistogramVec["route"], + pub(crate) response_time: HistogramVec["route"], /// The time it takes to render a rustdoc page - rustdoc_rendering_times: HistogramVec["step"], + pub(crate) rustdoc_rendering_times: HistogramVec["step"], /// Number of crates built - total_builds: IntCounter, + pub(crate) total_builds: IntCounter, /// Number of builds that successfully generated docs - successful_builds: IntCounter, + pub(crate) successful_builds: IntCounter, /// Number of builds that generated a compiler error - failed_builds: IntCounter, + pub(crate) failed_builds: IntCounter, /// Number of builds that did not complete due to not being a library - non_library_builds: IntCounter, + pub(crate) non_library_builds: IntCounter, /// Number of files uploaded to the storage backend - uploaded_files_total: IntCounter, + pub(crate) uploaded_files_total: IntCounter, /// The number of attempted files that failed due to a memory limit - html_rewrite_ooms: IntCounter, + pub(crate) html_rewrite_ooms: IntCounter, } - metrics visibility: pub(crate), namespace: "docsrs", } From 13827d643b69c4ffeab6abb27f89abefa35f195f Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Sun, 9 Aug 2020 21:01:52 +0200 Subject: [PATCH 045/139] metrics: add comment explaining the namespace --- src/metrics/mod.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/metrics/mod.rs b/src/metrics/mod.rs index b41d98dda..6914168ce 100644 --- a/src/metrics/mod.rs +++ b/src/metrics/mod.rs @@ -58,6 +58,10 @@ metrics! { pub(crate) html_rewrite_ooms: IntCounter, } + // The Rust prometheus library treats the namespace as the "prefix" of the metric name: a + // metric named `foo` with a prefix of `docsrs` will expose a metric called `docsrs_foo`. + // + // https://docs.rs/prometheus/0.9.0/prometheus/struct.Opts.html#structfield.namespace namespace: "docsrs", } From ba416f631c266745b137a1916ba5faaf8f20bf5a Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Thu, 13 Aug 2020 22:24:39 +0300 Subject: [PATCH 046/139] Rebased --- src/metrics/macros.rs | 16 ++++++++++++++-- src/metrics/mod.rs | 4 +++- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/metrics/macros.rs b/src/metrics/macros.rs index 1ed94b1fe..cf7ccd256 100644 --- a/src/metrics/macros.rs +++ b/src/metrics/macros.rs @@ -8,6 +8,7 @@ macro_rules! metrics { $vis:vis struct $name:ident { $( #[doc = $help:expr] + $(#[$meta:meta])* $metric_vis:vis $metric:ident: $ty:ty $([$($label:expr),* $(,)?])? ),* $(,)? } @@ -15,20 +16,31 @@ macro_rules! metrics { ) => { $vis struct $name { registry: prometheus::Registry, - $($metric_vis $metric: $ty,)* + $( + $(#[$meta])* + $metric_vis $metric: $ty, + )* } impl $name { $vis fn new() -> Result { let registry = prometheus::Registry::new(); $( + $(#[$meta])* let $metric = <$ty>::from_opts( prometheus::Opts::new(stringify!($metric), $help) .namespace($namespace) $(.variable_labels(vec![$($label.into()),*]))? )?; + $(#[$meta])* registry.register(Box::new($metric.clone()))?; )* - Ok(Self { registry, $($metric,)* }) + Ok(Self { + registry, + $( + $(#[$meta])* + $metric, + )* + }) } } impl std::fmt::Debug for $name { diff --git a/src/metrics/mod.rs b/src/metrics/mod.rs index 6914168ce..264ee9e79 100644 --- a/src/metrics/mod.rs +++ b/src/metrics/mod.rs @@ -31,8 +31,10 @@ metrics! { pub(crate) failed_db_connections: IntCounter, /// The number of currently opened file descriptors + #[cfg(linux)] open_file_descriptors: IntGauge, /// The number of threads being used by docs.rs + #[cfg(linux)] running_threads: IntGauge, /// The traffic of various docs.rs routes @@ -87,7 +89,7 @@ impl Metrics { #[cfg(not(linux))] fn gather_system_performance(&self) {} - #[cfg(target_os = "linux")] + #[cfg(linux)] fn gather_system_performance(&self) { use procfs::process::Process; From e54a21f8bc1b873b086821380c598197529dd24a Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Wed, 12 Aug 2020 23:22:06 +0200 Subject: [PATCH 047/139] metrics: switch to cfg(target_os = "linux") --- src/metrics/mod.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/metrics/mod.rs b/src/metrics/mod.rs index 264ee9e79..9f5eabebd 100644 --- a/src/metrics/mod.rs +++ b/src/metrics/mod.rs @@ -31,10 +31,10 @@ metrics! { pub(crate) failed_db_connections: IntCounter, /// The number of currently opened file descriptors - #[cfg(linux)] + #[cfg(target_os = "linux")] open_file_descriptors: IntGauge, /// The number of threads being used by docs.rs - #[cfg(linux)] + #[cfg(target_os = "linux")] running_threads: IntGauge, /// The traffic of various docs.rs routes @@ -86,10 +86,10 @@ impl Metrics { Ok(self.registry.gather()) } - #[cfg(not(linux))] + #[cfg(not(target_os = "linux"))] fn gather_system_performance(&self) {} - #[cfg(linux)] + #[cfg(target_os = "linux")] fn gather_system_performance(&self) { use procfs::process::Process; From 7266be332c6959900da86e5200f63e46b346d9de Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Wed, 12 Aug 2020 23:26:28 +0200 Subject: [PATCH 048/139] update the coverage styling in the dropdown --- templates/rustdoc/body.html | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/templates/rustdoc/body.html b/templates/rustdoc/body.html index eb5b87b62..17ce5cf03 100644 --- a/templates/rustdoc/body.html +++ b/templates/rustdoc/body.html @@ -117,21 +117,7 @@
    - {%- if krate.documented_items and krate.total_items -%} - {% set percent = krate.documented_items * 100 / krate.total_items %} -
    -
    -
      -
    • Coverage
      • -
    • {{ percent | round(precision=2) }} %
      - {{ krate.documented_items }} out of {{ krate.total_items }} items documented -
      • -
    -
    -
    - {%- endif -%} - -
    +
    • Dependencies
      • @@ -169,6 +155,21 @@
    + {%- if krate.documented_items and krate.total_items -%} + {% set percent = krate.documented_items * 100 / krate.total_items %} +
    +
    + +
    +
    + {%- endif -%}
    From f25e599451e57a952c3e4b7397abde6aa822f8ae Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Thu, 13 Aug 2020 14:23:09 +0200 Subject: [PATCH 049/139] coverage: improve tests --- src/test/fakes.rs | 21 +++++++++++--------- src/web/mod.rs | 37 ++++++++++++++++++------------------ templates/crate/details.html | 2 +- 3 files changed, 31 insertions(+), 29 deletions(-) diff --git a/src/test/fakes.rs b/src/test/fakes.rs index a3372a68d..15cc910c4 100644 --- a/src/test/fakes.rs +++ b/src/test/fakes.rs @@ -59,10 +59,7 @@ impl<'a> FakeRelease<'a> { docsrs_version: "docs.rs 1.0.0 (000000000 1970-01-01)".into(), build_log: "It works!".into(), successful: true, - doc_coverage: Some(DocCoverage { - total_items: 10, - documented_items: 6, - }), + doc_coverage: None, }, source_files: Vec::new(), rustdoc_files: Vec::new(), @@ -186,6 +183,14 @@ impl<'a> FakeRelease<'a> { self } + pub(crate) fn coverage(mut self, documented_items: i32, total_items: i32) -> Self { + self.build_result.doc_coverage = Some(DocCoverage { + total_items, + documented_items, + }); + self + } + /// Returns the release_id pub(crate) fn create(self) -> Result { use std::fs; @@ -277,11 +282,9 @@ impl<'a> FakeRelease<'a> { &self.registry_crate_data, )?; crate::db::add_build_into_database(&mut db.conn(), release_id, &self.build_result)?; - crate::db::add_doc_coverage( - &mut db.conn(), - release_id, - self.build_result.doc_coverage.clone().unwrap(), - )?; + if let Some(coverage) = self.build_result.doc_coverage { + crate::db::add_doc_coverage(&mut db.conn(), release_id, coverage)?; + } Ok(release_id) } diff --git a/src/web/mod.rs b/src/web/mod.rs index a46867cd7..c81e762ee 100644 --- a/src/web/mod.rs +++ b/src/web/mod.rs @@ -638,14 +638,6 @@ mod test { node.select("#clipboard").unwrap().count() == 1 } - fn check_doc_coverage_is_present_for_path(path: &str, web: &TestFrontend) -> bool { - let data = web.get(path).send().unwrap().text().unwrap(); - let node = kuchiki::parse_html().one(data); - node.select(".pure-menu-heading") - .unwrap() - .any(|e| e.text_contents().contains("Coverage")) - } - #[test] fn test_index_returns_success() { wrapper(|env| { @@ -659,20 +651,27 @@ mod test { fn test_doc_coverage_for_crate_pages() { wrapper(|env| { env.fake_release() - .name("fake_crate") + .name("foo") .version("0.0.1") .source_file("test.rs", &[]) - .create() - .unwrap(); + .coverage(6, 10) + .create()?; let web = env.frontend(); - assert!(check_doc_coverage_is_present_for_path( - "/crate/fake_crate/0.0.1", - web - )); - assert!(check_doc_coverage_is_present_for_path( - "/fake_crate/0.0.1/fake_crate", - web - )); + + let foo_crate = kuchiki::parse_html().one(web.get("/crate/foo/0.0.1").send()?.text()?); + for value in &["60%", "6", "10"] { + assert!(foo_crate + .select(".pure-menu-item b") + .unwrap() + .any(|e| e.text_contents().contains(value))); + } + + let foo_doc = kuchiki::parse_html().one(web.get("/foo/0.0.1/foo").send()?.text()?); + assert!(foo_doc + .select(".pure-menu-link b") + .unwrap() + .any(|e| e.text_contents().contains("60%"))); + Ok(()) }); } diff --git a/templates/crate/details.html b/templates/crate/details.html index d83b65ceb..4868474a9 100644 --- a/templates/crate/details.html +++ b/templates/crate/details.html @@ -19,7 +19,7 @@ {%- if details.documented_items and details.total_items -%} {% set percent = details.documented_items * 100 / details.total_items %}
  • Coverage
    • -
  • {{ percent | round(precision=2) }} %
    +
  • {{ percent | round(precision=2) }}%
    {{ details.documented_items }} out of {{ details.total_items }} items documented
    • {%- endif -%} From 4751bf231b8d711c3265d706c4fcf1c8fdad00a7 Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Thu, 13 Aug 2020 22:28:00 +0300 Subject: [PATCH 050/139] Rebased --- src/config.rs | 2 +- src/utils/daemon.rs | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/config.rs b/src/config.rs index a3ae7dc9f..d84f12872 100644 --- a/src/config.rs +++ b/src/config.rs @@ -30,7 +30,7 @@ pub struct Config { pub(crate) max_file_size_html: usize, // The most memory that can be used to parse an HTML file pub(crate) max_parse_memory: usize, - // Time between 'git gc' calls in seconds + // Time between 'git gc --auto' calls in seconds pub(crate) registry_gc_interval: u64, } diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs index 673f05204..af8436c94 100644 --- a/src/utils/daemon.rs +++ b/src/utils/daemon.rs @@ -22,7 +22,6 @@ fn start_registry_watcher(opts: DocBuilderOptions, context: &dyn Context) -> Res // space this out to prevent it from clashing against the queue-builder thread on launch thread::sleep(Duration::from_secs(30)); let mut last_gc = Instant::now(); - loop { let mut doc_builder = DocBuilder::new(opts.clone(), pool.clone(), build_queue.clone()); From 11930e0a31f5ba46b07810e0e347fe3702aac003 Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Thu, 13 Aug 2020 22:28:39 +0300 Subject: [PATCH 051/139] Rebased --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index d7823cbaf..5b7d9caab 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ target .vagrant .rustwide .rustwide-docker +.idea/ \ No newline at end of file From 80d2cbc32651aa30f12e627a22295a182d098dfa Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Thu, 13 Aug 2020 22:30:47 +0300 Subject: [PATCH 052/139] Rebased --- git_hooks/pre-commit | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100755 git_hooks/pre-commit diff --git a/git_hooks/pre-commit b/git_hooks/pre-commit new file mode 100755 index 000000000..8ae45b3f3 --- /dev/null +++ b/git_hooks/pre-commit @@ -0,0 +1,19 @@ +#!/usr/bin/env sh +cargo fmt -- --check +if [ "$?" != "0" ] ; then + echo "\n" + echo "\033[0;31mpre-commit hook failed during:\033[0m" + echo "\033[0;31m\tcargo fmt -- --check\033[0m" + exit 1 +fi +cargo clippy --locked -- -D warnings +if [ "$?" != "0" ] ; then + echo "\n" + echo "\033[0;31mpre-commit hook failed during:\033[0m" + echo "\033[0;31m\tclippy --locked -- -D warning\033[0m" + exit 1 +fi + +echo "\n" +echo "\033[0;32mpre-commit hook succeeded\033[0m" +exit 0 \ No newline at end of file From de82defe690f89e670a088e4b1f0fbf181ec382c Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Tue, 11 Aug 2020 22:09:34 +0300 Subject: [PATCH 053/139] Run git gc periodically on the crates.io index #778 --- src/config.rs | 3 ++- src/utils/daemon.rs | 26 ++++++++++++++++++++------ 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/src/config.rs b/src/config.rs index d84f12872..497ad575d 100644 --- a/src/config.rs +++ b/src/config.rs @@ -30,6 +30,7 @@ pub struct Config { pub(crate) max_file_size_html: usize, // The most memory that can be used to parse an HTML file pub(crate) max_parse_memory: usize, + // Time between 'git gc --auto' calls in seconds pub(crate) registry_gc_interval: u64, } @@ -63,7 +64,7 @@ impl Config { // LOL HTML only uses as much memory as the size of the start tag! // https://github.com/rust-lang/docs.rs/pull/930#issuecomment-667729380 max_parse_memory: env("DOCSRS_MAX_PARSE_MEMORY", 5 * 1024 * 1024)?, - registry_gc_interval: env("DOCSRS_REGISTRY_GC_INTERVAL", 60 * 60)?, + registry_gc_interval: env("REGISTRY_GC_INTERVAL", 60 * 60)?, }) } diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs index af8436c94..68276ec54 100644 --- a/src/utils/daemon.rs +++ b/src/utils/daemon.rs @@ -11,17 +11,30 @@ use failure::Error; use log::{debug, error, info}; use std::thread; use std::time::{Duration, Instant}; +use std::process::Command; -fn start_registry_watcher(opts: DocBuilderOptions, context: &dyn Context) -> Result<(), Error> { - let pool = context.pool()?; - let build_queue = context.build_queue()?; - let config = context.config()?; + +fn run_git_gc() { + Command::new("git") + .args(&["gc"]) + .output() + .expect("Failed to execute git gc"); +} + +fn start_registry_watcher( + opts: DocBuilderOptions, + pool: Pool, + build_queue: Arc, + config: Arc, +) -> Result<(), Error> { thread::Builder::new() .name("registry index reader".to_string()) .spawn(move || { // space this out to prevent it from clashing against the queue-builder thread on launch thread::sleep(Duration::from_secs(30)); let mut last_gc = Instant::now(); + run_git_gc(); + loop { let mut doc_builder = DocBuilder::new(opts.clone(), pool.clone(), build_queue.clone()); @@ -37,7 +50,7 @@ fn start_registry_watcher(opts: DocBuilderOptions, context: &dyn Context) -> Res } if last_gc.elapsed().as_secs() >= config.registry_gc_interval { - doc_builder.run_git_gc(); + run_git_gc(); last_gc = Instant::now(); } thread::sleep(Duration::from_secs(60)); @@ -56,7 +69,8 @@ pub fn start_daemon(context: &dyn Context, enable_registry_watcher: bool) -> Res if enable_registry_watcher { // check new crates every minute - start_registry_watcher(dbopts.clone(), context)?; + start_registry_watcher(dbopts.clone(), db.clone(), build_queue.clone(), + config.clone())?; } // build new crates every minute From 3440e52194169a21da429a8926b99b98af769fc0 Mon Sep 17 00:00:00 2001 From: ohaddahan Date: Tue, 11 Aug 2020 22:14:34 +0300 Subject: [PATCH 054/139] Update config.rs --- src/config.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config.rs b/src/config.rs index 497ad575d..871002745 100644 --- a/src/config.rs +++ b/src/config.rs @@ -31,7 +31,7 @@ pub struct Config { // The most memory that can be used to parse an HTML file pub(crate) max_parse_memory: usize, - // Time between 'git gc --auto' calls in seconds + // Time between 'git gc' calls in seconds pub(crate) registry_gc_interval: u64, } From a94cb903be19e51497f0fcec5d3ea11ec0364604 Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Tue, 11 Aug 2020 22:20:18 +0300 Subject: [PATCH 055/139] Fixed "cargo fmt" --- src/utils/daemon.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs index 68276ec54..bb086db06 100644 --- a/src/utils/daemon.rs +++ b/src/utils/daemon.rs @@ -9,10 +9,10 @@ use crate::{ use chrono::{Timelike, Utc}; use failure::Error; use log::{debug, error, info}; +use std::process::Command; +use std::sync::Arc; use std::thread; use std::time::{Duration, Instant}; -use std::process::Command; - fn run_git_gc() { Command::new("git") @@ -69,8 +69,12 @@ pub fn start_daemon(context: &dyn Context, enable_registry_watcher: bool) -> Res if enable_registry_watcher { // check new crates every minute - start_registry_watcher(dbopts.clone(), db.clone(), build_queue.clone(), - config.clone())?; + start_registry_watcher( + dbopts.clone(), + db.clone(), + build_queue.clone(), + config.clone(), + )?; } // build new crates every minute From fcf33023be6bbe79f50053f1a520701ffba4bc83 Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Wed, 12 Aug 2020 09:01:44 +0300 Subject: [PATCH 056/139] Added --auto --- src/utils/daemon.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs index bb086db06..c993d4c16 100644 --- a/src/utils/daemon.rs +++ b/src/utils/daemon.rs @@ -16,7 +16,7 @@ use std::time::{Duration, Instant}; fn run_git_gc() { Command::new("git") - .args(&["gc"]) + .args(&["gc", "--auto"]) .output() .expect("Failed to execute git gc"); } From 819d32b8e58ab77f0ac6c2dbab43ec4a73dc457d Mon Sep 17 00:00:00 2001 From: ohaddahan Date: Tue, 11 Aug 2020 23:22:02 +0300 Subject: [PATCH 057/139] Update src/utils/daemon.rs Co-authored-by: Chase Wilson --- src/utils/daemon.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs index c993d4c16..db70b1cad 100644 --- a/src/utils/daemon.rs +++ b/src/utils/daemon.rs @@ -32,8 +32,8 @@ fn start_registry_watcher( .spawn(move || { // space this out to prevent it from clashing against the queue-builder thread on launch thread::sleep(Duration::from_secs(30)); - let mut last_gc = Instant::now(); run_git_gc(); + let mut last_gc = Instant::now(); loop { let mut doc_builder = From eb477856ea2158a12e7f16dc81dbb9116532e316 Mon Sep 17 00:00:00 2001 From: ohaddahan Date: Tue, 11 Aug 2020 23:22:11 +0300 Subject: [PATCH 058/139] Update src/utils/daemon.rs Co-authored-by: Chase Wilson --- src/utils/daemon.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs index db70b1cad..de8700fc5 100644 --- a/src/utils/daemon.rs +++ b/src/utils/daemon.rs @@ -15,10 +15,13 @@ use std::thread; use std::time::{Duration, Instant}; fn run_git_gc() { - Command::new("git") - .args(&["gc", "--auto"]) - .output() - .expect("Failed to execute git gc"); + let gc = Command::new("git") + .args(&["gc"]) + .output(); + + if let Err(err) = gc { + log::error!("failed to run `git gc`: {:?}", err); + } } fn start_registry_watcher( From 8a4eec3ffd7a53f19ca5f0c2bd63883d2f101124 Mon Sep 17 00:00:00 2001 From: ohaddahan Date: Tue, 11 Aug 2020 23:22:27 +0300 Subject: [PATCH 059/139] Update src/utils/daemon.rs Co-authored-by: Chase Wilson --- src/utils/daemon.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs index de8700fc5..299e3a88f 100644 --- a/src/utils/daemon.rs +++ b/src/utils/daemon.rs @@ -16,7 +16,7 @@ use std::time::{Duration, Instant}; fn run_git_gc() { let gc = Command::new("git") - .args(&["gc"]) + .args(&["gc", "--auto"]) .output(); if let Err(err) = gc { From ffc3d9e4ddf1a703b4847dbe1aa3fd79148eca36 Mon Sep 17 00:00:00 2001 From: ohaddahan Date: Tue, 11 Aug 2020 23:22:36 +0300 Subject: [PATCH 060/139] Update src/config.rs Co-authored-by: Chase Wilson --- src/config.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/config.rs b/src/config.rs index 871002745..86fc11db5 100644 --- a/src/config.rs +++ b/src/config.rs @@ -30,7 +30,6 @@ pub struct Config { pub(crate) max_file_size_html: usize, // The most memory that can be used to parse an HTML file pub(crate) max_parse_memory: usize, - // Time between 'git gc' calls in seconds pub(crate) registry_gc_interval: u64, } From c8744feb717aaecb0eeceb9e58b80cd8e4d32ed1 Mon Sep 17 00:00:00 2001 From: ohaddahan Date: Tue, 11 Aug 2020 23:22:43 +0300 Subject: [PATCH 061/139] Update .gitignore Co-authored-by: Chase Wilson --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index 5b7d9caab..d7823cbaf 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,3 @@ target .vagrant .rustwide .rustwide-docker -.idea/ \ No newline at end of file From a9f04c4619c04ff4195453f135847d2412ae193e Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Wed, 12 Aug 2020 09:11:54 +0300 Subject: [PATCH 062/139] Fix cargo fmt --- src/utils/daemon.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs index 299e3a88f..071ba3154 100644 --- a/src/utils/daemon.rs +++ b/src/utils/daemon.rs @@ -15,10 +15,8 @@ use std::thread; use std::time::{Duration, Instant}; fn run_git_gc() { - let gc = Command::new("git") - .args(&["gc", "--auto"]) - .output(); - + let gc = Command::new("git").args(&["gc", "--auto"]).output(); + if let Err(err) = gc { log::error!("failed to run `git gc`: {:?}", err); } From e8daa7652143be9c7dd47a6080ff9dcf28b7555c Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Wed, 12 Aug 2020 09:51:30 +0300 Subject: [PATCH 063/139] Adding pre-commit hook, adding dead-code lint since without it clippy rails with "unused static" --- README.md | 2 +- src/web/metrics.rs | 146 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 147 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index c84156f16..53775185d 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ This gives you reasonable incremental build times without having to add new user ### Git Hooks For ease of use, `git_hooks` directory contains useful `git hooks` to make your development easier. ```bash -cd .git/hooks && ln -s ../../.git_hooks/* . && cd ../.. +cp git_hooks/* .git/hooks/ ``` ### Dependencies diff --git a/src/web/metrics.rs b/src/web/metrics.rs index 44c26358d..c87f1d700 100644 --- a/src/web/metrics.rs +++ b/src/web/metrics.rs @@ -7,6 +7,152 @@ use iron::status::Status; use prometheus::{Encoder, HistogramVec, TextEncoder}; use std::time::{Duration, Instant}; +static QUEUED_CRATES_COUNT: Lazy = Lazy::new(|| { + register_int_gauge!( + "docsrs_queued_crates_count", + "Number of crates in the build queue" + ) + .unwrap() +}); + +pub static PRIORITIZED_CRATES_COUNT: Lazy = Lazy::new(|| { + register_int_gauge!( + "docsrs_prioritized_crates_count", + "Number of crates in the build queue that have a positive priority" + ) + .unwrap() +}); + +static FAILED_CRATES_COUNT: Lazy = Lazy::new(|| { + register_int_gauge!( + "docsrs_failed_crates_count", + "Number of crates that failed to build" + ) + .unwrap() +}); + +pub static TOTAL_BUILDS: Lazy = + Lazy::new(|| register_int_counter!("docsrs_total_builds", "Number of crates built").unwrap()); + +pub static SUCCESSFUL_BUILDS: Lazy = Lazy::new(|| { + register_int_counter!( + "docsrs_successful_builds", + "Number of builds that successfully generated docs" + ) + .unwrap() +}); + +pub static FAILED_BUILDS: Lazy = Lazy::new(|| { + register_int_counter!( + "docsrs_failed_builds", + "Number of builds that generated a compile error" + ) + .unwrap() +}); + +pub static NON_LIBRARY_BUILDS: Lazy = Lazy::new(|| { + register_int_counter!( + "docsrs_non_library_builds", + "Number of builds that did not complete due to not being a library" + ) + .unwrap() +}); + +pub static UPLOADED_FILES_TOTAL: Lazy = Lazy::new(|| { + register_int_counter!( + "docsrs_uploaded_files_total", + "Number of files uploaded to S3 or stored in the database" + ) + .unwrap() +}); + +pub static ROUTES_VISITED: Lazy = Lazy::new(|| { + register_int_counter_vec!( + "docsrs_routes_visited", + "The traffic of various docs.rs routes", + &["route"] + ) + .unwrap() +}); + +pub static RESPONSE_TIMES: Lazy = Lazy::new(|| { + register_histogram_vec!( + "docsrs_response_time", + "The response times of various docs.rs routes", + &["route"] + ) + .unwrap() +}); + +pub static RUSTDOC_RENDERING_TIMES: Lazy = Lazy::new(|| { + register_histogram_vec!( + "docsrs_rustdoc_rendering_time", + "The time it takes to render a rustdoc page", + &["step"] + ) + .unwrap() +}); + +pub static FAILED_DB_CONNECTIONS: Lazy = Lazy::new(|| { + register_int_counter!( + "docsrs_failed_db_connections", + "Number of attempted and failed connections to the database" + ) + .unwrap() +}); + +pub static USED_DB_CONNECTIONS: Lazy = Lazy::new(|| { + register_int_gauge!( + "docsrs_used_db_connections", + "The number of used database connections" + ) + .unwrap() +}); + +pub static IDLE_DB_CONNECTIONS: Lazy = Lazy::new(|| { + register_int_gauge!( + "docsrs_idle_db_connections", + "The number of idle database connections" + ) + .unwrap() +}); + +pub static MAX_DB_CONNECTIONS: Lazy = Lazy::new(|| { + register_int_gauge!( + "docsrs_max_db_connections", + "The maximum database connections" + ) + .unwrap() +}); + +#[cfg(not(windows))] +#[allow(dead_code)] +pub static OPEN_FILE_DESCRIPTORS: Lazy = Lazy::new(|| { + register_int_gauge!( + "docsrs_open_file_descriptors", + "The number of currently opened file descriptors" + ) + .unwrap() +}); + +#[cfg(not(windows))] +#[allow(dead_code)] +pub static CURRENTLY_RUNNING_THREADS: Lazy = Lazy::new(|| { + register_int_gauge!( + "docsrs_running_threads", + "The number of threads being used by docs.rs" + ) + .unwrap() +}); + +pub static HTML_REWRITE_OOMS: Lazy = Lazy::new(|| { + register_int_gauge!( + "docsrs_html_rewrite_ooms", + "The number of attempted files that failed due to a memory limit" + ) + .unwrap() +}); + pub fn metrics_handler(req: &mut Request) -> IronResult { let metrics = extension!(req, Metrics); let pool = extension!(req, Pool); From b622d569ac2a091e01594d3d3f3b973ba5bc1861 Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Wed, 12 Aug 2020 19:59:52 +0300 Subject: [PATCH 064/139] Changed envar prefix and replace echo with printf for pre-commit hook --- git_hooks/pre-commit | 16 ++++++++-------- src/config.rs | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/git_hooks/pre-commit b/git_hooks/pre-commit index 8ae45b3f3..f6855e88e 100755 --- a/git_hooks/pre-commit +++ b/git_hooks/pre-commit @@ -1,19 +1,19 @@ #!/usr/bin/env sh cargo fmt -- --check if [ "$?" != "0" ] ; then - echo "\n" - echo "\033[0;31mpre-commit hook failed during:\033[0m" - echo "\033[0;31m\tcargo fmt -- --check\033[0m" + printf "\n" + printf "\033[0;31mpre-commit hook failed during:\033[0m\n" + printf "\033[0;31m\tcargo fmt -- --check\033[0m\n" exit 1 fi cargo clippy --locked -- -D warnings if [ "$?" != "0" ] ; then - echo "\n" - echo "\033[0;31mpre-commit hook failed during:\033[0m" - echo "\033[0;31m\tclippy --locked -- -D warning\033[0m" + printf "\n" + printf "\033[0;31mpre-commit hook failed during:\033[0m\n" + printf "\033[0;31m\tclippy --locked -- -D warning\033[0m\n" exit 1 fi -echo "\n" -echo "\033[0;32mpre-commit hook succeeded\033[0m" +printf "\n" +printf "\033[0;32mpre-commit hook succeeded\033[0m\n" exit 0 \ No newline at end of file diff --git a/src/config.rs b/src/config.rs index 86fc11db5..a3ae7dc9f 100644 --- a/src/config.rs +++ b/src/config.rs @@ -63,7 +63,7 @@ impl Config { // LOL HTML only uses as much memory as the size of the start tag! // https://github.com/rust-lang/docs.rs/pull/930#issuecomment-667729380 max_parse_memory: env("DOCSRS_MAX_PARSE_MEMORY", 5 * 1024 * 1024)?, - registry_gc_interval: env("REGISTRY_GC_INTERVAL", 60 * 60)?, + registry_gc_interval: env("DOCSRS_REGISTRY_GC_INTERVAL", 60 * 60)?, }) } From 860c276505c84ef2b4f216eff1751865001b87dc Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Wed, 12 Aug 2020 20:33:36 +0300 Subject: [PATCH 065/139] Removed allow(dead_code) and not(windows) , now OPEN_FILE_DESCRIPTORS and CURRENTLY_RUNNING_THREADS declaration match the usage, "target_os = linux" --- src/web/metrics.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/web/metrics.rs b/src/web/metrics.rs index c87f1d700..d222f1924 100644 --- a/src/web/metrics.rs +++ b/src/web/metrics.rs @@ -125,8 +125,7 @@ pub static MAX_DB_CONNECTIONS: Lazy = Lazy::new(|| { .unwrap() }); -#[cfg(not(windows))] -#[allow(dead_code)] +#[cfg(target_os = "linux")] pub static OPEN_FILE_DESCRIPTORS: Lazy = Lazy::new(|| { register_int_gauge!( "docsrs_open_file_descriptors", @@ -135,8 +134,7 @@ pub static OPEN_FILE_DESCRIPTORS: Lazy = Lazy::new(|| { .unwrap() }); -#[cfg(not(windows))] -#[allow(dead_code)] +#[cfg(target_os = "linux")] pub static CURRENTLY_RUNNING_THREADS: Lazy = Lazy::new(|| { register_int_gauge!( "docsrs_running_threads", From ad575fa4862edbf1651073d793fb6ba65b12f82a Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Thu, 13 Aug 2020 15:57:22 +0300 Subject: [PATCH 066/139] Updated pre-commit script and instructions --- .git_hooks/pre-commit | 1 - README.md | 2 +- git_hooks/pre-commit | 19 ------------------- 3 files changed, 1 insertion(+), 21 deletions(-) delete mode 100755 git_hooks/pre-commit diff --git a/.git_hooks/pre-commit b/.git_hooks/pre-commit index 105684ef7..872c2cdda 100755 --- a/.git_hooks/pre-commit +++ b/.git_hooks/pre-commit @@ -16,4 +16,3 @@ fi printf "\n" printf "\033[0;32mpre-commit hook succeeded\033[0m\n" exit 0 - diff --git a/README.md b/README.md index 53775185d..c84156f16 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ This gives you reasonable incremental build times without having to add new user ### Git Hooks For ease of use, `git_hooks` directory contains useful `git hooks` to make your development easier. ```bash -cp git_hooks/* .git/hooks/ +cd .git/hooks && ln -s ../../.git_hooks/* . && cd ../.. ``` ### Dependencies diff --git a/git_hooks/pre-commit b/git_hooks/pre-commit deleted file mode 100755 index f6855e88e..000000000 --- a/git_hooks/pre-commit +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env sh -cargo fmt -- --check -if [ "$?" != "0" ] ; then - printf "\n" - printf "\033[0;31mpre-commit hook failed during:\033[0m\n" - printf "\033[0;31m\tcargo fmt -- --check\033[0m\n" - exit 1 -fi -cargo clippy --locked -- -D warnings -if [ "$?" != "0" ] ; then - printf "\n" - printf "\033[0;31mpre-commit hook failed during:\033[0m\n" - printf "\033[0;31m\tclippy --locked -- -D warning\033[0m\n" - exit 1 -fi - -printf "\n" -printf "\033[0;32mpre-commit hook succeeded\033[0m\n" -exit 0 \ No newline at end of file From 9897721e70837ee96e09b946aaff901e36eecb3a Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Thu, 13 Aug 2020 15:58:30 +0300 Subject: [PATCH 067/139] Added newline at end of pre-commit script --- .git_hooks/pre-commit | 1 + 1 file changed, 1 insertion(+) diff --git a/.git_hooks/pre-commit b/.git_hooks/pre-commit index 872c2cdda..105684ef7 100755 --- a/.git_hooks/pre-commit +++ b/.git_hooks/pre-commit @@ -16,3 +16,4 @@ fi printf "\n" printf "\033[0;32mpre-commit hook succeeded\033[0m\n" exit 0 + From 95b60617972aa20c91589ae5ad157b14fed9706e Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Thu, 13 Aug 2020 17:54:29 +0300 Subject: [PATCH 068/139] Move run_git_gc to Index --- src/index/mod.rs | 2 +- src/utils/daemon.rs | 12 +----------- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/src/index/mod.rs b/src/index/mod.rs index e00c2c357..a4fef6e0e 100644 --- a/src/index/mod.rs +++ b/src/index/mod.rs @@ -52,7 +52,7 @@ impl Index { } pub fn run_git_gc(&self) { - let cmd = format!("cd {} && git gc --auto", self.path.to_str().unwrap()); + let cmd = format!("cd {} && gc --auto", self.path.to_str().unwrap()); let gc = Command::new("sh").args(&["-c", cmd.as_str()]).output(); if let Err(err) = gc { log::error!("Failed to run `{}`: {:?}", cmd, err); diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs index 071ba3154..7b886eab8 100644 --- a/src/utils/daemon.rs +++ b/src/utils/daemon.rs @@ -9,19 +9,10 @@ use crate::{ use chrono::{Timelike, Utc}; use failure::Error; use log::{debug, error, info}; -use std::process::Command; use std::sync::Arc; use std::thread; use std::time::{Duration, Instant}; -fn run_git_gc() { - let gc = Command::new("git").args(&["gc", "--auto"]).output(); - - if let Err(err) = gc { - log::error!("failed to run `git gc`: {:?}", err); - } -} - fn start_registry_watcher( opts: DocBuilderOptions, pool: Pool, @@ -33,7 +24,6 @@ fn start_registry_watcher( .spawn(move || { // space this out to prevent it from clashing against the queue-builder thread on launch thread::sleep(Duration::from_secs(30)); - run_git_gc(); let mut last_gc = Instant::now(); loop { @@ -51,7 +41,7 @@ fn start_registry_watcher( } if last_gc.elapsed().as_secs() >= config.registry_gc_interval { - run_git_gc(); + doc_builder.run_git_gc(); last_gc = Instant::now(); } thread::sleep(Duration::from_secs(60)); From b53b87d9923cbf84643c0e161e4dd1a9c46ef991 Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Thu, 13 Aug 2020 19:48:32 +0300 Subject: [PATCH 069/139] Fix typo --- src/index/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/index/mod.rs b/src/index/mod.rs index a4fef6e0e..e00c2c357 100644 --- a/src/index/mod.rs +++ b/src/index/mod.rs @@ -52,7 +52,7 @@ impl Index { } pub fn run_git_gc(&self) { - let cmd = format!("cd {} && gc --auto", self.path.to_str().unwrap()); + let cmd = format!("cd {} && git gc --auto", self.path.to_str().unwrap()); let gc = Command::new("sh").args(&["-c", cmd.as_str()]).output(); if let Err(err) = gc { log::error!("Failed to run `{}`: {:?}", cmd, err); From 618d9663b9fec29134171a1c2e5f120e6d604975 Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Thu, 13 Aug 2020 21:42:57 +0300 Subject: [PATCH 070/139] Rebased --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index d7823cbaf..5b7d9caab 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ target .vagrant .rustwide .rustwide-docker +.idea/ \ No newline at end of file From 4e7b8a6075ee9a6d51018bed603cf0a34d240383 Mon Sep 17 00:00:00 2001 From: ohaddahan Date: Tue, 11 Aug 2020 23:22:43 +0300 Subject: [PATCH 071/139] Update .gitignore Co-authored-by: Chase Wilson --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index 5b7d9caab..d7823cbaf 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,3 @@ target .vagrant .rustwide .rustwide-docker -.idea/ \ No newline at end of file From 0a0128fc77cc933a810d53df3704f90ca5ad0839 Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Thu, 13 Aug 2020 21:48:01 +0300 Subject: [PATCH 072/139] Rebased --- git_hooks/pre-commit | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100755 git_hooks/pre-commit diff --git a/git_hooks/pre-commit b/git_hooks/pre-commit new file mode 100755 index 000000000..8ae45b3f3 --- /dev/null +++ b/git_hooks/pre-commit @@ -0,0 +1,19 @@ +#!/usr/bin/env sh +cargo fmt -- --check +if [ "$?" != "0" ] ; then + echo "\n" + echo "\033[0;31mpre-commit hook failed during:\033[0m" + echo "\033[0;31m\tcargo fmt -- --check\033[0m" + exit 1 +fi +cargo clippy --locked -- -D warnings +if [ "$?" != "0" ] ; then + echo "\n" + echo "\033[0;31mpre-commit hook failed during:\033[0m" + echo "\033[0;31m\tclippy --locked -- -D warning\033[0m" + exit 1 +fi + +echo "\n" +echo "\033[0;32mpre-commit hook succeeded\033[0m" +exit 0 \ No newline at end of file From 4867b38f9d9cd4b179f51bf3924439e7a7097a52 Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Wed, 12 Aug 2020 19:59:52 +0300 Subject: [PATCH 073/139] Changed envar prefix and replace echo with printf for pre-commit hook --- git_hooks/pre-commit | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/git_hooks/pre-commit b/git_hooks/pre-commit index 8ae45b3f3..f6855e88e 100755 --- a/git_hooks/pre-commit +++ b/git_hooks/pre-commit @@ -1,19 +1,19 @@ #!/usr/bin/env sh cargo fmt -- --check if [ "$?" != "0" ] ; then - echo "\n" - echo "\033[0;31mpre-commit hook failed during:\033[0m" - echo "\033[0;31m\tcargo fmt -- --check\033[0m" + printf "\n" + printf "\033[0;31mpre-commit hook failed during:\033[0m\n" + printf "\033[0;31m\tcargo fmt -- --check\033[0m\n" exit 1 fi cargo clippy --locked -- -D warnings if [ "$?" != "0" ] ; then - echo "\n" - echo "\033[0;31mpre-commit hook failed during:\033[0m" - echo "\033[0;31m\tclippy --locked -- -D warning\033[0m" + printf "\n" + printf "\033[0;31mpre-commit hook failed during:\033[0m\n" + printf "\033[0;31m\tclippy --locked -- -D warning\033[0m\n" exit 1 fi -echo "\n" -echo "\033[0;32mpre-commit hook succeeded\033[0m" +printf "\n" +printf "\033[0;32mpre-commit hook succeeded\033[0m\n" exit 0 \ No newline at end of file From 16fb2a72aa128091dfcd3d2c9f7c89f9d50c41ad Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Thu, 13 Aug 2020 21:48:48 +0300 Subject: [PATCH 074/139] Rebased --- .git_hooks/pre-commit | 1 - git_hooks/pre-commit | 19 ------------------- 2 files changed, 20 deletions(-) delete mode 100755 git_hooks/pre-commit diff --git a/.git_hooks/pre-commit b/.git_hooks/pre-commit index 105684ef7..872c2cdda 100755 --- a/.git_hooks/pre-commit +++ b/.git_hooks/pre-commit @@ -16,4 +16,3 @@ fi printf "\n" printf "\033[0;32mpre-commit hook succeeded\033[0m\n" exit 0 - diff --git a/git_hooks/pre-commit b/git_hooks/pre-commit deleted file mode 100755 index f6855e88e..000000000 --- a/git_hooks/pre-commit +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env sh -cargo fmt -- --check -if [ "$?" != "0" ] ; then - printf "\n" - printf "\033[0;31mpre-commit hook failed during:\033[0m\n" - printf "\033[0;31m\tcargo fmt -- --check\033[0m\n" - exit 1 -fi -cargo clippy --locked -- -D warnings -if [ "$?" != "0" ] ; then - printf "\n" - printf "\033[0;31mpre-commit hook failed during:\033[0m\n" - printf "\033[0;31m\tclippy --locked -- -D warning\033[0m\n" - exit 1 -fi - -printf "\n" -printf "\033[0;32mpre-commit hook succeeded\033[0m\n" -exit 0 \ No newline at end of file From 58ab2b28238547732b960dcaef10f798f9304e60 Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Thu, 13 Aug 2020 15:58:30 +0300 Subject: [PATCH 075/139] Added newline at end of pre-commit script --- .git_hooks/pre-commit | 1 + 1 file changed, 1 insertion(+) diff --git a/.git_hooks/pre-commit b/.git_hooks/pre-commit index 872c2cdda..105684ef7 100755 --- a/.git_hooks/pre-commit +++ b/.git_hooks/pre-commit @@ -16,3 +16,4 @@ fi printf "\n" printf "\033[0;32mpre-commit hook succeeded\033[0m\n" exit 0 + From d6383e7abab0c2209ed6ca32764a28bc48135c25 Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Thu, 13 Aug 2020 22:16:15 +0300 Subject: [PATCH 076/139] Rebased --- src/utils/daemon.rs | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs index 7b886eab8..d142fa544 100644 --- a/src/utils/daemon.rs +++ b/src/utils/daemon.rs @@ -9,16 +9,13 @@ use crate::{ use chrono::{Timelike, Utc}; use failure::Error; use log::{debug, error, info}; -use std::sync::Arc; use std::thread; use std::time::{Duration, Instant}; -fn start_registry_watcher( - opts: DocBuilderOptions, - pool: Pool, - build_queue: Arc, - config: Arc, -) -> Result<(), Error> { +fn start_registry_watcher(opts: DocBuilderOptions, context: &dyn Context) -> Result<(), Error> { + let pool = context.pool()?; + let build_queue = context.build_queue()?; + let config = context.config()?; thread::Builder::new() .name("registry index reader".to_string()) .spawn(move || { @@ -60,24 +57,18 @@ pub fn start_daemon(context: &dyn Context, enable_registry_watcher: bool) -> Res if enable_registry_watcher { // check new crates every minute - start_registry_watcher( - dbopts.clone(), - db.clone(), - build_queue.clone(), - config.clone(), - )?; + start_registry_watcher(dbopts.clone(), context)?; } // build new crates every minute let pool = context.pool()?; let build_queue = context.build_queue()?; let storage = context.storage()?; - let metrics = context.metrics()?; thread::Builder::new() .name("build queue reader".to_string()) .spawn(move || { let doc_builder = DocBuilder::new(dbopts.clone(), pool.clone(), build_queue.clone()); - queue_builder(doc_builder, pool, build_queue, metrics, storage).unwrap(); + queue_builder(doc_builder, pool, build_queue, storage).unwrap(); }) .unwrap(); From 19ca7027cad104d2f16faec178130b46517c9dd9 Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Thu, 13 Aug 2020 22:23:40 +0300 Subject: [PATCH 077/139] Rebased --- src/metrics/macros.rs | 34 ++--- src/metrics/mod.rs | 31 ++--- src/utils/daemon.rs | 3 +- src/web/metrics.rs | 310 +++++++++++++++++------------------------- 4 files changed, 151 insertions(+), 227 deletions(-) diff --git a/src/metrics/macros.rs b/src/metrics/macros.rs index cf7ccd256..c79bd60df 100644 --- a/src/metrics/macros.rs +++ b/src/metrics/macros.rs @@ -6,41 +6,27 @@ pub(super) trait MetricFromOpts: Sized { macro_rules! metrics { ( $vis:vis struct $name:ident { - $( - #[doc = $help:expr] - $(#[$meta:meta])* - $metric_vis:vis $metric:ident: $ty:ty $([$($label:expr),* $(,)?])? - ),* $(,)? + $(#[doc = $help:expr] $metric:ident: $ty:ty $([$($label:expr),*])?,)* } + metrics visibility: $metric_vis:vis, namespace: $namespace:expr, ) => { $vis struct $name { - registry: prometheus::Registry, - $( - $(#[$meta])* - $metric_vis $metric: $ty, - )* + registry: Registry, + $($metric_vis $metric: $ty,)* } impl $name { - $vis fn new() -> Result { - let registry = prometheus::Registry::new(); + $vis fn new() -> Result { + let registry = Registry::new(); $( - $(#[$meta])* let $metric = <$ty>::from_opts( - prometheus::Opts::new(stringify!($metric), $help) + Opts::new(stringify!($metric), $help) .namespace($namespace) $(.variable_labels(vec![$($label.into()),*]))? )?; - $(#[$meta])* registry.register(Box::new($metric.clone()))?; )* - Ok(Self { - registry, - $( - $(#[$meta])* - $metric, - )* - }) + Ok(Self { registry, $($metric,)* }) } } impl std::fmt::Debug for $name { @@ -56,7 +42,7 @@ macro_rules! load_metric_type { ($name:ident as single) => { use prometheus::$name; impl MetricFromOpts for $name { - fn from_opts(opts: prometheus::Opts) -> Result { + fn from_opts(opts: Opts) -> Result { $name::with_opts(opts) } } @@ -64,7 +50,7 @@ macro_rules! load_metric_type { ($name:ident as vec) => { use prometheus::$name; impl MetricFromOpts for $name { - fn from_opts(opts: prometheus::Opts) -> Result { + fn from_opts(opts: Opts) -> Result { $name::new( opts.clone().into(), opts.variable_labels diff --git a/src/metrics/mod.rs b/src/metrics/mod.rs index 9f5eabebd..870109a7e 100644 --- a/src/metrics/mod.rs +++ b/src/metrics/mod.rs @@ -5,7 +5,7 @@ use self::macros::MetricFromOpts; use crate::db::Pool; use crate::BuildQueue; use failure::Error; -use prometheus::proto::MetricFamily; +use prometheus::{proto::MetricFamily, Opts, Registry}; load_metric_type!(IntGauge as single); load_metric_type!(IntCounter as single); @@ -28,42 +28,37 @@ metrics! { /// The maximum number of database connections max_db_connections: IntGauge, /// Number of attempted and failed connections to the database - pub(crate) failed_db_connections: IntCounter, + failed_db_connections: IntCounter, /// The number of currently opened file descriptors - #[cfg(target_os = "linux")] open_file_descriptors: IntGauge, /// The number of threads being used by docs.rs - #[cfg(target_os = "linux")] running_threads: IntGauge, /// The traffic of various docs.rs routes - pub(crate) routes_visited: IntCounterVec["route"], + routes_visited: IntCounterVec["route"], /// The response times of various docs.rs routes - pub(crate) response_time: HistogramVec["route"], + response_time: HistogramVec["route"], /// The time it takes to render a rustdoc page - pub(crate) rustdoc_rendering_times: HistogramVec["step"], + rustdoc_rendering_times: HistogramVec["step"], /// Number of crates built - pub(crate) total_builds: IntCounter, + total_builds: IntCounter, /// Number of builds that successfully generated docs - pub(crate) successful_builds: IntCounter, + successful_builds: IntCounter, /// Number of builds that generated a compiler error - pub(crate) failed_builds: IntCounter, + failed_builds: IntCounter, /// Number of builds that did not complete due to not being a library - pub(crate) non_library_builds: IntCounter, + non_library_builds: IntCounter, /// Number of files uploaded to the storage backend - pub(crate) uploaded_files_total: IntCounter, + uploaded_files_total: IntCounter, /// The number of attempted files that failed due to a memory limit - pub(crate) html_rewrite_ooms: IntCounter, + html_rewrite_ooms: IntCounter, } - // The Rust prometheus library treats the namespace as the "prefix" of the metric name: a - // metric named `foo` with a prefix of `docsrs` will expose a metric called `docsrs_foo`. - // - // https://docs.rs/prometheus/0.9.0/prometheus/struct.Opts.html#structfield.namespace + metrics visibility: pub(crate), namespace: "docsrs", } @@ -86,7 +81,7 @@ impl Metrics { Ok(self.registry.gather()) } - #[cfg(not(target_os = "linux"))] + #[cfg(not(linux))] fn gather_system_performance(&self) {} #[cfg(target_os = "linux")] diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs index d142fa544..673f05204 100644 --- a/src/utils/daemon.rs +++ b/src/utils/daemon.rs @@ -64,11 +64,12 @@ pub fn start_daemon(context: &dyn Context, enable_registry_watcher: bool) -> Res let pool = context.pool()?; let build_queue = context.build_queue()?; let storage = context.storage()?; + let metrics = context.metrics()?; thread::Builder::new() .name("build queue reader".to_string()) .spawn(move || { let doc_builder = DocBuilder::new(dbopts.clone(), pool.clone(), build_queue.clone()); - queue_builder(doc_builder, pool, build_queue, storage).unwrap(); + queue_builder(doc_builder, pool, build_queue, metrics, storage).unwrap(); }) .unwrap(); diff --git a/src/web/metrics.rs b/src/web/metrics.rs index d222f1924..dbb910499 100644 --- a/src/web/metrics.rs +++ b/src/web/metrics.rs @@ -7,150 +7,6 @@ use iron::status::Status; use prometheus::{Encoder, HistogramVec, TextEncoder}; use std::time::{Duration, Instant}; -static QUEUED_CRATES_COUNT: Lazy = Lazy::new(|| { - register_int_gauge!( - "docsrs_queued_crates_count", - "Number of crates in the build queue" - ) - .unwrap() -}); - -pub static PRIORITIZED_CRATES_COUNT: Lazy = Lazy::new(|| { - register_int_gauge!( - "docsrs_prioritized_crates_count", - "Number of crates in the build queue that have a positive priority" - ) - .unwrap() -}); - -static FAILED_CRATES_COUNT: Lazy = Lazy::new(|| { - register_int_gauge!( - "docsrs_failed_crates_count", - "Number of crates that failed to build" - ) - .unwrap() -}); - -pub static TOTAL_BUILDS: Lazy = - Lazy::new(|| register_int_counter!("docsrs_total_builds", "Number of crates built").unwrap()); - -pub static SUCCESSFUL_BUILDS: Lazy = Lazy::new(|| { - register_int_counter!( - "docsrs_successful_builds", - "Number of builds that successfully generated docs" - ) - .unwrap() -}); - -pub static FAILED_BUILDS: Lazy = Lazy::new(|| { - register_int_counter!( - "docsrs_failed_builds", - "Number of builds that generated a compile error" - ) - .unwrap() -}); - -pub static NON_LIBRARY_BUILDS: Lazy = Lazy::new(|| { - register_int_counter!( - "docsrs_non_library_builds", - "Number of builds that did not complete due to not being a library" - ) - .unwrap() -}); - -pub static UPLOADED_FILES_TOTAL: Lazy = Lazy::new(|| { - register_int_counter!( - "docsrs_uploaded_files_total", - "Number of files uploaded to S3 or stored in the database" - ) - .unwrap() -}); - -pub static ROUTES_VISITED: Lazy = Lazy::new(|| { - register_int_counter_vec!( - "docsrs_routes_visited", - "The traffic of various docs.rs routes", - &["route"] - ) - .unwrap() -}); - -pub static RESPONSE_TIMES: Lazy = Lazy::new(|| { - register_histogram_vec!( - "docsrs_response_time", - "The response times of various docs.rs routes", - &["route"] - ) - .unwrap() -}); - -pub static RUSTDOC_RENDERING_TIMES: Lazy = Lazy::new(|| { - register_histogram_vec!( - "docsrs_rustdoc_rendering_time", - "The time it takes to render a rustdoc page", - &["step"] - ) - .unwrap() -}); - -pub static FAILED_DB_CONNECTIONS: Lazy = Lazy::new(|| { - register_int_counter!( - "docsrs_failed_db_connections", - "Number of attempted and failed connections to the database" - ) - .unwrap() -}); - -pub static USED_DB_CONNECTIONS: Lazy = Lazy::new(|| { - register_int_gauge!( - "docsrs_used_db_connections", - "The number of used database connections" - ) - .unwrap() -}); - -pub static IDLE_DB_CONNECTIONS: Lazy = Lazy::new(|| { - register_int_gauge!( - "docsrs_idle_db_connections", - "The number of idle database connections" - ) - .unwrap() -}); - -pub static MAX_DB_CONNECTIONS: Lazy = Lazy::new(|| { - register_int_gauge!( - "docsrs_max_db_connections", - "The maximum database connections" - ) - .unwrap() -}); - -#[cfg(target_os = "linux")] -pub static OPEN_FILE_DESCRIPTORS: Lazy = Lazy::new(|| { - register_int_gauge!( - "docsrs_open_file_descriptors", - "The number of currently opened file descriptors" - ) - .unwrap() -}); - -#[cfg(target_os = "linux")] -pub static CURRENTLY_RUNNING_THREADS: Lazy = Lazy::new(|| { - register_int_gauge!( - "docsrs_running_threads", - "The number of threads being used by docs.rs" - ) - .unwrap() -}); - -pub static HTML_REWRITE_OOMS: Lazy = Lazy::new(|| { - register_int_gauge!( - "docsrs_html_rewrite_ooms", - "The number of attempted files that failed due to a memory limit" - ) - .unwrap() -}); - pub fn metrics_handler(req: &mut Request) -> IronResult { let metrics = extension!(req, Metrics); let pool = extension!(req, Pool); @@ -252,32 +108,80 @@ impl Drop for RenderingTimesRecorder<'_> { #[cfg(test)] mod tests { use crate::test::{assert_success, wrapper}; - use std::collections::HashMap; #[test] - fn test_response_times_count_being_collected() { - const ROUTES: &[(&str, &str)] = &[ - ("", "/"), - ("/", "/"), - ("/crate/hexponent/0.2.0", "/crate/:name/:version"), - ("/crate/rcc/0.0.0", "/crate/:name/:version"), - ("/index.js", "static resource"), - ("/menu.js", "static resource"), - ("/opensearch.xml", "static resource"), - ("/releases", "/releases"), - ("/releases/feed", "static resource"), - ("/releases/queue", "/releases/queue"), - ("/releases/recent-failures", "/releases/recent-failures"), - ( - "/releases/recent-failures/1", - "/releases/recent-failures/:page", - ), - ("/releases/recent/1", "/releases/recent/:page"), - ("/robots.txt", "static resource"), - ("/sitemap.xml", "static resource"), - ("/style.css", "static resource"), - ]; + fn home_page() { + wrapper(|env| { + let frontend = env.frontend(); + let metrics = env.metrics(); + + frontend.get("/").send()?; + frontend.get("/").send()?; + assert_eq!(metrics.routes_visited.with_label_values(&["/"]).get(), 2); + assert_eq!( + metrics + .response_time + .with_label_values(&["/"]) + .get_sample_count(), + 2 + ); + + frontend.get("").send()?; + frontend.get("").send()?; + assert_eq!(metrics.routes_visited.with_label_values(&["/"]).get(), 4); + assert_eq!( + metrics + .response_time + .with_label_values(&["/"]) + .get_sample_count(), + 4 + ); + + Ok(()) + }) + } + + #[test] + fn resources() { + wrapper(|env| { + let frontend = env.frontend(); + let metrics = env.metrics(); + + let routes = [ + "/style.css", + "/index.js", + "/menu.js", + "/sitemap.xml", + "/opensearch.xml", + "/robots.txt", + ]; + + for route in routes.iter() { + frontend.get(route).send()?; + frontend.get(route).send()?; + } + + assert_eq!( + metrics + .routes_visited + .with_label_values(&["static resource"]) + .get(), + 12 + ); + assert_eq!( + metrics + .response_time + .with_label_values(&["static resource"]) + .get_sample_count(), + 12 + ); + Ok(()) + }) + } + + #[test] + fn releases() { wrapper(|env| { env.fake_release() .name("rcc") @@ -289,36 +193,36 @@ mod tests { .version("1.0.0") .build_result_successful(false) .create()?; - env.fake_release() - .name("hexponent") - .version("0.2.0") - .create()?; let frontend = env.frontend(); let metrics = env.metrics(); - for (route, _) in ROUTES.iter() { + let routes = [ + ("/releases", "/releases"), + ("/releases/recent/1", "/releases/recent/:page"), + ("/releases/feed", "static resource"), + ("/releases/queue", "/releases/queue"), + ("/releases/recent-failures", "/releases/recent-failures"), + ( + "/releases/recent-failures/1", + "/releases/recent-failures/:page", + ), + ]; + + for (route, correct) in routes.iter() { frontend.get(route).send()?; frontend.get(route).send()?; - } - let mut expected = HashMap::new(); - for (_, correct) in ROUTES.iter() { - let entry = expected.entry(*correct).or_insert(0); - *entry += 2; - } - - for (label, count) in expected.iter() { assert_eq!( - metrics.routes_visited.with_label_values(&[*label]).get(), - *count + metrics.routes_visited.with_label_values(&[*correct]).get(), + 2 ); assert_eq!( metrics .response_time - .with_label_values(&[*label]) + .with_label_values(&[*correct]) .get_sample_count(), - *count as u64 + 2 ); } @@ -326,6 +230,44 @@ mod tests { }) } + #[test] + fn crates() { + wrapper(|env| { + env.fake_release().name("rcc").version("0.0.0").create()?; + env.fake_release() + .name("hexponent") + .version("0.2.0") + .create()?; + + let frontend = env.frontend(); + let metrics = env.metrics(); + + let routes = ["/crate/rcc/0.0.0", "/crate/hexponent/0.2.0"]; + + for route in routes.iter() { + frontend.get(route).send()?; + frontend.get(route).send()?; + } + + assert_eq!( + metrics + .routes_visited + .with_label_values(&["/crate/:name/:version"]) + .get(), + 4 + ); + assert_eq!( + metrics + .response_time + .with_label_values(&["/crate/:name/:version"]) + .get_sample_count(), + 4 + ); + + Ok(()) + }) + } + #[test] fn test_metrics_page_success() { wrapper(|env| { From a1ef4bce2fd8f719e7a0b290bbb844140dc7b2d9 Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Thu, 13 Aug 2020 22:24:39 +0300 Subject: [PATCH 078/139] Rebased --- src/metrics/macros.rs | 23 +++++++++++++++++++---- src/metrics/mod.rs | 4 +++- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/src/metrics/macros.rs b/src/metrics/macros.rs index c79bd60df..2571f5b1e 100644 --- a/src/metrics/macros.rs +++ b/src/metrics/macros.rs @@ -6,27 +6,42 @@ pub(super) trait MetricFromOpts: Sized { macro_rules! metrics { ( $vis:vis struct $name:ident { - $(#[doc = $help:expr] $metric:ident: $ty:ty $([$($label:expr),*])?,)* + $( + #[doc = $help:expr] + $(#[$meta:meta])* + $metric_vis:vis $metric:ident: $ty:ty $([$($label:expr),* $(,)?])? + ),* $(,)? } metrics visibility: $metric_vis:vis, namespace: $namespace:expr, ) => { $vis struct $name { - registry: Registry, - $($metric_vis $metric: $ty,)* + registry: prometheus::Registry, + $( + $(#[$meta])* + $metric_vis $metric: $ty, + )* } impl $name { $vis fn new() -> Result { let registry = Registry::new(); $( + $(#[$meta])* let $metric = <$ty>::from_opts( Opts::new(stringify!($metric), $help) .namespace($namespace) $(.variable_labels(vec![$($label.into()),*]))? )?; + $(#[$meta])* registry.register(Box::new($metric.clone()))?; )* - Ok(Self { registry, $($metric,)* }) + Ok(Self { + registry, + $( + $(#[$meta])* + $metric, + )* + }) } } impl std::fmt::Debug for $name { diff --git a/src/metrics/mod.rs b/src/metrics/mod.rs index 870109a7e..3616ef657 100644 --- a/src/metrics/mod.rs +++ b/src/metrics/mod.rs @@ -31,8 +31,10 @@ metrics! { failed_db_connections: IntCounter, /// The number of currently opened file descriptors + #[cfg(linux)] open_file_descriptors: IntGauge, /// The number of threads being used by docs.rs + #[cfg(linux)] running_threads: IntGauge, /// The traffic of various docs.rs routes @@ -84,7 +86,7 @@ impl Metrics { #[cfg(not(linux))] fn gather_system_performance(&self) {} - #[cfg(target_os = "linux")] + #[cfg(linux)] fn gather_system_performance(&self) { use procfs::process::Process; From 6715bed86e79cdfa02eaea18e69a3dda708584fa Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Thu, 13 Aug 2020 22:28:00 +0300 Subject: [PATCH 079/139] Rebased --- src/config.rs | 2 +- src/utils/daemon.rs | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/config.rs b/src/config.rs index a3ae7dc9f..d84f12872 100644 --- a/src/config.rs +++ b/src/config.rs @@ -30,7 +30,7 @@ pub struct Config { pub(crate) max_file_size_html: usize, // The most memory that can be used to parse an HTML file pub(crate) max_parse_memory: usize, - // Time between 'git gc' calls in seconds + // Time between 'git gc --auto' calls in seconds pub(crate) registry_gc_interval: u64, } diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs index 673f05204..af8436c94 100644 --- a/src/utils/daemon.rs +++ b/src/utils/daemon.rs @@ -22,7 +22,6 @@ fn start_registry_watcher(opts: DocBuilderOptions, context: &dyn Context) -> Res // space this out to prevent it from clashing against the queue-builder thread on launch thread::sleep(Duration::from_secs(30)); let mut last_gc = Instant::now(); - loop { let mut doc_builder = DocBuilder::new(opts.clone(), pool.clone(), build_queue.clone()); From b073e270b9cdf6cad008ec809194244c88f6a4ab Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Thu, 13 Aug 2020 22:28:39 +0300 Subject: [PATCH 080/139] Rebased --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index d7823cbaf..5b7d9caab 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ target .vagrant .rustwide .rustwide-docker +.idea/ \ No newline at end of file From f2310727d21394cc166c3d3a17d8b0d856718ea5 Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Thu, 13 Aug 2020 22:30:47 +0300 Subject: [PATCH 081/139] Rebased --- git_hooks/pre-commit | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100755 git_hooks/pre-commit diff --git a/git_hooks/pre-commit b/git_hooks/pre-commit new file mode 100755 index 000000000..8ae45b3f3 --- /dev/null +++ b/git_hooks/pre-commit @@ -0,0 +1,19 @@ +#!/usr/bin/env sh +cargo fmt -- --check +if [ "$?" != "0" ] ; then + echo "\n" + echo "\033[0;31mpre-commit hook failed during:\033[0m" + echo "\033[0;31m\tcargo fmt -- --check\033[0m" + exit 1 +fi +cargo clippy --locked -- -D warnings +if [ "$?" != "0" ] ; then + echo "\n" + echo "\033[0;31mpre-commit hook failed during:\033[0m" + echo "\033[0;31m\tclippy --locked -- -D warning\033[0m" + exit 1 +fi + +echo "\n" +echo "\033[0;32mpre-commit hook succeeded\033[0m" +exit 0 \ No newline at end of file From ad351258ed38fdfba802288cddf8f7dd1be747b3 Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Tue, 11 Aug 2020 22:09:34 +0300 Subject: [PATCH 082/139] Run git gc periodically on the crates.io index #778 --- src/config.rs | 3 ++- src/utils/daemon.rs | 26 ++++++++++++++++++++------ 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/src/config.rs b/src/config.rs index d84f12872..497ad575d 100644 --- a/src/config.rs +++ b/src/config.rs @@ -30,6 +30,7 @@ pub struct Config { pub(crate) max_file_size_html: usize, // The most memory that can be used to parse an HTML file pub(crate) max_parse_memory: usize, + // Time between 'git gc --auto' calls in seconds pub(crate) registry_gc_interval: u64, } @@ -63,7 +64,7 @@ impl Config { // LOL HTML only uses as much memory as the size of the start tag! // https://github.com/rust-lang/docs.rs/pull/930#issuecomment-667729380 max_parse_memory: env("DOCSRS_MAX_PARSE_MEMORY", 5 * 1024 * 1024)?, - registry_gc_interval: env("DOCSRS_REGISTRY_GC_INTERVAL", 60 * 60)?, + registry_gc_interval: env("REGISTRY_GC_INTERVAL", 60 * 60)?, }) } diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs index af8436c94..68276ec54 100644 --- a/src/utils/daemon.rs +++ b/src/utils/daemon.rs @@ -11,17 +11,30 @@ use failure::Error; use log::{debug, error, info}; use std::thread; use std::time::{Duration, Instant}; +use std::process::Command; -fn start_registry_watcher(opts: DocBuilderOptions, context: &dyn Context) -> Result<(), Error> { - let pool = context.pool()?; - let build_queue = context.build_queue()?; - let config = context.config()?; + +fn run_git_gc() { + Command::new("git") + .args(&["gc"]) + .output() + .expect("Failed to execute git gc"); +} + +fn start_registry_watcher( + opts: DocBuilderOptions, + pool: Pool, + build_queue: Arc, + config: Arc, +) -> Result<(), Error> { thread::Builder::new() .name("registry index reader".to_string()) .spawn(move || { // space this out to prevent it from clashing against the queue-builder thread on launch thread::sleep(Duration::from_secs(30)); let mut last_gc = Instant::now(); + run_git_gc(); + loop { let mut doc_builder = DocBuilder::new(opts.clone(), pool.clone(), build_queue.clone()); @@ -37,7 +50,7 @@ fn start_registry_watcher(opts: DocBuilderOptions, context: &dyn Context) -> Res } if last_gc.elapsed().as_secs() >= config.registry_gc_interval { - doc_builder.run_git_gc(); + run_git_gc(); last_gc = Instant::now(); } thread::sleep(Duration::from_secs(60)); @@ -56,7 +69,8 @@ pub fn start_daemon(context: &dyn Context, enable_registry_watcher: bool) -> Res if enable_registry_watcher { // check new crates every minute - start_registry_watcher(dbopts.clone(), context)?; + start_registry_watcher(dbopts.clone(), db.clone(), build_queue.clone(), + config.clone())?; } // build new crates every minute From 4f9fd37adad8001559c04ce980e87089ce36cfc1 Mon Sep 17 00:00:00 2001 From: ohaddahan Date: Tue, 11 Aug 2020 22:14:34 +0300 Subject: [PATCH 083/139] Update config.rs --- src/config.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config.rs b/src/config.rs index 497ad575d..871002745 100644 --- a/src/config.rs +++ b/src/config.rs @@ -31,7 +31,7 @@ pub struct Config { // The most memory that can be used to parse an HTML file pub(crate) max_parse_memory: usize, - // Time between 'git gc --auto' calls in seconds + // Time between 'git gc' calls in seconds pub(crate) registry_gc_interval: u64, } From 755eda1bf17ecc415150a02bd396ec12a5c724bb Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Tue, 11 Aug 2020 22:20:18 +0300 Subject: [PATCH 084/139] Fixed "cargo fmt" --- src/utils/daemon.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs index 68276ec54..bb086db06 100644 --- a/src/utils/daemon.rs +++ b/src/utils/daemon.rs @@ -9,10 +9,10 @@ use crate::{ use chrono::{Timelike, Utc}; use failure::Error; use log::{debug, error, info}; +use std::process::Command; +use std::sync::Arc; use std::thread; use std::time::{Duration, Instant}; -use std::process::Command; - fn run_git_gc() { Command::new("git") @@ -69,8 +69,12 @@ pub fn start_daemon(context: &dyn Context, enable_registry_watcher: bool) -> Res if enable_registry_watcher { // check new crates every minute - start_registry_watcher(dbopts.clone(), db.clone(), build_queue.clone(), - config.clone())?; + start_registry_watcher( + dbopts.clone(), + db.clone(), + build_queue.clone(), + config.clone(), + )?; } // build new crates every minute From 4b48570a034cca1bcbff28a2618d29639ae94a18 Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Wed, 12 Aug 2020 09:01:44 +0300 Subject: [PATCH 085/139] Added --auto --- src/utils/daemon.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs index bb086db06..c993d4c16 100644 --- a/src/utils/daemon.rs +++ b/src/utils/daemon.rs @@ -16,7 +16,7 @@ use std::time::{Duration, Instant}; fn run_git_gc() { Command::new("git") - .args(&["gc"]) + .args(&["gc", "--auto"]) .output() .expect("Failed to execute git gc"); } From 50473e7c8a2d6804b193871361f13760ca802e72 Mon Sep 17 00:00:00 2001 From: ohaddahan Date: Tue, 11 Aug 2020 23:22:02 +0300 Subject: [PATCH 086/139] Update src/utils/daemon.rs Co-authored-by: Chase Wilson --- src/utils/daemon.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs index c993d4c16..db70b1cad 100644 --- a/src/utils/daemon.rs +++ b/src/utils/daemon.rs @@ -32,8 +32,8 @@ fn start_registry_watcher( .spawn(move || { // space this out to prevent it from clashing against the queue-builder thread on launch thread::sleep(Duration::from_secs(30)); - let mut last_gc = Instant::now(); run_git_gc(); + let mut last_gc = Instant::now(); loop { let mut doc_builder = From c651b1386764d0bc64dcade04333b23fb9fb17a7 Mon Sep 17 00:00:00 2001 From: ohaddahan Date: Tue, 11 Aug 2020 23:22:11 +0300 Subject: [PATCH 087/139] Update src/utils/daemon.rs Co-authored-by: Chase Wilson --- src/utils/daemon.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs index db70b1cad..de8700fc5 100644 --- a/src/utils/daemon.rs +++ b/src/utils/daemon.rs @@ -15,10 +15,13 @@ use std::thread; use std::time::{Duration, Instant}; fn run_git_gc() { - Command::new("git") - .args(&["gc", "--auto"]) - .output() - .expect("Failed to execute git gc"); + let gc = Command::new("git") + .args(&["gc"]) + .output(); + + if let Err(err) = gc { + log::error!("failed to run `git gc`: {:?}", err); + } } fn start_registry_watcher( From 20bed437c31fdc95721f162d35ff38d273d1f01d Mon Sep 17 00:00:00 2001 From: ohaddahan Date: Tue, 11 Aug 2020 23:22:27 +0300 Subject: [PATCH 088/139] Update src/utils/daemon.rs Co-authored-by: Chase Wilson --- src/utils/daemon.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs index de8700fc5..299e3a88f 100644 --- a/src/utils/daemon.rs +++ b/src/utils/daemon.rs @@ -16,7 +16,7 @@ use std::time::{Duration, Instant}; fn run_git_gc() { let gc = Command::new("git") - .args(&["gc"]) + .args(&["gc", "--auto"]) .output(); if let Err(err) = gc { From 0ab6ece3a1fc5c0b6f65748008739ed7068630e9 Mon Sep 17 00:00:00 2001 From: ohaddahan Date: Tue, 11 Aug 2020 23:22:36 +0300 Subject: [PATCH 089/139] Update src/config.rs Co-authored-by: Chase Wilson --- src/config.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/config.rs b/src/config.rs index 871002745..86fc11db5 100644 --- a/src/config.rs +++ b/src/config.rs @@ -30,7 +30,6 @@ pub struct Config { pub(crate) max_file_size_html: usize, // The most memory that can be used to parse an HTML file pub(crate) max_parse_memory: usize, - // Time between 'git gc' calls in seconds pub(crate) registry_gc_interval: u64, } From 655dd1853020f451915c2e0bfd8938b0b8371f2d Mon Sep 17 00:00:00 2001 From: ohaddahan Date: Tue, 11 Aug 2020 23:22:43 +0300 Subject: [PATCH 090/139] Update .gitignore Co-authored-by: Chase Wilson --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index 5b7d9caab..d7823cbaf 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,3 @@ target .vagrant .rustwide .rustwide-docker -.idea/ \ No newline at end of file From 26bd4e308e79f3ecc9e1985aa32957c994e5535f Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Wed, 12 Aug 2020 09:11:54 +0300 Subject: [PATCH 091/139] Fix cargo fmt --- src/utils/daemon.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs index 299e3a88f..071ba3154 100644 --- a/src/utils/daemon.rs +++ b/src/utils/daemon.rs @@ -15,10 +15,8 @@ use std::thread; use std::time::{Duration, Instant}; fn run_git_gc() { - let gc = Command::new("git") - .args(&["gc", "--auto"]) - .output(); - + let gc = Command::new("git").args(&["gc", "--auto"]).output(); + if let Err(err) = gc { log::error!("failed to run `git gc`: {:?}", err); } From d585733fc0c059ea8e548f6e58f9321b547a7e05 Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Wed, 12 Aug 2020 09:51:30 +0300 Subject: [PATCH 092/139] Adding pre-commit hook, adding dead-code lint since without it clippy rails with "unused static" --- README.md | 2 +- src/web/metrics.rs | 146 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 147 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index c84156f16..53775185d 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ This gives you reasonable incremental build times without having to add new user ### Git Hooks For ease of use, `git_hooks` directory contains useful `git hooks` to make your development easier. ```bash -cd .git/hooks && ln -s ../../.git_hooks/* . && cd ../.. +cp git_hooks/* .git/hooks/ ``` ### Dependencies diff --git a/src/web/metrics.rs b/src/web/metrics.rs index dbb910499..8419ee576 100644 --- a/src/web/metrics.rs +++ b/src/web/metrics.rs @@ -7,6 +7,152 @@ use iron::status::Status; use prometheus::{Encoder, HistogramVec, TextEncoder}; use std::time::{Duration, Instant}; +static QUEUED_CRATES_COUNT: Lazy = Lazy::new(|| { + register_int_gauge!( + "docsrs_queued_crates_count", + "Number of crates in the build queue" + ) + .unwrap() +}); + +pub static PRIORITIZED_CRATES_COUNT: Lazy = Lazy::new(|| { + register_int_gauge!( + "docsrs_prioritized_crates_count", + "Number of crates in the build queue that have a positive priority" + ) + .unwrap() +}); + +static FAILED_CRATES_COUNT: Lazy = Lazy::new(|| { + register_int_gauge!( + "docsrs_failed_crates_count", + "Number of crates that failed to build" + ) + .unwrap() +}); + +pub static TOTAL_BUILDS: Lazy = + Lazy::new(|| register_int_counter!("docsrs_total_builds", "Number of crates built").unwrap()); + +pub static SUCCESSFUL_BUILDS: Lazy = Lazy::new(|| { + register_int_counter!( + "docsrs_successful_builds", + "Number of builds that successfully generated docs" + ) + .unwrap() +}); + +pub static FAILED_BUILDS: Lazy = Lazy::new(|| { + register_int_counter!( + "docsrs_failed_builds", + "Number of builds that generated a compile error" + ) + .unwrap() +}); + +pub static NON_LIBRARY_BUILDS: Lazy = Lazy::new(|| { + register_int_counter!( + "docsrs_non_library_builds", + "Number of builds that did not complete due to not being a library" + ) + .unwrap() +}); + +pub static UPLOADED_FILES_TOTAL: Lazy = Lazy::new(|| { + register_int_counter!( + "docsrs_uploaded_files_total", + "Number of files uploaded to S3 or stored in the database" + ) + .unwrap() +}); + +pub static ROUTES_VISITED: Lazy = Lazy::new(|| { + register_int_counter_vec!( + "docsrs_routes_visited", + "The traffic of various docs.rs routes", + &["route"] + ) + .unwrap() +}); + +pub static RESPONSE_TIMES: Lazy = Lazy::new(|| { + register_histogram_vec!( + "docsrs_response_time", + "The response times of various docs.rs routes", + &["route"] + ) + .unwrap() +}); + +pub static RUSTDOC_RENDERING_TIMES: Lazy = Lazy::new(|| { + register_histogram_vec!( + "docsrs_rustdoc_rendering_time", + "The time it takes to render a rustdoc page", + &["step"] + ) + .unwrap() +}); + +pub static FAILED_DB_CONNECTIONS: Lazy = Lazy::new(|| { + register_int_counter!( + "docsrs_failed_db_connections", + "Number of attempted and failed connections to the database" + ) + .unwrap() +}); + +pub static USED_DB_CONNECTIONS: Lazy = Lazy::new(|| { + register_int_gauge!( + "docsrs_used_db_connections", + "The number of used database connections" + ) + .unwrap() +}); + +pub static IDLE_DB_CONNECTIONS: Lazy = Lazy::new(|| { + register_int_gauge!( + "docsrs_idle_db_connections", + "The number of idle database connections" + ) + .unwrap() +}); + +pub static MAX_DB_CONNECTIONS: Lazy = Lazy::new(|| { + register_int_gauge!( + "docsrs_max_db_connections", + "The maximum database connections" + ) + .unwrap() +}); + +#[cfg(not(windows))] +#[allow(dead_code)] +pub static OPEN_FILE_DESCRIPTORS: Lazy = Lazy::new(|| { + register_int_gauge!( + "docsrs_open_file_descriptors", + "The number of currently opened file descriptors" + ) + .unwrap() +}); + +#[cfg(not(windows))] +#[allow(dead_code)] +pub static CURRENTLY_RUNNING_THREADS: Lazy = Lazy::new(|| { + register_int_gauge!( + "docsrs_running_threads", + "The number of threads being used by docs.rs" + ) + .unwrap() +}); + +pub static HTML_REWRITE_OOMS: Lazy = Lazy::new(|| { + register_int_gauge!( + "docsrs_html_rewrite_ooms", + "The number of attempted files that failed due to a memory limit" + ) + .unwrap() +}); + pub fn metrics_handler(req: &mut Request) -> IronResult { let metrics = extension!(req, Metrics); let pool = extension!(req, Pool); From a17c0c5bfbdac511d35bd1b6b945f889964df13d Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Wed, 12 Aug 2020 19:59:52 +0300 Subject: [PATCH 093/139] Changed envar prefix and replace echo with printf for pre-commit hook --- git_hooks/pre-commit | 16 ++++++++-------- src/config.rs | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/git_hooks/pre-commit b/git_hooks/pre-commit index 8ae45b3f3..f6855e88e 100755 --- a/git_hooks/pre-commit +++ b/git_hooks/pre-commit @@ -1,19 +1,19 @@ #!/usr/bin/env sh cargo fmt -- --check if [ "$?" != "0" ] ; then - echo "\n" - echo "\033[0;31mpre-commit hook failed during:\033[0m" - echo "\033[0;31m\tcargo fmt -- --check\033[0m" + printf "\n" + printf "\033[0;31mpre-commit hook failed during:\033[0m\n" + printf "\033[0;31m\tcargo fmt -- --check\033[0m\n" exit 1 fi cargo clippy --locked -- -D warnings if [ "$?" != "0" ] ; then - echo "\n" - echo "\033[0;31mpre-commit hook failed during:\033[0m" - echo "\033[0;31m\tclippy --locked -- -D warning\033[0m" + printf "\n" + printf "\033[0;31mpre-commit hook failed during:\033[0m\n" + printf "\033[0;31m\tclippy --locked -- -D warning\033[0m\n" exit 1 fi -echo "\n" -echo "\033[0;32mpre-commit hook succeeded\033[0m" +printf "\n" +printf "\033[0;32mpre-commit hook succeeded\033[0m\n" exit 0 \ No newline at end of file diff --git a/src/config.rs b/src/config.rs index 86fc11db5..a3ae7dc9f 100644 --- a/src/config.rs +++ b/src/config.rs @@ -63,7 +63,7 @@ impl Config { // LOL HTML only uses as much memory as the size of the start tag! // https://github.com/rust-lang/docs.rs/pull/930#issuecomment-667729380 max_parse_memory: env("DOCSRS_MAX_PARSE_MEMORY", 5 * 1024 * 1024)?, - registry_gc_interval: env("REGISTRY_GC_INTERVAL", 60 * 60)?, + registry_gc_interval: env("DOCSRS_REGISTRY_GC_INTERVAL", 60 * 60)?, }) } From 0478cdfe4aa8ce6803f930d67db8e5041d255d44 Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Wed, 12 Aug 2020 20:33:36 +0300 Subject: [PATCH 094/139] Removed allow(dead_code) and not(windows) , now OPEN_FILE_DESCRIPTORS and CURRENTLY_RUNNING_THREADS declaration match the usage, "target_os = linux" --- src/web/metrics.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/web/metrics.rs b/src/web/metrics.rs index 8419ee576..2f6dccf9d 100644 --- a/src/web/metrics.rs +++ b/src/web/metrics.rs @@ -125,8 +125,7 @@ pub static MAX_DB_CONNECTIONS: Lazy = Lazy::new(|| { .unwrap() }); -#[cfg(not(windows))] -#[allow(dead_code)] +#[cfg(target_os = "linux")] pub static OPEN_FILE_DESCRIPTORS: Lazy = Lazy::new(|| { register_int_gauge!( "docsrs_open_file_descriptors", @@ -135,8 +134,7 @@ pub static OPEN_FILE_DESCRIPTORS: Lazy = Lazy::new(|| { .unwrap() }); -#[cfg(not(windows))] -#[allow(dead_code)] +#[cfg(target_os = "linux")] pub static CURRENTLY_RUNNING_THREADS: Lazy = Lazy::new(|| { register_int_gauge!( "docsrs_running_threads", From 7688b67dbcd92aa905b369c03a5e4baacecf27a8 Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Thu, 13 Aug 2020 15:57:22 +0300 Subject: [PATCH 095/139] Updated pre-commit script and instructions --- .git_hooks/pre-commit | 1 - README.md | 2 +- git_hooks/pre-commit | 19 ------------------- 3 files changed, 1 insertion(+), 21 deletions(-) delete mode 100755 git_hooks/pre-commit diff --git a/.git_hooks/pre-commit b/.git_hooks/pre-commit index 105684ef7..872c2cdda 100755 --- a/.git_hooks/pre-commit +++ b/.git_hooks/pre-commit @@ -16,4 +16,3 @@ fi printf "\n" printf "\033[0;32mpre-commit hook succeeded\033[0m\n" exit 0 - diff --git a/README.md b/README.md index 53775185d..c84156f16 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ This gives you reasonable incremental build times without having to add new user ### Git Hooks For ease of use, `git_hooks` directory contains useful `git hooks` to make your development easier. ```bash -cp git_hooks/* .git/hooks/ +cd .git/hooks && ln -s ../../.git_hooks/* . && cd ../.. ``` ### Dependencies diff --git a/git_hooks/pre-commit b/git_hooks/pre-commit deleted file mode 100755 index f6855e88e..000000000 --- a/git_hooks/pre-commit +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env sh -cargo fmt -- --check -if [ "$?" != "0" ] ; then - printf "\n" - printf "\033[0;31mpre-commit hook failed during:\033[0m\n" - printf "\033[0;31m\tcargo fmt -- --check\033[0m\n" - exit 1 -fi -cargo clippy --locked -- -D warnings -if [ "$?" != "0" ] ; then - printf "\n" - printf "\033[0;31mpre-commit hook failed during:\033[0m\n" - printf "\033[0;31m\tclippy --locked -- -D warning\033[0m\n" - exit 1 -fi - -printf "\n" -printf "\033[0;32mpre-commit hook succeeded\033[0m\n" -exit 0 \ No newline at end of file From 252edf9c1952f1154b6f634ddaecf47217ec60fb Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Thu, 13 Aug 2020 15:58:30 +0300 Subject: [PATCH 096/139] Added newline at end of pre-commit script --- .git_hooks/pre-commit | 1 + 1 file changed, 1 insertion(+) diff --git a/.git_hooks/pre-commit b/.git_hooks/pre-commit index 872c2cdda..105684ef7 100755 --- a/.git_hooks/pre-commit +++ b/.git_hooks/pre-commit @@ -16,3 +16,4 @@ fi printf "\n" printf "\033[0;32mpre-commit hook succeeded\033[0m\n" exit 0 + From 69b363eee19991bcb9823f611040d27bdb6a56c0 Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Thu, 13 Aug 2020 17:54:29 +0300 Subject: [PATCH 097/139] Move run_git_gc to Index --- src/index/mod.rs | 2 +- src/utils/daemon.rs | 12 +----------- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/src/index/mod.rs b/src/index/mod.rs index e00c2c357..a4fef6e0e 100644 --- a/src/index/mod.rs +++ b/src/index/mod.rs @@ -52,7 +52,7 @@ impl Index { } pub fn run_git_gc(&self) { - let cmd = format!("cd {} && git gc --auto", self.path.to_str().unwrap()); + let cmd = format!("cd {} && gc --auto", self.path.to_str().unwrap()); let gc = Command::new("sh").args(&["-c", cmd.as_str()]).output(); if let Err(err) = gc { log::error!("Failed to run `{}`: {:?}", cmd, err); diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs index 071ba3154..7b886eab8 100644 --- a/src/utils/daemon.rs +++ b/src/utils/daemon.rs @@ -9,19 +9,10 @@ use crate::{ use chrono::{Timelike, Utc}; use failure::Error; use log::{debug, error, info}; -use std::process::Command; use std::sync::Arc; use std::thread; use std::time::{Duration, Instant}; -fn run_git_gc() { - let gc = Command::new("git").args(&["gc", "--auto"]).output(); - - if let Err(err) = gc { - log::error!("failed to run `git gc`: {:?}", err); - } -} - fn start_registry_watcher( opts: DocBuilderOptions, pool: Pool, @@ -33,7 +24,6 @@ fn start_registry_watcher( .spawn(move || { // space this out to prevent it from clashing against the queue-builder thread on launch thread::sleep(Duration::from_secs(30)); - run_git_gc(); let mut last_gc = Instant::now(); loop { @@ -51,7 +41,7 @@ fn start_registry_watcher( } if last_gc.elapsed().as_secs() >= config.registry_gc_interval { - run_git_gc(); + doc_builder.run_git_gc(); last_gc = Instant::now(); } thread::sleep(Duration::from_secs(60)); From 37bc3aab432e63fad98f79651d742484790ac39b Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Thu, 13 Aug 2020 19:48:32 +0300 Subject: [PATCH 098/139] Fix typo --- src/index/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/index/mod.rs b/src/index/mod.rs index a4fef6e0e..e00c2c357 100644 --- a/src/index/mod.rs +++ b/src/index/mod.rs @@ -52,7 +52,7 @@ impl Index { } pub fn run_git_gc(&self) { - let cmd = format!("cd {} && gc --auto", self.path.to_str().unwrap()); + let cmd = format!("cd {} && git gc --auto", self.path.to_str().unwrap()); let gc = Command::new("sh").args(&["-c", cmd.as_str()]).output(); if let Err(err) = gc { log::error!("Failed to run `{}`: {:?}", cmd, err); From 5285793ffbcce37983b110bedab2fb706776fd59 Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Thu, 13 Aug 2020 21:42:57 +0300 Subject: [PATCH 099/139] Rebased --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index d7823cbaf..5b7d9caab 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ target .vagrant .rustwide .rustwide-docker +.idea/ \ No newline at end of file From 993aef8dd0457bf6383e4814b2b035afd218ae40 Mon Sep 17 00:00:00 2001 From: ohaddahan Date: Tue, 11 Aug 2020 23:22:43 +0300 Subject: [PATCH 100/139] Update .gitignore Co-authored-by: Chase Wilson --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index 5b7d9caab..d7823cbaf 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,3 @@ target .vagrant .rustwide .rustwide-docker -.idea/ \ No newline at end of file From b992cc274c5650efdd52fa583e754bd6ecac418c Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Thu, 13 Aug 2020 21:48:01 +0300 Subject: [PATCH 101/139] Rebased --- git_hooks/pre-commit | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100755 git_hooks/pre-commit diff --git a/git_hooks/pre-commit b/git_hooks/pre-commit new file mode 100755 index 000000000..8ae45b3f3 --- /dev/null +++ b/git_hooks/pre-commit @@ -0,0 +1,19 @@ +#!/usr/bin/env sh +cargo fmt -- --check +if [ "$?" != "0" ] ; then + echo "\n" + echo "\033[0;31mpre-commit hook failed during:\033[0m" + echo "\033[0;31m\tcargo fmt -- --check\033[0m" + exit 1 +fi +cargo clippy --locked -- -D warnings +if [ "$?" != "0" ] ; then + echo "\n" + echo "\033[0;31mpre-commit hook failed during:\033[0m" + echo "\033[0;31m\tclippy --locked -- -D warning\033[0m" + exit 1 +fi + +echo "\n" +echo "\033[0;32mpre-commit hook succeeded\033[0m" +exit 0 \ No newline at end of file From 18e89b5530199ddb781c277c5a61f30816f79d06 Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Wed, 12 Aug 2020 19:59:52 +0300 Subject: [PATCH 102/139] Changed envar prefix and replace echo with printf for pre-commit hook --- git_hooks/pre-commit | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/git_hooks/pre-commit b/git_hooks/pre-commit index 8ae45b3f3..f6855e88e 100755 --- a/git_hooks/pre-commit +++ b/git_hooks/pre-commit @@ -1,19 +1,19 @@ #!/usr/bin/env sh cargo fmt -- --check if [ "$?" != "0" ] ; then - echo "\n" - echo "\033[0;31mpre-commit hook failed during:\033[0m" - echo "\033[0;31m\tcargo fmt -- --check\033[0m" + printf "\n" + printf "\033[0;31mpre-commit hook failed during:\033[0m\n" + printf "\033[0;31m\tcargo fmt -- --check\033[0m\n" exit 1 fi cargo clippy --locked -- -D warnings if [ "$?" != "0" ] ; then - echo "\n" - echo "\033[0;31mpre-commit hook failed during:\033[0m" - echo "\033[0;31m\tclippy --locked -- -D warning\033[0m" + printf "\n" + printf "\033[0;31mpre-commit hook failed during:\033[0m\n" + printf "\033[0;31m\tclippy --locked -- -D warning\033[0m\n" exit 1 fi -echo "\n" -echo "\033[0;32mpre-commit hook succeeded\033[0m" +printf "\n" +printf "\033[0;32mpre-commit hook succeeded\033[0m\n" exit 0 \ No newline at end of file From 7a5b5d71f5533cf2bf353dd4305bd68808e9c361 Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Thu, 13 Aug 2020 21:48:48 +0300 Subject: [PATCH 103/139] Rebased --- .git_hooks/pre-commit | 1 - git_hooks/pre-commit | 19 ------------------- 2 files changed, 20 deletions(-) delete mode 100755 git_hooks/pre-commit diff --git a/.git_hooks/pre-commit b/.git_hooks/pre-commit index 105684ef7..872c2cdda 100755 --- a/.git_hooks/pre-commit +++ b/.git_hooks/pre-commit @@ -16,4 +16,3 @@ fi printf "\n" printf "\033[0;32mpre-commit hook succeeded\033[0m\n" exit 0 - diff --git a/git_hooks/pre-commit b/git_hooks/pre-commit deleted file mode 100755 index f6855e88e..000000000 --- a/git_hooks/pre-commit +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env sh -cargo fmt -- --check -if [ "$?" != "0" ] ; then - printf "\n" - printf "\033[0;31mpre-commit hook failed during:\033[0m\n" - printf "\033[0;31m\tcargo fmt -- --check\033[0m\n" - exit 1 -fi -cargo clippy --locked -- -D warnings -if [ "$?" != "0" ] ; then - printf "\n" - printf "\033[0;31mpre-commit hook failed during:\033[0m\n" - printf "\033[0;31m\tclippy --locked -- -D warning\033[0m\n" - exit 1 -fi - -printf "\n" -printf "\033[0;32mpre-commit hook succeeded\033[0m\n" -exit 0 \ No newline at end of file From 2868deab5b155d6ffaa4a601758c39e15e96c307 Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Thu, 13 Aug 2020 15:58:30 +0300 Subject: [PATCH 104/139] Added newline at end of pre-commit script --- .git_hooks/pre-commit | 1 + 1 file changed, 1 insertion(+) diff --git a/.git_hooks/pre-commit b/.git_hooks/pre-commit index 872c2cdda..105684ef7 100755 --- a/.git_hooks/pre-commit +++ b/.git_hooks/pre-commit @@ -16,3 +16,4 @@ fi printf "\n" printf "\033[0;32mpre-commit hook succeeded\033[0m\n" exit 0 + From 6087b6159d55790bd3a496c7c1a7217efc25177d Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Thu, 13 Aug 2020 22:16:15 +0300 Subject: [PATCH 105/139] Rebased --- src/utils/daemon.rs | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs index 7b886eab8..d142fa544 100644 --- a/src/utils/daemon.rs +++ b/src/utils/daemon.rs @@ -9,16 +9,13 @@ use crate::{ use chrono::{Timelike, Utc}; use failure::Error; use log::{debug, error, info}; -use std::sync::Arc; use std::thread; use std::time::{Duration, Instant}; -fn start_registry_watcher( - opts: DocBuilderOptions, - pool: Pool, - build_queue: Arc, - config: Arc, -) -> Result<(), Error> { +fn start_registry_watcher(opts: DocBuilderOptions, context: &dyn Context) -> Result<(), Error> { + let pool = context.pool()?; + let build_queue = context.build_queue()?; + let config = context.config()?; thread::Builder::new() .name("registry index reader".to_string()) .spawn(move || { @@ -60,24 +57,18 @@ pub fn start_daemon(context: &dyn Context, enable_registry_watcher: bool) -> Res if enable_registry_watcher { // check new crates every minute - start_registry_watcher( - dbopts.clone(), - db.clone(), - build_queue.clone(), - config.clone(), - )?; + start_registry_watcher(dbopts.clone(), context)?; } // build new crates every minute let pool = context.pool()?; let build_queue = context.build_queue()?; let storage = context.storage()?; - let metrics = context.metrics()?; thread::Builder::new() .name("build queue reader".to_string()) .spawn(move || { let doc_builder = DocBuilder::new(dbopts.clone(), pool.clone(), build_queue.clone()); - queue_builder(doc_builder, pool, build_queue, metrics, storage).unwrap(); + queue_builder(doc_builder, pool, build_queue, storage).unwrap(); }) .unwrap(); From c69d553c5d11cd883a1b1b0b0b2765879de06f84 Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Thu, 13 Aug 2020 22:23:40 +0300 Subject: [PATCH 106/139] Rebased --- src/metrics/macros.rs | 23 ++----- src/metrics/mod.rs | 4 +- src/utils/daemon.rs | 3 +- src/web/metrics.rs | 144 ------------------------------------------ 4 files changed, 7 insertions(+), 167 deletions(-) diff --git a/src/metrics/macros.rs b/src/metrics/macros.rs index 2571f5b1e..c79bd60df 100644 --- a/src/metrics/macros.rs +++ b/src/metrics/macros.rs @@ -6,42 +6,27 @@ pub(super) trait MetricFromOpts: Sized { macro_rules! metrics { ( $vis:vis struct $name:ident { - $( - #[doc = $help:expr] - $(#[$meta:meta])* - $metric_vis:vis $metric:ident: $ty:ty $([$($label:expr),* $(,)?])? - ),* $(,)? + $(#[doc = $help:expr] $metric:ident: $ty:ty $([$($label:expr),*])?,)* } metrics visibility: $metric_vis:vis, namespace: $namespace:expr, ) => { $vis struct $name { - registry: prometheus::Registry, - $( - $(#[$meta])* - $metric_vis $metric: $ty, - )* + registry: Registry, + $($metric_vis $metric: $ty,)* } impl $name { $vis fn new() -> Result { let registry = Registry::new(); $( - $(#[$meta])* let $metric = <$ty>::from_opts( Opts::new(stringify!($metric), $help) .namespace($namespace) $(.variable_labels(vec![$($label.into()),*]))? )?; - $(#[$meta])* registry.register(Box::new($metric.clone()))?; )* - Ok(Self { - registry, - $( - $(#[$meta])* - $metric, - )* - }) + Ok(Self { registry, $($metric,)* }) } } impl std::fmt::Debug for $name { diff --git a/src/metrics/mod.rs b/src/metrics/mod.rs index 3616ef657..870109a7e 100644 --- a/src/metrics/mod.rs +++ b/src/metrics/mod.rs @@ -31,10 +31,8 @@ metrics! { failed_db_connections: IntCounter, /// The number of currently opened file descriptors - #[cfg(linux)] open_file_descriptors: IntGauge, /// The number of threads being used by docs.rs - #[cfg(linux)] running_threads: IntGauge, /// The traffic of various docs.rs routes @@ -86,7 +84,7 @@ impl Metrics { #[cfg(not(linux))] fn gather_system_performance(&self) {} - #[cfg(linux)] + #[cfg(target_os = "linux")] fn gather_system_performance(&self) { use procfs::process::Process; diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs index d142fa544..673f05204 100644 --- a/src/utils/daemon.rs +++ b/src/utils/daemon.rs @@ -64,11 +64,12 @@ pub fn start_daemon(context: &dyn Context, enable_registry_watcher: bool) -> Res let pool = context.pool()?; let build_queue = context.build_queue()?; let storage = context.storage()?; + let metrics = context.metrics()?; thread::Builder::new() .name("build queue reader".to_string()) .spawn(move || { let doc_builder = DocBuilder::new(dbopts.clone(), pool.clone(), build_queue.clone()); - queue_builder(doc_builder, pool, build_queue, storage).unwrap(); + queue_builder(doc_builder, pool, build_queue, metrics, storage).unwrap(); }) .unwrap(); diff --git a/src/web/metrics.rs b/src/web/metrics.rs index 2f6dccf9d..dbb910499 100644 --- a/src/web/metrics.rs +++ b/src/web/metrics.rs @@ -7,150 +7,6 @@ use iron::status::Status; use prometheus::{Encoder, HistogramVec, TextEncoder}; use std::time::{Duration, Instant}; -static QUEUED_CRATES_COUNT: Lazy = Lazy::new(|| { - register_int_gauge!( - "docsrs_queued_crates_count", - "Number of crates in the build queue" - ) - .unwrap() -}); - -pub static PRIORITIZED_CRATES_COUNT: Lazy = Lazy::new(|| { - register_int_gauge!( - "docsrs_prioritized_crates_count", - "Number of crates in the build queue that have a positive priority" - ) - .unwrap() -}); - -static FAILED_CRATES_COUNT: Lazy = Lazy::new(|| { - register_int_gauge!( - "docsrs_failed_crates_count", - "Number of crates that failed to build" - ) - .unwrap() -}); - -pub static TOTAL_BUILDS: Lazy = - Lazy::new(|| register_int_counter!("docsrs_total_builds", "Number of crates built").unwrap()); - -pub static SUCCESSFUL_BUILDS: Lazy = Lazy::new(|| { - register_int_counter!( - "docsrs_successful_builds", - "Number of builds that successfully generated docs" - ) - .unwrap() -}); - -pub static FAILED_BUILDS: Lazy = Lazy::new(|| { - register_int_counter!( - "docsrs_failed_builds", - "Number of builds that generated a compile error" - ) - .unwrap() -}); - -pub static NON_LIBRARY_BUILDS: Lazy = Lazy::new(|| { - register_int_counter!( - "docsrs_non_library_builds", - "Number of builds that did not complete due to not being a library" - ) - .unwrap() -}); - -pub static UPLOADED_FILES_TOTAL: Lazy = Lazy::new(|| { - register_int_counter!( - "docsrs_uploaded_files_total", - "Number of files uploaded to S3 or stored in the database" - ) - .unwrap() -}); - -pub static ROUTES_VISITED: Lazy = Lazy::new(|| { - register_int_counter_vec!( - "docsrs_routes_visited", - "The traffic of various docs.rs routes", - &["route"] - ) - .unwrap() -}); - -pub static RESPONSE_TIMES: Lazy = Lazy::new(|| { - register_histogram_vec!( - "docsrs_response_time", - "The response times of various docs.rs routes", - &["route"] - ) - .unwrap() -}); - -pub static RUSTDOC_RENDERING_TIMES: Lazy = Lazy::new(|| { - register_histogram_vec!( - "docsrs_rustdoc_rendering_time", - "The time it takes to render a rustdoc page", - &["step"] - ) - .unwrap() -}); - -pub static FAILED_DB_CONNECTIONS: Lazy = Lazy::new(|| { - register_int_counter!( - "docsrs_failed_db_connections", - "Number of attempted and failed connections to the database" - ) - .unwrap() -}); - -pub static USED_DB_CONNECTIONS: Lazy = Lazy::new(|| { - register_int_gauge!( - "docsrs_used_db_connections", - "The number of used database connections" - ) - .unwrap() -}); - -pub static IDLE_DB_CONNECTIONS: Lazy = Lazy::new(|| { - register_int_gauge!( - "docsrs_idle_db_connections", - "The number of idle database connections" - ) - .unwrap() -}); - -pub static MAX_DB_CONNECTIONS: Lazy = Lazy::new(|| { - register_int_gauge!( - "docsrs_max_db_connections", - "The maximum database connections" - ) - .unwrap() -}); - -#[cfg(target_os = "linux")] -pub static OPEN_FILE_DESCRIPTORS: Lazy = Lazy::new(|| { - register_int_gauge!( - "docsrs_open_file_descriptors", - "The number of currently opened file descriptors" - ) - .unwrap() -}); - -#[cfg(target_os = "linux")] -pub static CURRENTLY_RUNNING_THREADS: Lazy = Lazy::new(|| { - register_int_gauge!( - "docsrs_running_threads", - "The number of threads being used by docs.rs" - ) - .unwrap() -}); - -pub static HTML_REWRITE_OOMS: Lazy = Lazy::new(|| { - register_int_gauge!( - "docsrs_html_rewrite_ooms", - "The number of attempted files that failed due to a memory limit" - ) - .unwrap() -}); - pub fn metrics_handler(req: &mut Request) -> IronResult { let metrics = extension!(req, Metrics); let pool = extension!(req, Pool); From 29412b9ad46b9efdc3aaf1921b91eb51f20a21df Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Thu, 13 Aug 2020 22:24:39 +0300 Subject: [PATCH 107/139] Rebased --- src/metrics/macros.rs | 23 +++++++++++++++++++---- src/metrics/mod.rs | 4 +++- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/src/metrics/macros.rs b/src/metrics/macros.rs index c79bd60df..2571f5b1e 100644 --- a/src/metrics/macros.rs +++ b/src/metrics/macros.rs @@ -6,27 +6,42 @@ pub(super) trait MetricFromOpts: Sized { macro_rules! metrics { ( $vis:vis struct $name:ident { - $(#[doc = $help:expr] $metric:ident: $ty:ty $([$($label:expr),*])?,)* + $( + #[doc = $help:expr] + $(#[$meta:meta])* + $metric_vis:vis $metric:ident: $ty:ty $([$($label:expr),* $(,)?])? + ),* $(,)? } metrics visibility: $metric_vis:vis, namespace: $namespace:expr, ) => { $vis struct $name { - registry: Registry, - $($metric_vis $metric: $ty,)* + registry: prometheus::Registry, + $( + $(#[$meta])* + $metric_vis $metric: $ty, + )* } impl $name { $vis fn new() -> Result { let registry = Registry::new(); $( + $(#[$meta])* let $metric = <$ty>::from_opts( Opts::new(stringify!($metric), $help) .namespace($namespace) $(.variable_labels(vec![$($label.into()),*]))? )?; + $(#[$meta])* registry.register(Box::new($metric.clone()))?; )* - Ok(Self { registry, $($metric,)* }) + Ok(Self { + registry, + $( + $(#[$meta])* + $metric, + )* + }) } } impl std::fmt::Debug for $name { diff --git a/src/metrics/mod.rs b/src/metrics/mod.rs index 870109a7e..3616ef657 100644 --- a/src/metrics/mod.rs +++ b/src/metrics/mod.rs @@ -31,8 +31,10 @@ metrics! { failed_db_connections: IntCounter, /// The number of currently opened file descriptors + #[cfg(linux)] open_file_descriptors: IntGauge, /// The number of threads being used by docs.rs + #[cfg(linux)] running_threads: IntGauge, /// The traffic of various docs.rs routes @@ -84,7 +86,7 @@ impl Metrics { #[cfg(not(linux))] fn gather_system_performance(&self) {} - #[cfg(target_os = "linux")] + #[cfg(linux)] fn gather_system_performance(&self) { use procfs::process::Process; From c063e0c5a55e4b5d3a2798cdb848c0c3ca65298b Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Thu, 13 Aug 2020 22:28:00 +0300 Subject: [PATCH 108/139] Rebased --- src/config.rs | 2 +- src/utils/daemon.rs | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/config.rs b/src/config.rs index a3ae7dc9f..d84f12872 100644 --- a/src/config.rs +++ b/src/config.rs @@ -30,7 +30,7 @@ pub struct Config { pub(crate) max_file_size_html: usize, // The most memory that can be used to parse an HTML file pub(crate) max_parse_memory: usize, - // Time between 'git gc' calls in seconds + // Time between 'git gc --auto' calls in seconds pub(crate) registry_gc_interval: u64, } diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs index 673f05204..af8436c94 100644 --- a/src/utils/daemon.rs +++ b/src/utils/daemon.rs @@ -22,7 +22,6 @@ fn start_registry_watcher(opts: DocBuilderOptions, context: &dyn Context) -> Res // space this out to prevent it from clashing against the queue-builder thread on launch thread::sleep(Duration::from_secs(30)); let mut last_gc = Instant::now(); - loop { let mut doc_builder = DocBuilder::new(opts.clone(), pool.clone(), build_queue.clone()); From 7374345390b8fdd12bc6be5e704d23bbcbe7e6dc Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Thu, 13 Aug 2020 22:28:39 +0300 Subject: [PATCH 109/139] Rebased --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index d7823cbaf..5b7d9caab 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ target .vagrant .rustwide .rustwide-docker +.idea/ \ No newline at end of file From def42a4298976816b542b2282abfde2761e4beb1 Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Thu, 13 Aug 2020 22:30:47 +0300 Subject: [PATCH 110/139] Rebased --- git_hooks/pre-commit | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100755 git_hooks/pre-commit diff --git a/git_hooks/pre-commit b/git_hooks/pre-commit new file mode 100755 index 000000000..8ae45b3f3 --- /dev/null +++ b/git_hooks/pre-commit @@ -0,0 +1,19 @@ +#!/usr/bin/env sh +cargo fmt -- --check +if [ "$?" != "0" ] ; then + echo "\n" + echo "\033[0;31mpre-commit hook failed during:\033[0m" + echo "\033[0;31m\tcargo fmt -- --check\033[0m" + exit 1 +fi +cargo clippy --locked -- -D warnings +if [ "$?" != "0" ] ; then + echo "\n" + echo "\033[0;31mpre-commit hook failed during:\033[0m" + echo "\033[0;31m\tclippy --locked -- -D warning\033[0m" + exit 1 +fi + +echo "\n" +echo "\033[0;32mpre-commit hook succeeded\033[0m" +exit 0 \ No newline at end of file From fe68adba0c2ada6c8500851460a06017f135a4b7 Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Tue, 11 Aug 2020 22:09:34 +0300 Subject: [PATCH 111/139] Run git gc periodically on the crates.io index #778 --- src/config.rs | 3 ++- src/utils/daemon.rs | 26 ++++++++++++++++++++------ 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/src/config.rs b/src/config.rs index d84f12872..497ad575d 100644 --- a/src/config.rs +++ b/src/config.rs @@ -30,6 +30,7 @@ pub struct Config { pub(crate) max_file_size_html: usize, // The most memory that can be used to parse an HTML file pub(crate) max_parse_memory: usize, + // Time between 'git gc --auto' calls in seconds pub(crate) registry_gc_interval: u64, } @@ -63,7 +64,7 @@ impl Config { // LOL HTML only uses as much memory as the size of the start tag! // https://github.com/rust-lang/docs.rs/pull/930#issuecomment-667729380 max_parse_memory: env("DOCSRS_MAX_PARSE_MEMORY", 5 * 1024 * 1024)?, - registry_gc_interval: env("DOCSRS_REGISTRY_GC_INTERVAL", 60 * 60)?, + registry_gc_interval: env("REGISTRY_GC_INTERVAL", 60 * 60)?, }) } diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs index af8436c94..68276ec54 100644 --- a/src/utils/daemon.rs +++ b/src/utils/daemon.rs @@ -11,17 +11,30 @@ use failure::Error; use log::{debug, error, info}; use std::thread; use std::time::{Duration, Instant}; +use std::process::Command; -fn start_registry_watcher(opts: DocBuilderOptions, context: &dyn Context) -> Result<(), Error> { - let pool = context.pool()?; - let build_queue = context.build_queue()?; - let config = context.config()?; + +fn run_git_gc() { + Command::new("git") + .args(&["gc"]) + .output() + .expect("Failed to execute git gc"); +} + +fn start_registry_watcher( + opts: DocBuilderOptions, + pool: Pool, + build_queue: Arc, + config: Arc, +) -> Result<(), Error> { thread::Builder::new() .name("registry index reader".to_string()) .spawn(move || { // space this out to prevent it from clashing against the queue-builder thread on launch thread::sleep(Duration::from_secs(30)); let mut last_gc = Instant::now(); + run_git_gc(); + loop { let mut doc_builder = DocBuilder::new(opts.clone(), pool.clone(), build_queue.clone()); @@ -37,7 +50,7 @@ fn start_registry_watcher(opts: DocBuilderOptions, context: &dyn Context) -> Res } if last_gc.elapsed().as_secs() >= config.registry_gc_interval { - doc_builder.run_git_gc(); + run_git_gc(); last_gc = Instant::now(); } thread::sleep(Duration::from_secs(60)); @@ -56,7 +69,8 @@ pub fn start_daemon(context: &dyn Context, enable_registry_watcher: bool) -> Res if enable_registry_watcher { // check new crates every minute - start_registry_watcher(dbopts.clone(), context)?; + start_registry_watcher(dbopts.clone(), db.clone(), build_queue.clone(), + config.clone())?; } // build new crates every minute From 29ecb456973715b0592f2e08ba68d0612860d5d3 Mon Sep 17 00:00:00 2001 From: ohaddahan Date: Tue, 11 Aug 2020 22:14:34 +0300 Subject: [PATCH 112/139] Update config.rs --- src/config.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config.rs b/src/config.rs index 497ad575d..871002745 100644 --- a/src/config.rs +++ b/src/config.rs @@ -31,7 +31,7 @@ pub struct Config { // The most memory that can be used to parse an HTML file pub(crate) max_parse_memory: usize, - // Time between 'git gc --auto' calls in seconds + // Time between 'git gc' calls in seconds pub(crate) registry_gc_interval: u64, } From d2c20c47a8972ebd56dd4808f7bcbf854305a246 Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Tue, 11 Aug 2020 22:20:18 +0300 Subject: [PATCH 113/139] Fixed "cargo fmt" --- src/utils/daemon.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs index 68276ec54..bb086db06 100644 --- a/src/utils/daemon.rs +++ b/src/utils/daemon.rs @@ -9,10 +9,10 @@ use crate::{ use chrono::{Timelike, Utc}; use failure::Error; use log::{debug, error, info}; +use std::process::Command; +use std::sync::Arc; use std::thread; use std::time::{Duration, Instant}; -use std::process::Command; - fn run_git_gc() { Command::new("git") @@ -69,8 +69,12 @@ pub fn start_daemon(context: &dyn Context, enable_registry_watcher: bool) -> Res if enable_registry_watcher { // check new crates every minute - start_registry_watcher(dbopts.clone(), db.clone(), build_queue.clone(), - config.clone())?; + start_registry_watcher( + dbopts.clone(), + db.clone(), + build_queue.clone(), + config.clone(), + )?; } // build new crates every minute From 3214302b4156654e5de3a5af79f22c9a7629bd0d Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Wed, 12 Aug 2020 09:01:44 +0300 Subject: [PATCH 114/139] Added --auto --- src/utils/daemon.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs index bb086db06..c993d4c16 100644 --- a/src/utils/daemon.rs +++ b/src/utils/daemon.rs @@ -16,7 +16,7 @@ use std::time::{Duration, Instant}; fn run_git_gc() { Command::new("git") - .args(&["gc"]) + .args(&["gc", "--auto"]) .output() .expect("Failed to execute git gc"); } From d072f02f5406a91a2dc1fcf9ad7216207c980001 Mon Sep 17 00:00:00 2001 From: ohaddahan Date: Tue, 11 Aug 2020 23:22:02 +0300 Subject: [PATCH 115/139] Update src/utils/daemon.rs Co-authored-by: Chase Wilson --- src/utils/daemon.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs index c993d4c16..db70b1cad 100644 --- a/src/utils/daemon.rs +++ b/src/utils/daemon.rs @@ -32,8 +32,8 @@ fn start_registry_watcher( .spawn(move || { // space this out to prevent it from clashing against the queue-builder thread on launch thread::sleep(Duration::from_secs(30)); - let mut last_gc = Instant::now(); run_git_gc(); + let mut last_gc = Instant::now(); loop { let mut doc_builder = From cb406bebb5e4bfe2ad601b2b2ec482e353f5c36e Mon Sep 17 00:00:00 2001 From: ohaddahan Date: Tue, 11 Aug 2020 23:22:11 +0300 Subject: [PATCH 116/139] Update src/utils/daemon.rs Co-authored-by: Chase Wilson --- src/utils/daemon.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs index db70b1cad..de8700fc5 100644 --- a/src/utils/daemon.rs +++ b/src/utils/daemon.rs @@ -15,10 +15,13 @@ use std::thread; use std::time::{Duration, Instant}; fn run_git_gc() { - Command::new("git") - .args(&["gc", "--auto"]) - .output() - .expect("Failed to execute git gc"); + let gc = Command::new("git") + .args(&["gc"]) + .output(); + + if let Err(err) = gc { + log::error!("failed to run `git gc`: {:?}", err); + } } fn start_registry_watcher( From a6a5c65fc3bc3cbc7f84a6509db4eedd28ef1ffd Mon Sep 17 00:00:00 2001 From: ohaddahan Date: Tue, 11 Aug 2020 23:22:27 +0300 Subject: [PATCH 117/139] Update src/utils/daemon.rs Co-authored-by: Chase Wilson --- src/utils/daemon.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs index de8700fc5..299e3a88f 100644 --- a/src/utils/daemon.rs +++ b/src/utils/daemon.rs @@ -16,7 +16,7 @@ use std::time::{Duration, Instant}; fn run_git_gc() { let gc = Command::new("git") - .args(&["gc"]) + .args(&["gc", "--auto"]) .output(); if let Err(err) = gc { From 28065e1b7a5e60e65d18151ab5b20d10c361fd19 Mon Sep 17 00:00:00 2001 From: ohaddahan Date: Tue, 11 Aug 2020 23:22:36 +0300 Subject: [PATCH 118/139] Update src/config.rs Co-authored-by: Chase Wilson --- src/config.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/config.rs b/src/config.rs index 871002745..86fc11db5 100644 --- a/src/config.rs +++ b/src/config.rs @@ -30,7 +30,6 @@ pub struct Config { pub(crate) max_file_size_html: usize, // The most memory that can be used to parse an HTML file pub(crate) max_parse_memory: usize, - // Time between 'git gc' calls in seconds pub(crate) registry_gc_interval: u64, } From be9e3e0c63067995fea7d08fda85a9f787c33e1b Mon Sep 17 00:00:00 2001 From: ohaddahan Date: Tue, 11 Aug 2020 23:22:43 +0300 Subject: [PATCH 119/139] Update .gitignore Co-authored-by: Chase Wilson --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index 5b7d9caab..d7823cbaf 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,3 @@ target .vagrant .rustwide .rustwide-docker -.idea/ \ No newline at end of file From 89578dd18ce5df441c5a8cf0638912d018700858 Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Wed, 12 Aug 2020 09:11:54 +0300 Subject: [PATCH 120/139] Fix cargo fmt --- src/utils/daemon.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs index 299e3a88f..071ba3154 100644 --- a/src/utils/daemon.rs +++ b/src/utils/daemon.rs @@ -15,10 +15,8 @@ use std::thread; use std::time::{Duration, Instant}; fn run_git_gc() { - let gc = Command::new("git") - .args(&["gc", "--auto"]) - .output(); - + let gc = Command::new("git").args(&["gc", "--auto"]).output(); + if let Err(err) = gc { log::error!("failed to run `git gc`: {:?}", err); } From 87ed97deefdb8c8d37ae98dff2a42715f31f8d70 Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Wed, 12 Aug 2020 09:51:30 +0300 Subject: [PATCH 121/139] Adding pre-commit hook, adding dead-code lint since without it clippy rails with "unused static" --- README.md | 2 +- src/web/metrics.rs | 146 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 147 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index c84156f16..53775185d 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ This gives you reasonable incremental build times without having to add new user ### Git Hooks For ease of use, `git_hooks` directory contains useful `git hooks` to make your development easier. ```bash -cd .git/hooks && ln -s ../../.git_hooks/* . && cd ../.. +cp git_hooks/* .git/hooks/ ``` ### Dependencies diff --git a/src/web/metrics.rs b/src/web/metrics.rs index dbb910499..8419ee576 100644 --- a/src/web/metrics.rs +++ b/src/web/metrics.rs @@ -7,6 +7,152 @@ use iron::status::Status; use prometheus::{Encoder, HistogramVec, TextEncoder}; use std::time::{Duration, Instant}; +static QUEUED_CRATES_COUNT: Lazy = Lazy::new(|| { + register_int_gauge!( + "docsrs_queued_crates_count", + "Number of crates in the build queue" + ) + .unwrap() +}); + +pub static PRIORITIZED_CRATES_COUNT: Lazy = Lazy::new(|| { + register_int_gauge!( + "docsrs_prioritized_crates_count", + "Number of crates in the build queue that have a positive priority" + ) + .unwrap() +}); + +static FAILED_CRATES_COUNT: Lazy = Lazy::new(|| { + register_int_gauge!( + "docsrs_failed_crates_count", + "Number of crates that failed to build" + ) + .unwrap() +}); + +pub static TOTAL_BUILDS: Lazy = + Lazy::new(|| register_int_counter!("docsrs_total_builds", "Number of crates built").unwrap()); + +pub static SUCCESSFUL_BUILDS: Lazy = Lazy::new(|| { + register_int_counter!( + "docsrs_successful_builds", + "Number of builds that successfully generated docs" + ) + .unwrap() +}); + +pub static FAILED_BUILDS: Lazy = Lazy::new(|| { + register_int_counter!( + "docsrs_failed_builds", + "Number of builds that generated a compile error" + ) + .unwrap() +}); + +pub static NON_LIBRARY_BUILDS: Lazy = Lazy::new(|| { + register_int_counter!( + "docsrs_non_library_builds", + "Number of builds that did not complete due to not being a library" + ) + .unwrap() +}); + +pub static UPLOADED_FILES_TOTAL: Lazy = Lazy::new(|| { + register_int_counter!( + "docsrs_uploaded_files_total", + "Number of files uploaded to S3 or stored in the database" + ) + .unwrap() +}); + +pub static ROUTES_VISITED: Lazy = Lazy::new(|| { + register_int_counter_vec!( + "docsrs_routes_visited", + "The traffic of various docs.rs routes", + &["route"] + ) + .unwrap() +}); + +pub static RESPONSE_TIMES: Lazy = Lazy::new(|| { + register_histogram_vec!( + "docsrs_response_time", + "The response times of various docs.rs routes", + &["route"] + ) + .unwrap() +}); + +pub static RUSTDOC_RENDERING_TIMES: Lazy = Lazy::new(|| { + register_histogram_vec!( + "docsrs_rustdoc_rendering_time", + "The time it takes to render a rustdoc page", + &["step"] + ) + .unwrap() +}); + +pub static FAILED_DB_CONNECTIONS: Lazy = Lazy::new(|| { + register_int_counter!( + "docsrs_failed_db_connections", + "Number of attempted and failed connections to the database" + ) + .unwrap() +}); + +pub static USED_DB_CONNECTIONS: Lazy = Lazy::new(|| { + register_int_gauge!( + "docsrs_used_db_connections", + "The number of used database connections" + ) + .unwrap() +}); + +pub static IDLE_DB_CONNECTIONS: Lazy = Lazy::new(|| { + register_int_gauge!( + "docsrs_idle_db_connections", + "The number of idle database connections" + ) + .unwrap() +}); + +pub static MAX_DB_CONNECTIONS: Lazy = Lazy::new(|| { + register_int_gauge!( + "docsrs_max_db_connections", + "The maximum database connections" + ) + .unwrap() +}); + +#[cfg(not(windows))] +#[allow(dead_code)] +pub static OPEN_FILE_DESCRIPTORS: Lazy = Lazy::new(|| { + register_int_gauge!( + "docsrs_open_file_descriptors", + "The number of currently opened file descriptors" + ) + .unwrap() +}); + +#[cfg(not(windows))] +#[allow(dead_code)] +pub static CURRENTLY_RUNNING_THREADS: Lazy = Lazy::new(|| { + register_int_gauge!( + "docsrs_running_threads", + "The number of threads being used by docs.rs" + ) + .unwrap() +}); + +pub static HTML_REWRITE_OOMS: Lazy = Lazy::new(|| { + register_int_gauge!( + "docsrs_html_rewrite_ooms", + "The number of attempted files that failed due to a memory limit" + ) + .unwrap() +}); + pub fn metrics_handler(req: &mut Request) -> IronResult { let metrics = extension!(req, Metrics); let pool = extension!(req, Pool); From 2bf3ae3fefa3fe07715a36cbd1907cf17e33009e Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Wed, 12 Aug 2020 19:59:52 +0300 Subject: [PATCH 122/139] Changed envar prefix and replace echo with printf for pre-commit hook --- git_hooks/pre-commit | 16 ++++++++-------- src/config.rs | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/git_hooks/pre-commit b/git_hooks/pre-commit index 8ae45b3f3..f6855e88e 100755 --- a/git_hooks/pre-commit +++ b/git_hooks/pre-commit @@ -1,19 +1,19 @@ #!/usr/bin/env sh cargo fmt -- --check if [ "$?" != "0" ] ; then - echo "\n" - echo "\033[0;31mpre-commit hook failed during:\033[0m" - echo "\033[0;31m\tcargo fmt -- --check\033[0m" + printf "\n" + printf "\033[0;31mpre-commit hook failed during:\033[0m\n" + printf "\033[0;31m\tcargo fmt -- --check\033[0m\n" exit 1 fi cargo clippy --locked -- -D warnings if [ "$?" != "0" ] ; then - echo "\n" - echo "\033[0;31mpre-commit hook failed during:\033[0m" - echo "\033[0;31m\tclippy --locked -- -D warning\033[0m" + printf "\n" + printf "\033[0;31mpre-commit hook failed during:\033[0m\n" + printf "\033[0;31m\tclippy --locked -- -D warning\033[0m\n" exit 1 fi -echo "\n" -echo "\033[0;32mpre-commit hook succeeded\033[0m" +printf "\n" +printf "\033[0;32mpre-commit hook succeeded\033[0m\n" exit 0 \ No newline at end of file diff --git a/src/config.rs b/src/config.rs index 86fc11db5..a3ae7dc9f 100644 --- a/src/config.rs +++ b/src/config.rs @@ -63,7 +63,7 @@ impl Config { // LOL HTML only uses as much memory as the size of the start tag! // https://github.com/rust-lang/docs.rs/pull/930#issuecomment-667729380 max_parse_memory: env("DOCSRS_MAX_PARSE_MEMORY", 5 * 1024 * 1024)?, - registry_gc_interval: env("REGISTRY_GC_INTERVAL", 60 * 60)?, + registry_gc_interval: env("DOCSRS_REGISTRY_GC_INTERVAL", 60 * 60)?, }) } From a9efd0531b49629ea90e762c8879cef69d032314 Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Wed, 12 Aug 2020 20:33:36 +0300 Subject: [PATCH 123/139] Removed allow(dead_code) and not(windows) , now OPEN_FILE_DESCRIPTORS and CURRENTLY_RUNNING_THREADS declaration match the usage, "target_os = linux" --- src/web/metrics.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/web/metrics.rs b/src/web/metrics.rs index 8419ee576..2f6dccf9d 100644 --- a/src/web/metrics.rs +++ b/src/web/metrics.rs @@ -125,8 +125,7 @@ pub static MAX_DB_CONNECTIONS: Lazy = Lazy::new(|| { .unwrap() }); -#[cfg(not(windows))] -#[allow(dead_code)] +#[cfg(target_os = "linux")] pub static OPEN_FILE_DESCRIPTORS: Lazy = Lazy::new(|| { register_int_gauge!( "docsrs_open_file_descriptors", @@ -135,8 +134,7 @@ pub static OPEN_FILE_DESCRIPTORS: Lazy = Lazy::new(|| { .unwrap() }); -#[cfg(not(windows))] -#[allow(dead_code)] +#[cfg(target_os = "linux")] pub static CURRENTLY_RUNNING_THREADS: Lazy = Lazy::new(|| { register_int_gauge!( "docsrs_running_threads", From d7c0ec1e734207b14045e1aafe442cd245740e7c Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Thu, 13 Aug 2020 15:57:22 +0300 Subject: [PATCH 124/139] Updated pre-commit script and instructions --- .git_hooks/pre-commit | 1 - README.md | 2 +- git_hooks/pre-commit | 19 ------------------- 3 files changed, 1 insertion(+), 21 deletions(-) delete mode 100755 git_hooks/pre-commit diff --git a/.git_hooks/pre-commit b/.git_hooks/pre-commit index 105684ef7..872c2cdda 100755 --- a/.git_hooks/pre-commit +++ b/.git_hooks/pre-commit @@ -16,4 +16,3 @@ fi printf "\n" printf "\033[0;32mpre-commit hook succeeded\033[0m\n" exit 0 - diff --git a/README.md b/README.md index 53775185d..c84156f16 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ This gives you reasonable incremental build times without having to add new user ### Git Hooks For ease of use, `git_hooks` directory contains useful `git hooks` to make your development easier. ```bash -cp git_hooks/* .git/hooks/ +cd .git/hooks && ln -s ../../.git_hooks/* . && cd ../.. ``` ### Dependencies diff --git a/git_hooks/pre-commit b/git_hooks/pre-commit deleted file mode 100755 index f6855e88e..000000000 --- a/git_hooks/pre-commit +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env sh -cargo fmt -- --check -if [ "$?" != "0" ] ; then - printf "\n" - printf "\033[0;31mpre-commit hook failed during:\033[0m\n" - printf "\033[0;31m\tcargo fmt -- --check\033[0m\n" - exit 1 -fi -cargo clippy --locked -- -D warnings -if [ "$?" != "0" ] ; then - printf "\n" - printf "\033[0;31mpre-commit hook failed during:\033[0m\n" - printf "\033[0;31m\tclippy --locked -- -D warning\033[0m\n" - exit 1 -fi - -printf "\n" -printf "\033[0;32mpre-commit hook succeeded\033[0m\n" -exit 0 \ No newline at end of file From a7568a2b1972c0e3ffd65fc81c82f65517afce51 Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Thu, 13 Aug 2020 15:58:30 +0300 Subject: [PATCH 125/139] Added newline at end of pre-commit script --- .git_hooks/pre-commit | 1 + 1 file changed, 1 insertion(+) diff --git a/.git_hooks/pre-commit b/.git_hooks/pre-commit index 872c2cdda..105684ef7 100755 --- a/.git_hooks/pre-commit +++ b/.git_hooks/pre-commit @@ -16,3 +16,4 @@ fi printf "\n" printf "\033[0;32mpre-commit hook succeeded\033[0m\n" exit 0 + From db8ce50e4d5fd3d5103ea9978f4717f887d0597c Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Thu, 13 Aug 2020 17:54:29 +0300 Subject: [PATCH 126/139] Move run_git_gc to Index --- src/index/mod.rs | 2 +- src/utils/daemon.rs | 12 +----------- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/src/index/mod.rs b/src/index/mod.rs index e00c2c357..a4fef6e0e 100644 --- a/src/index/mod.rs +++ b/src/index/mod.rs @@ -52,7 +52,7 @@ impl Index { } pub fn run_git_gc(&self) { - let cmd = format!("cd {} && git gc --auto", self.path.to_str().unwrap()); + let cmd = format!("cd {} && gc --auto", self.path.to_str().unwrap()); let gc = Command::new("sh").args(&["-c", cmd.as_str()]).output(); if let Err(err) = gc { log::error!("Failed to run `{}`: {:?}", cmd, err); diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs index 071ba3154..7b886eab8 100644 --- a/src/utils/daemon.rs +++ b/src/utils/daemon.rs @@ -9,19 +9,10 @@ use crate::{ use chrono::{Timelike, Utc}; use failure::Error; use log::{debug, error, info}; -use std::process::Command; use std::sync::Arc; use std::thread; use std::time::{Duration, Instant}; -fn run_git_gc() { - let gc = Command::new("git").args(&["gc", "--auto"]).output(); - - if let Err(err) = gc { - log::error!("failed to run `git gc`: {:?}", err); - } -} - fn start_registry_watcher( opts: DocBuilderOptions, pool: Pool, @@ -33,7 +24,6 @@ fn start_registry_watcher( .spawn(move || { // space this out to prevent it from clashing against the queue-builder thread on launch thread::sleep(Duration::from_secs(30)); - run_git_gc(); let mut last_gc = Instant::now(); loop { @@ -51,7 +41,7 @@ fn start_registry_watcher( } if last_gc.elapsed().as_secs() >= config.registry_gc_interval { - run_git_gc(); + doc_builder.run_git_gc(); last_gc = Instant::now(); } thread::sleep(Duration::from_secs(60)); From 4121fb68d68df99b4cdff4dc63867e5b08801ffe Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Thu, 13 Aug 2020 19:48:32 +0300 Subject: [PATCH 127/139] Fix typo --- src/index/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/index/mod.rs b/src/index/mod.rs index a4fef6e0e..e00c2c357 100644 --- a/src/index/mod.rs +++ b/src/index/mod.rs @@ -52,7 +52,7 @@ impl Index { } pub fn run_git_gc(&self) { - let cmd = format!("cd {} && gc --auto", self.path.to_str().unwrap()); + let cmd = format!("cd {} && git gc --auto", self.path.to_str().unwrap()); let gc = Command::new("sh").args(&["-c", cmd.as_str()]).output(); if let Err(err) = gc { log::error!("Failed to run `{}`: {:?}", cmd, err); From c7fcff65df09fa3bcb43515655b9f195887713c4 Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Thu, 13 Aug 2020 21:42:57 +0300 Subject: [PATCH 128/139] Rebased --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index d7823cbaf..5b7d9caab 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ target .vagrant .rustwide .rustwide-docker +.idea/ \ No newline at end of file From d8b8a75d661050d469d4f27c21a80f40f4fc4ddd Mon Sep 17 00:00:00 2001 From: ohaddahan Date: Tue, 11 Aug 2020 23:22:43 +0300 Subject: [PATCH 129/139] Update .gitignore Co-authored-by: Chase Wilson --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index 5b7d9caab..d7823cbaf 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,3 @@ target .vagrant .rustwide .rustwide-docker -.idea/ \ No newline at end of file From 60c58fbe3c17c90392a1ef3efec996d5d6e9f9f6 Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Thu, 13 Aug 2020 21:48:01 +0300 Subject: [PATCH 130/139] Rebased --- git_hooks/pre-commit | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100755 git_hooks/pre-commit diff --git a/git_hooks/pre-commit b/git_hooks/pre-commit new file mode 100755 index 000000000..8ae45b3f3 --- /dev/null +++ b/git_hooks/pre-commit @@ -0,0 +1,19 @@ +#!/usr/bin/env sh +cargo fmt -- --check +if [ "$?" != "0" ] ; then + echo "\n" + echo "\033[0;31mpre-commit hook failed during:\033[0m" + echo "\033[0;31m\tcargo fmt -- --check\033[0m" + exit 1 +fi +cargo clippy --locked -- -D warnings +if [ "$?" != "0" ] ; then + echo "\n" + echo "\033[0;31mpre-commit hook failed during:\033[0m" + echo "\033[0;31m\tclippy --locked -- -D warning\033[0m" + exit 1 +fi + +echo "\n" +echo "\033[0;32mpre-commit hook succeeded\033[0m" +exit 0 \ No newline at end of file From 3ec3b839917c77a5af845b3976c977cfa3882920 Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Wed, 12 Aug 2020 19:59:52 +0300 Subject: [PATCH 131/139] Changed envar prefix and replace echo with printf for pre-commit hook --- git_hooks/pre-commit | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/git_hooks/pre-commit b/git_hooks/pre-commit index 8ae45b3f3..f6855e88e 100755 --- a/git_hooks/pre-commit +++ b/git_hooks/pre-commit @@ -1,19 +1,19 @@ #!/usr/bin/env sh cargo fmt -- --check if [ "$?" != "0" ] ; then - echo "\n" - echo "\033[0;31mpre-commit hook failed during:\033[0m" - echo "\033[0;31m\tcargo fmt -- --check\033[0m" + printf "\n" + printf "\033[0;31mpre-commit hook failed during:\033[0m\n" + printf "\033[0;31m\tcargo fmt -- --check\033[0m\n" exit 1 fi cargo clippy --locked -- -D warnings if [ "$?" != "0" ] ; then - echo "\n" - echo "\033[0;31mpre-commit hook failed during:\033[0m" - echo "\033[0;31m\tclippy --locked -- -D warning\033[0m" + printf "\n" + printf "\033[0;31mpre-commit hook failed during:\033[0m\n" + printf "\033[0;31m\tclippy --locked -- -D warning\033[0m\n" exit 1 fi -echo "\n" -echo "\033[0;32mpre-commit hook succeeded\033[0m" +printf "\n" +printf "\033[0;32mpre-commit hook succeeded\033[0m\n" exit 0 \ No newline at end of file From 79a295d3506c46f5279769d9343b2eafa65e0390 Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Thu, 13 Aug 2020 21:48:48 +0300 Subject: [PATCH 132/139] Rebased --- .git_hooks/pre-commit | 1 - git_hooks/pre-commit | 19 ------------------- 2 files changed, 20 deletions(-) delete mode 100755 git_hooks/pre-commit diff --git a/.git_hooks/pre-commit b/.git_hooks/pre-commit index 105684ef7..872c2cdda 100755 --- a/.git_hooks/pre-commit +++ b/.git_hooks/pre-commit @@ -16,4 +16,3 @@ fi printf "\n" printf "\033[0;32mpre-commit hook succeeded\033[0m\n" exit 0 - diff --git a/git_hooks/pre-commit b/git_hooks/pre-commit deleted file mode 100755 index f6855e88e..000000000 --- a/git_hooks/pre-commit +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env sh -cargo fmt -- --check -if [ "$?" != "0" ] ; then - printf "\n" - printf "\033[0;31mpre-commit hook failed during:\033[0m\n" - printf "\033[0;31m\tcargo fmt -- --check\033[0m\n" - exit 1 -fi -cargo clippy --locked -- -D warnings -if [ "$?" != "0" ] ; then - printf "\n" - printf "\033[0;31mpre-commit hook failed during:\033[0m\n" - printf "\033[0;31m\tclippy --locked -- -D warning\033[0m\n" - exit 1 -fi - -printf "\n" -printf "\033[0;32mpre-commit hook succeeded\033[0m\n" -exit 0 \ No newline at end of file From 02bc0ffecf7d42d764f5eae2170c563319ca924e Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Thu, 13 Aug 2020 15:58:30 +0300 Subject: [PATCH 133/139] Added newline at end of pre-commit script --- .git_hooks/pre-commit | 1 + 1 file changed, 1 insertion(+) diff --git a/.git_hooks/pre-commit b/.git_hooks/pre-commit index 872c2cdda..105684ef7 100755 --- a/.git_hooks/pre-commit +++ b/.git_hooks/pre-commit @@ -16,3 +16,4 @@ fi printf "\n" printf "\033[0;32mpre-commit hook succeeded\033[0m\n" exit 0 + From ff02d23f1f134dbb20de4043325b2ad207e6ae73 Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Thu, 13 Aug 2020 22:16:15 +0300 Subject: [PATCH 134/139] Rebased --- src/utils/daemon.rs | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs index 7b886eab8..d142fa544 100644 --- a/src/utils/daemon.rs +++ b/src/utils/daemon.rs @@ -9,16 +9,13 @@ use crate::{ use chrono::{Timelike, Utc}; use failure::Error; use log::{debug, error, info}; -use std::sync::Arc; use std::thread; use std::time::{Duration, Instant}; -fn start_registry_watcher( - opts: DocBuilderOptions, - pool: Pool, - build_queue: Arc, - config: Arc, -) -> Result<(), Error> { +fn start_registry_watcher(opts: DocBuilderOptions, context: &dyn Context) -> Result<(), Error> { + let pool = context.pool()?; + let build_queue = context.build_queue()?; + let config = context.config()?; thread::Builder::new() .name("registry index reader".to_string()) .spawn(move || { @@ -60,24 +57,18 @@ pub fn start_daemon(context: &dyn Context, enable_registry_watcher: bool) -> Res if enable_registry_watcher { // check new crates every minute - start_registry_watcher( - dbopts.clone(), - db.clone(), - build_queue.clone(), - config.clone(), - )?; + start_registry_watcher(dbopts.clone(), context)?; } // build new crates every minute let pool = context.pool()?; let build_queue = context.build_queue()?; let storage = context.storage()?; - let metrics = context.metrics()?; thread::Builder::new() .name("build queue reader".to_string()) .spawn(move || { let doc_builder = DocBuilder::new(dbopts.clone(), pool.clone(), build_queue.clone()); - queue_builder(doc_builder, pool, build_queue, metrics, storage).unwrap(); + queue_builder(doc_builder, pool, build_queue, storage).unwrap(); }) .unwrap(); From 5722ae1b6730f62ff911c26769ef154a60bc3745 Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Thu, 13 Aug 2020 22:23:40 +0300 Subject: [PATCH 135/139] Rebased --- src/metrics/macros.rs | 23 ++----- src/metrics/mod.rs | 2 - src/utils/daemon.rs | 3 +- src/web/metrics.rs | 144 ------------------------------------------ 4 files changed, 6 insertions(+), 166 deletions(-) diff --git a/src/metrics/macros.rs b/src/metrics/macros.rs index 2571f5b1e..c79bd60df 100644 --- a/src/metrics/macros.rs +++ b/src/metrics/macros.rs @@ -6,42 +6,27 @@ pub(super) trait MetricFromOpts: Sized { macro_rules! metrics { ( $vis:vis struct $name:ident { - $( - #[doc = $help:expr] - $(#[$meta:meta])* - $metric_vis:vis $metric:ident: $ty:ty $([$($label:expr),* $(,)?])? - ),* $(,)? + $(#[doc = $help:expr] $metric:ident: $ty:ty $([$($label:expr),*])?,)* } metrics visibility: $metric_vis:vis, namespace: $namespace:expr, ) => { $vis struct $name { - registry: prometheus::Registry, - $( - $(#[$meta])* - $metric_vis $metric: $ty, - )* + registry: Registry, + $($metric_vis $metric: $ty,)* } impl $name { $vis fn new() -> Result { let registry = Registry::new(); $( - $(#[$meta])* let $metric = <$ty>::from_opts( Opts::new(stringify!($metric), $help) .namespace($namespace) $(.variable_labels(vec![$($label.into()),*]))? )?; - $(#[$meta])* registry.register(Box::new($metric.clone()))?; )* - Ok(Self { - registry, - $( - $(#[$meta])* - $metric, - )* - }) + Ok(Self { registry, $($metric,)* }) } } impl std::fmt::Debug for $name { diff --git a/src/metrics/mod.rs b/src/metrics/mod.rs index 3616ef657..4fc568957 100644 --- a/src/metrics/mod.rs +++ b/src/metrics/mod.rs @@ -31,10 +31,8 @@ metrics! { failed_db_connections: IntCounter, /// The number of currently opened file descriptors - #[cfg(linux)] open_file_descriptors: IntGauge, /// The number of threads being used by docs.rs - #[cfg(linux)] running_threads: IntGauge, /// The traffic of various docs.rs routes diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs index d142fa544..673f05204 100644 --- a/src/utils/daemon.rs +++ b/src/utils/daemon.rs @@ -64,11 +64,12 @@ pub fn start_daemon(context: &dyn Context, enable_registry_watcher: bool) -> Res let pool = context.pool()?; let build_queue = context.build_queue()?; let storage = context.storage()?; + let metrics = context.metrics()?; thread::Builder::new() .name("build queue reader".to_string()) .spawn(move || { let doc_builder = DocBuilder::new(dbopts.clone(), pool.clone(), build_queue.clone()); - queue_builder(doc_builder, pool, build_queue, storage).unwrap(); + queue_builder(doc_builder, pool, build_queue, metrics, storage).unwrap(); }) .unwrap(); diff --git a/src/web/metrics.rs b/src/web/metrics.rs index 2f6dccf9d..dbb910499 100644 --- a/src/web/metrics.rs +++ b/src/web/metrics.rs @@ -7,150 +7,6 @@ use iron::status::Status; use prometheus::{Encoder, HistogramVec, TextEncoder}; use std::time::{Duration, Instant}; -static QUEUED_CRATES_COUNT: Lazy = Lazy::new(|| { - register_int_gauge!( - "docsrs_queued_crates_count", - "Number of crates in the build queue" - ) - .unwrap() -}); - -pub static PRIORITIZED_CRATES_COUNT: Lazy = Lazy::new(|| { - register_int_gauge!( - "docsrs_prioritized_crates_count", - "Number of crates in the build queue that have a positive priority" - ) - .unwrap() -}); - -static FAILED_CRATES_COUNT: Lazy = Lazy::new(|| { - register_int_gauge!( - "docsrs_failed_crates_count", - "Number of crates that failed to build" - ) - .unwrap() -}); - -pub static TOTAL_BUILDS: Lazy = - Lazy::new(|| register_int_counter!("docsrs_total_builds", "Number of crates built").unwrap()); - -pub static SUCCESSFUL_BUILDS: Lazy = Lazy::new(|| { - register_int_counter!( - "docsrs_successful_builds", - "Number of builds that successfully generated docs" - ) - .unwrap() -}); - -pub static FAILED_BUILDS: Lazy = Lazy::new(|| { - register_int_counter!( - "docsrs_failed_builds", - "Number of builds that generated a compile error" - ) - .unwrap() -}); - -pub static NON_LIBRARY_BUILDS: Lazy = Lazy::new(|| { - register_int_counter!( - "docsrs_non_library_builds", - "Number of builds that did not complete due to not being a library" - ) - .unwrap() -}); - -pub static UPLOADED_FILES_TOTAL: Lazy = Lazy::new(|| { - register_int_counter!( - "docsrs_uploaded_files_total", - "Number of files uploaded to S3 or stored in the database" - ) - .unwrap() -}); - -pub static ROUTES_VISITED: Lazy = Lazy::new(|| { - register_int_counter_vec!( - "docsrs_routes_visited", - "The traffic of various docs.rs routes", - &["route"] - ) - .unwrap() -}); - -pub static RESPONSE_TIMES: Lazy = Lazy::new(|| { - register_histogram_vec!( - "docsrs_response_time", - "The response times of various docs.rs routes", - &["route"] - ) - .unwrap() -}); - -pub static RUSTDOC_RENDERING_TIMES: Lazy = Lazy::new(|| { - register_histogram_vec!( - "docsrs_rustdoc_rendering_time", - "The time it takes to render a rustdoc page", - &["step"] - ) - .unwrap() -}); - -pub static FAILED_DB_CONNECTIONS: Lazy = Lazy::new(|| { - register_int_counter!( - "docsrs_failed_db_connections", - "Number of attempted and failed connections to the database" - ) - .unwrap() -}); - -pub static USED_DB_CONNECTIONS: Lazy = Lazy::new(|| { - register_int_gauge!( - "docsrs_used_db_connections", - "The number of used database connections" - ) - .unwrap() -}); - -pub static IDLE_DB_CONNECTIONS: Lazy = Lazy::new(|| { - register_int_gauge!( - "docsrs_idle_db_connections", - "The number of idle database connections" - ) - .unwrap() -}); - -pub static MAX_DB_CONNECTIONS: Lazy = Lazy::new(|| { - register_int_gauge!( - "docsrs_max_db_connections", - "The maximum database connections" - ) - .unwrap() -}); - -#[cfg(target_os = "linux")] -pub static OPEN_FILE_DESCRIPTORS: Lazy = Lazy::new(|| { - register_int_gauge!( - "docsrs_open_file_descriptors", - "The number of currently opened file descriptors" - ) - .unwrap() -}); - -#[cfg(target_os = "linux")] -pub static CURRENTLY_RUNNING_THREADS: Lazy = Lazy::new(|| { - register_int_gauge!( - "docsrs_running_threads", - "The number of threads being used by docs.rs" - ) - .unwrap() -}); - -pub static HTML_REWRITE_OOMS: Lazy = Lazy::new(|| { - register_int_gauge!( - "docsrs_html_rewrite_ooms", - "The number of attempted files that failed due to a memory limit" - ) - .unwrap() -}); - pub fn metrics_handler(req: &mut Request) -> IronResult { let metrics = extension!(req, Metrics); let pool = extension!(req, Pool); From 6c7193b618ca8c5aa650c34da830e0d799a1e751 Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Thu, 13 Aug 2020 22:24:39 +0300 Subject: [PATCH 136/139] Rebased --- src/metrics/macros.rs | 23 +++++++++++++++++++---- src/metrics/mod.rs | 2 ++ 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/src/metrics/macros.rs b/src/metrics/macros.rs index c79bd60df..2571f5b1e 100644 --- a/src/metrics/macros.rs +++ b/src/metrics/macros.rs @@ -6,27 +6,42 @@ pub(super) trait MetricFromOpts: Sized { macro_rules! metrics { ( $vis:vis struct $name:ident { - $(#[doc = $help:expr] $metric:ident: $ty:ty $([$($label:expr),*])?,)* + $( + #[doc = $help:expr] + $(#[$meta:meta])* + $metric_vis:vis $metric:ident: $ty:ty $([$($label:expr),* $(,)?])? + ),* $(,)? } metrics visibility: $metric_vis:vis, namespace: $namespace:expr, ) => { $vis struct $name { - registry: Registry, - $($metric_vis $metric: $ty,)* + registry: prometheus::Registry, + $( + $(#[$meta])* + $metric_vis $metric: $ty, + )* } impl $name { $vis fn new() -> Result { let registry = Registry::new(); $( + $(#[$meta])* let $metric = <$ty>::from_opts( Opts::new(stringify!($metric), $help) .namespace($namespace) $(.variable_labels(vec![$($label.into()),*]))? )?; + $(#[$meta])* registry.register(Box::new($metric.clone()))?; )* - Ok(Self { registry, $($metric,)* }) + Ok(Self { + registry, + $( + $(#[$meta])* + $metric, + )* + }) } } impl std::fmt::Debug for $name { diff --git a/src/metrics/mod.rs b/src/metrics/mod.rs index 4fc568957..3616ef657 100644 --- a/src/metrics/mod.rs +++ b/src/metrics/mod.rs @@ -31,8 +31,10 @@ metrics! { failed_db_connections: IntCounter, /// The number of currently opened file descriptors + #[cfg(linux)] open_file_descriptors: IntGauge, /// The number of threads being used by docs.rs + #[cfg(linux)] running_threads: IntGauge, /// The traffic of various docs.rs routes From ef9cd2282cf7a665610cb2dec9794f3a9d418256 Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Thu, 13 Aug 2020 22:28:00 +0300 Subject: [PATCH 137/139] Rebased --- src/config.rs | 2 +- src/utils/daemon.rs | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/config.rs b/src/config.rs index a3ae7dc9f..d84f12872 100644 --- a/src/config.rs +++ b/src/config.rs @@ -30,7 +30,7 @@ pub struct Config { pub(crate) max_file_size_html: usize, // The most memory that can be used to parse an HTML file pub(crate) max_parse_memory: usize, - // Time between 'git gc' calls in seconds + // Time between 'git gc --auto' calls in seconds pub(crate) registry_gc_interval: u64, } diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs index 673f05204..af8436c94 100644 --- a/src/utils/daemon.rs +++ b/src/utils/daemon.rs @@ -22,7 +22,6 @@ fn start_registry_watcher(opts: DocBuilderOptions, context: &dyn Context) -> Res // space this out to prevent it from clashing against the queue-builder thread on launch thread::sleep(Duration::from_secs(30)); let mut last_gc = Instant::now(); - loop { let mut doc_builder = DocBuilder::new(opts.clone(), pool.clone(), build_queue.clone()); From a593212f266d7c75f8caa39d7f030199d10ca5b9 Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Thu, 13 Aug 2020 22:28:39 +0300 Subject: [PATCH 138/139] Rebased --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index d7823cbaf..5b7d9caab 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ target .vagrant .rustwide .rustwide-docker +.idea/ \ No newline at end of file From 5b0cc9f4322c4a110b4af39eed7c4d97016451f3 Mon Sep 17 00:00:00 2001 From: Ohad Dahan Date: Thu, 13 Aug 2020 22:30:47 +0300 Subject: [PATCH 139/139] Rebased --- git_hooks/pre-commit | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100755 git_hooks/pre-commit diff --git a/git_hooks/pre-commit b/git_hooks/pre-commit new file mode 100755 index 000000000..8ae45b3f3 --- /dev/null +++ b/git_hooks/pre-commit @@ -0,0 +1,19 @@ +#!/usr/bin/env sh +cargo fmt -- --check +if [ "$?" != "0" ] ; then + echo "\n" + echo "\033[0;31mpre-commit hook failed during:\033[0m" + echo "\033[0;31m\tcargo fmt -- --check\033[0m" + exit 1 +fi +cargo clippy --locked -- -D warnings +if [ "$?" != "0" ] ; then + echo "\n" + echo "\033[0;31mpre-commit hook failed during:\033[0m" + echo "\033[0;31m\tclippy --locked -- -D warning\033[0m" + exit 1 +fi + +echo "\n" +echo "\033[0;32mpre-commit hook succeeded\033[0m" +exit 0 \ No newline at end of file