From f59bc5f030ef9d5e220bd6503c9e5fa5efbb6264 Mon Sep 17 00:00:00 2001 From: Onur Aslan Date: Fri, 3 Jun 2016 15:53:59 +0300 Subject: [PATCH 01/79] Initial commit of user interface of cratesfyi --- .gitignore | 3 + Cargo.lock | 438 ++++++++++++++++++++++++++++++++++++ Cargo.toml | 11 +- rustfmt.toml | 2 + src/bin/cratesfyi.rs | 5 + src/db/mod.rs | 52 ++++- src/lib.rs | 11 + src/web/builds.rs | 124 ++++++++++ src/web/crate_details.rs | 184 +++++++++++++++ src/web/file.rs | 87 +++++++ src/web/mod.rs | 312 +++++++++++++++++++++++++ src/web/page.rs | 89 ++++++++ src/web/pool.rs | 29 +++ src/web/releases.rs | 201 +++++++++++++++++ src/web/rustdoc.rs | 176 +++++++++++++++ src/web/source.rs | 245 ++++++++++++++++++++ templates/builds.hbs | 44 ++++ templates/crate_details.hbs | 57 +++++ templates/footer.hbs | 2 + templates/header.hbs | 17 ++ templates/navigation.hbs | 40 ++++ templates/releases.hbs | 50 ++++ templates/rustdoc.hbs | 17 ++ templates/source.hbs | 38 ++++ templates/style.scss | 390 ++++++++++++++++++++++++++++++++ 25 files changed, 2622 insertions(+), 2 deletions(-) create mode 100644 rustfmt.toml create mode 100644 src/web/builds.rs create mode 100644 src/web/crate_details.rs create mode 100644 src/web/file.rs create mode 100644 src/web/mod.rs create mode 100644 src/web/page.rs create mode 100644 src/web/pool.rs create mode 100644 src/web/releases.rs create mode 100644 src/web/rustdoc.rs create mode 100644 src/web/source.rs create mode 100644 templates/builds.hbs create mode 100644 templates/crate_details.hbs create mode 100644 templates/footer.hbs create mode 100644 templates/header.hbs create mode 100644 templates/navigation.hbs create mode 100644 templates/releases.hbs create mode 100644 templates/rustdoc.hbs create mode 100644 templates/source.hbs create mode 100644 templates/style.scss diff --git a/.gitignore b/.gitignore index eb5a316cb..220040f55 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,4 @@ target +*.css +*.css.map +.sass-cache diff --git a/Cargo.lock b/Cargo.lock index fbee77a8d..e6e56b072 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6,16 +6,25 @@ dependencies = [ "clap 2.5.2 (registry+https://github.com/rust-lang/crates.io-index)", "env_logger 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", "git2 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", + "handlebars-iron 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", "hyper 0.9.7 (registry+https://github.com/rust-lang/crates.io-index)", + "iron 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", "magic 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", + "params 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "postgres 0.11.8 (registry+https://github.com/rust-lang/crates.io-index)", + "pulldown-cmark 0.0.8 (registry+https://github.com/rust-lang/crates.io-index)", + "r2d2 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "r2d2_postgres 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)", "regex 0.1.71 (registry+https://github.com/rust-lang/crates.io-index)", + "router 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)", "semver 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", "slug 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "staticfile 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "tempdir 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)", + "url 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -60,6 +69,23 @@ name = "bitflags" version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "bodyparser" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "iron 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "persistent 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "plugin 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 0.7.10 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "buf_redux" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "bufstream" version = "0.1.2" @@ -129,6 +155,14 @@ dependencies = [ "gcc 0.3.28 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "conduit-mime-types" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "cookie" version = "0.2.5" @@ -196,6 +230,15 @@ dependencies = [ "regex 0.1.71 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "error" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "traitobject 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "typeable 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "filetime" version = "0.1.10" @@ -237,6 +280,11 @@ dependencies = [ "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "getopts" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "git2" version = "0.4.3" @@ -264,6 +312,32 @@ name = "glob" version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "handlebars" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "itertools 0.4.15 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", + "quick-error 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 0.1.71 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "handlebars-iron" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "handlebars 0.16.1 (registry+https://github.com/rust-lang/crates.io-index)", + "iron 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", + "plugin 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)", + "walkdir 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "hex" version = "0.2.0" @@ -282,6 +356,27 @@ name = "httparse" version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "hyper" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cookie 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", + "httparse 1.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "language-tags 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", + "mime 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "num_cpus 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)", + "openssl 0.7.13 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)", + "solicit 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)", + "time 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)", + "traitobject 0.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "typeable 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "unicase 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "url 0.5.9 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "hyper" version = "0.9.7" @@ -314,6 +409,28 @@ dependencies = [ "unicode-normalization 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "iron" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "conduit-mime-types 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", + "error 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", + "hyper 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", + "modifier 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "num_cpus 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)", + "plugin 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", + "typemap 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "url 0.5.9 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "itertools" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "kernel32-sys" version = "0.2.2" @@ -328,6 +445,11 @@ name = "language-tags" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "lazy_static" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "lazy_static" version = "0.2.1" @@ -431,6 +553,17 @@ dependencies = [ "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "mime_guess" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "mime 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "phf 0.7.14 (registry+https://github.com/rust-lang/crates.io-index)", + "phf_codegen 0.7.14 (registry+https://github.com/rust-lang/crates.io-index)", + "unicase 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "miniz-sys" version = "0.1.7" @@ -440,6 +573,36 @@ dependencies = [ "libc 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "modifier" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "mount" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "iron 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "sequence_trie 0.0.13 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "multipart" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "buf_redux 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "env_logger 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "hyper 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)", + "mime 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "mime_guess 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", + "tempdir 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "net2" version = "0.2.23" @@ -457,6 +620,72 @@ name = "nom" version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "num" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "num-bigint 0.1.32 (registry+https://github.com/rust-lang/crates.io-index)", + "num-complex 0.1.32 (registry+https://github.com/rust-lang/crates.io-index)", + "num-integer 0.1.32 (registry+https://github.com/rust-lang/crates.io-index)", + "num-iter 0.1.32 (registry+https://github.com/rust-lang/crates.io-index)", + "num-rational 0.1.32 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.1.32 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "num-bigint" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "num-integer 0.1.32 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.1.32 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "num-complex" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "num-traits 0.1.32 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "num-integer" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "num-traits 0.1.32 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "num-iter" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "num-integer 0.1.32 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.1.32 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "num-rational" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "num-bigint 0.1.32 (registry+https://github.com/rust-lang/crates.io-index)", + "num-integer 0.1.32 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.1.32 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "num-traits" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "num_cpus" version = "0.2.12" @@ -508,6 +737,29 @@ dependencies = [ "openssl 0.7.13 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "params" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bodyparser 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "iron 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "multipart 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "num 0.1.32 (registry+https://github.com/rust-lang/crates.io-index)", + "plugin 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", + "urlencoded 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "persistent" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "iron 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "plugin 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "phf" version = "0.7.14" @@ -516,16 +768,45 @@ dependencies = [ "phf_shared 0.7.14 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "phf_codegen" +version = "0.7.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "phf_generator 0.7.14 (registry+https://github.com/rust-lang/crates.io-index)", + "phf_shared 0.7.14 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "phf_generator" +version = "0.7.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "phf_shared 0.7.14 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "phf_shared" version = "0.7.14" source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unicase 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", +] [[package]] name = "pkg-config" version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "plugin" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "typemap 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "pnacl-build-helper" version = "1.4.10" @@ -549,6 +830,37 @@ dependencies = [ "time 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "pulldown-cmark" +version = "0.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bitflags 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", + "getopts 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "quick-error" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "r2d2" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "r2d2_postgres" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "postgres 0.11.8 (registry+https://github.com/rust-lang/crates.io-index)", + "r2d2 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "rand" version = "0.3.14" @@ -574,6 +886,20 @@ name = "regex-syntax" version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "route-recognizer" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "router" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "iron 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "route-recognizer 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "rustc-serialize" version = "0.3.19" @@ -600,6 +926,25 @@ dependencies = [ "nom 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "sequence_trie" +version = "0.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "serde" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "serde_json" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "num-traits 0.1.32 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 0.7.10 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "slug" version = "0.1.1" @@ -617,6 +962,19 @@ dependencies = [ "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "staticfile" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "filetime 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "iron 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", + "mount 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "time 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)", + "url 0.5.9 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "strsim" version = "0.3.0" @@ -693,11 +1051,24 @@ name = "traitobject" version = "0.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "traitobject" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "typeable" version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "typemap" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unsafe-any 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "unicase" version = "1.4.0" @@ -729,6 +1100,36 @@ name = "unidecode" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "unsafe-any" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "traitobject 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "url" +version = "0.2.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "matches 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)", + "uuid 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "url" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "matches 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-bidi 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-normalization 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "uuid 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "url" version = "1.1.1" @@ -738,6 +1139,17 @@ dependencies = [ "matches 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "urlencoded" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bodyparser 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "iron 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "plugin 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", + "url 0.2.38 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "user32-sys" version = "0.2.0" @@ -752,11 +1164,37 @@ name = "utf8-ranges" version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "uuid" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "uuid" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "vec_map" version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "walkdir" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "winapi" version = "0.2.7" diff --git a/Cargo.toml b/Cargo.toml index 53f6ccec8..e0a406c31 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,7 @@ build = "build.rs" log = "0.3" rustc-serialize = "0.3" regex = "0.1" -postgres = { version = "0.11.7", features = [ "time", "rustc-serialize" ] } +postgres = { version = "^0.11", features = [ "time", "rustc-serialize" ] } clap = "2.5.2" time = "0.1" hyper = "0.9.7" @@ -20,6 +20,15 @@ slug = "^0.1.1" env_logger = "0.3" git2 = "0.4" magic = "^0.10.0" +iron = "0.3.0" +router = "0.1.1" +staticfile = { version = "0.2.0", features = [ "cache" ] } +handlebars-iron = "0.15.2" +pulldown-cmark = "0.0.8" +r2d2 = "0.7.0" +r2d2_postgres = "0.10.1" +url = "1.1.1" +params = "0.2.2" [dependencies.cargo] git = "https://github.com/rust-lang/cargo.git" diff --git a/rustfmt.toml b/rustfmt.toml new file mode 100644 index 000000000..de740541c --- /dev/null +++ b/rustfmt.toml @@ -0,0 +1,2 @@ + +format_strings = false diff --git a/src/bin/cratesfyi.rs b/src/bin/cratesfyi.rs index 04123bf75..5d5edf16b 100644 --- a/src/bin/cratesfyi.rs +++ b/src/bin/cratesfyi.rs @@ -14,6 +14,7 @@ use std::path::PathBuf; use clap::{Arg, App, SubCommand}; use cratesfyi::{DocBuilder, DocBuilderOptions, db}; use cratesfyi::utils::build_doc; +use cratesfyi::start_web_server; pub fn main() { @@ -88,6 +89,8 @@ pub fn main() { .subcommand(SubCommand::with_name("init") .about("Initialize database. Currently \ only creates tables in database."))) + .subcommand(SubCommand::with_name("start-web-server") + .about("Starts web server")) .get_matches(); @@ -158,6 +161,8 @@ pub fn main() { process::exit(1); } } + } else if let Some(_) = matches.subcommand_matches("start-web-server") { + start_web_server(); } else { println!("{}", matches.usage()); } diff --git a/src/db/mod.rs b/src/db/mod.rs index f3479ac94..d9c646d2d 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -7,6 +7,8 @@ pub use self::file::add_path_into_database; use postgres::{Connection, SslMode}; use postgres::error::{Error, ConnectError}; use std::env; +use r2d2; +use r2d2_postgres; mod add_package; mod file; @@ -21,6 +23,52 @@ pub fn connect_db() -> Result { } +pub fn create_pool() -> r2d2::Pool { + let db_url = env::var("CRATESFYI_DATABASE_URL") + .expect("CRATESFYI_DATABASE_URL environment variable is not exists"); + let config = r2d2::Config::default(); + let manager = r2d2_postgres::PostgresConnectionManager::new(&db_url[..], + r2d2_postgres::SslMode::None) + .expect("Failed to create PostgresConnectionManager"); + r2d2::Pool::new(config, manager).expect("Failed to create r2d2 pool") +} + + +/// Updates content column in crates table. +/// +/// This column will be used for searches and always contains `tsvector` of: +/// +/// * crate name (rank A-weight) +/// * latest release description (rank B-weight) +/// * latest release keywords (rank B-weight) +/// * latest release readme (rank C-weight) +/// * latest release root rustdoc (rank C-weight) +pub fn update_search_index(conn: &Connection) -> Result { + conn.execute(" + WITH doc as ( + SELECT DISTINCT ON(releases.crate_id) + releases.id, + releases.crate_id, + setweight(to_tsvector(crates.name), 'A') || + setweight(to_tsvector(coalesce(releases.description, '')), 'B') || + setweight(to_tsvector(coalesce(( + SELECT string_agg(value, ' ') + FROM json_array_elements_text(releases.keywords)), '')), 'B') || + setweight(to_tsvector(coalesce(releases.readme, '')), 'C') || + setweight(to_tsvector(coalesce(releases.description_long, '')), 'C') as content + FROM releases + INNER JOIN crates ON crates.id = releases.crate_id + ORDER BY releases.crate_id, releases.release_time DESC + ) + UPDATE crates + SET latest_version_id = doc.id, + content = doc.content + FROM doc + WHERE crates.id = doc.crate_id AND + (crates.latest_version_id = 0 OR crates.latest_version_id != doc.id);", &[]) +} + + /// Creates database tables pub fn create_tables(conn: &Connection) -> Result<(), Error> { let queries = [ @@ -35,7 +83,8 @@ pub fn create_tables(conn: &Connection) -> Result<(), Error> { github_forks INT DEFAULT 0, \ github_issues INT DEFAULT 0, \ github_last_commit TIMESTAMP, \ - github_last_update TIMESTAMP \ + github_last_update TIMESTAMP, \ + content tsvector, \ )", "CREATE TABLE releases ( \ id SERIAL PRIMARY KEY, \ @@ -116,6 +165,7 @@ pub fn create_tables(conn: &Connection) -> Result<(), Error> { date_updated TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, \ content BYTEA \ )", + "CREATE INDEX content_idx ON crates USING gin(content)", ]; for query in queries.into_iter() { diff --git a/src/lib.rs b/src/lib.rs index 0d972fbe6..6cf89a7a3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -11,15 +11,26 @@ extern crate semver; extern crate slug; extern crate git2; extern crate magic; +extern crate iron; +extern crate router; +extern crate staticfile; +extern crate handlebars_iron; +extern crate pulldown_cmark; +extern crate r2d2; +extern crate r2d2_postgres; +extern crate url; +extern crate params; pub use self::docbuilder::DocBuilder; pub use self::docbuilder::ChrootBuilderResult; pub use self::docbuilder::error::DocBuilderError; pub use self::docbuilder::options::DocBuilderOptions; +pub use self::web::start_web_server; pub mod db; pub mod utils; mod docbuilder; +mod web; /// Version string generated at build time contains last git diff --git a/src/web/builds.rs b/src/web/builds.rs new file mode 100644 index 000000000..1c017f91c --- /dev/null +++ b/src/web/builds.rs @@ -0,0 +1,124 @@ + + +use std::collections::BTreeMap; +use super::MetaData; +use super::pool::Pool; +use super::duration_to_str; +use super::page::Page; +use iron::prelude::*; +use time; +use router::Router; +use rustc_serialize::json::{Json, ToJson}; + + + +#[derive(Clone)] +struct Build { + id: i32, + rustc_version: String, + cratesfyi_version: String, + build_status: bool, + build_time: time::Timespec, + output: Option, +} + + +struct BuildsPage { + metadata: Option, + builds: Vec, + build_details: Option, +} + + +impl ToJson for Build { + fn to_json(&self) -> Json { + let mut m: BTreeMap = BTreeMap::new(); + m.insert("id".to_owned(), self.id.to_json()); + m.insert("rustc_version".to_owned(), self.rustc_version.to_json()); + m.insert("cratesfyi_version".to_owned(), + self.cratesfyi_version.to_json()); + m.insert("build_status".to_owned(), self.build_status.to_json()); + m.insert("build_time".to_owned(), + duration_to_str(self.build_time).to_json()); + m.insert("output".to_owned(), self.output.to_json()); + m.to_json() + } +} + + +impl ToJson for BuildsPage { + fn to_json(&self) -> Json { + let mut m: BTreeMap = BTreeMap::new(); + m.insert("metadata".to_owned(), self.metadata.to_json()); + m.insert("builds".to_owned(), self.builds.to_json()); + m.insert("build_details".to_owned(), self.build_details.to_json()); + m.to_json() + } +} + + +pub fn build_list_handler(req: &mut Request) -> IronResult { + + let name = req.extensions.get::().unwrap().find("name").unwrap(); + let version = req.extensions.get::().unwrap().find("version").unwrap(); + let req_build_id: i32 = req.extensions + .get::() + .unwrap() + .find("id") + .unwrap_or("0") + .parse() + .unwrap_or(0); + + let conn = req.extensions.get::().unwrap(); + + let mut build_list: Vec = Vec::new(); + let mut build_details = None; + + // FIXME: getting builds.output may cause performance issues when release have tons of builds + for row in &conn.query("SELECT crates.name, \ + releases.version, \ + releases.description, \ + releases.rustdoc_status, \ + releases.target_name, \ + builds.id, \ + builds.rustc_version, \ + builds.cratesfyi_version, \ + builds.build_status, \ + builds.build_time, \ + builds.output \ + FROM builds \ + INNER JOIN releases ON releases.id = builds.rid \ + INNER JOIN crates ON releases.crate_id = crates.id \ + WHERE crates.name = $1 AND releases.version = $2", + &[&name, &version]) + .unwrap() { + + let id: i32 = row.get(5); + + let build = Build { + id: id, + rustc_version: row.get(6), + cratesfyi_version: row.get(7), + build_status: row.get(8), + build_time: row.get(9), + output: row.get(10), + }; + + if id == req_build_id { + build_details = Some(build.clone()); + } + + build_list.push(build); + } + + let builds_page = BuildsPage { + metadata: MetaData::from_crate(&conn, &name, &version), + builds: build_list, + build_details: build_details, + }; + + Page::new(builds_page) + .set_true("show_package_navigation") + .set_true("package_navigation_builds_tab") + .to_resp("builds") +} diff --git a/src/web/crate_details.rs b/src/web/crate_details.rs new file mode 100644 index 000000000..3f2c9551f --- /dev/null +++ b/src/web/crate_details.rs @@ -0,0 +1,184 @@ + + + +use super::{NoCrate, MetaData, duration_to_str, match_version, render_markdown}; +use super::page::Page; +use db::connect_db; +use iron::prelude::*; +use iron::status; +use std::collections::BTreeMap; +use time; +use rustc_serialize::json::{Json, ToJson}; +use router::Router; +use postgres::Connection; +use semver; + + +// TODO: Add target name and versions + + +#[derive(Debug)] +struct CrateDetails { + name: String, + version: String, + description: Option, + authors: Vec<(String, String)>, + authors_json: Option, + dependencies: Option, + readme: Option, + rustdoc: Option, // this is description_long in database + release_time: time::Timespec, + build_status: bool, + rustdoc_status: bool, + repository_url: Option, + homepage_url: Option, + keywords: Option, + have_examples: bool, // need to check this manually + target_name: Option, + versions: Vec, + metadata: MetaData, +} + + +impl ToJson for CrateDetails { + fn to_json(&self) -> Json { + let mut m: BTreeMap = BTreeMap::new(); + m.insert("name".to_string(), self.name.to_json()); + m.insert("version".to_string(), self.version.to_json()); + m.insert("description".to_string(), self.description.to_json()); + m.insert("authors".to_string(), self.authors.to_json()); + m.insert("authors_json".to_string(), self.authors_json.to_json()); + m.insert("dependencies".to_string(), self.dependencies.to_json()); + if let Some(ref readme) = self.readme { + m.insert("readme".to_string(), render_markdown(&readme).to_json()); + } + if let Some(ref rustdoc) = self.rustdoc { + m.insert("rustdoc".to_string(), render_markdown(&rustdoc).to_json()); + } + m.insert("release_time".to_string(), + duration_to_str(self.release_time).to_json()); + m.insert("build_status".to_string(), self.build_status.to_json()); + m.insert("rustdoc_status".to_string(), self.rustdoc_status.to_json()); + m.insert("repository_url".to_string(), self.repository_url.to_json()); + m.insert("homepage_url".to_string(), self.homepage_url.to_json()); + m.insert("keywords".to_string(), self.keywords.to_json()); + m.insert("have_examples".to_string(), self.have_examples.to_json()); + m.insert("target_name".to_string(), self.target_name.to_json()); + m.insert("versions".to_string(), self.versions.to_json()); + m.insert("metadata".to_string(), self.metadata.to_json()); + m.to_json() + } +} + + +impl CrateDetails { + fn new(conn: &Connection, name: &str, version: &str) -> Option { + + // get all stuff, I love you rustfmt + let query = "SELECT crates.name, \ + releases.version, \ + releases.description, \ + releases.authors, \ + releases.dependencies, \ + releases.readme, \ + releases.description_long, \ + releases.release_time, \ + releases.build_status, \ + releases.rustdoc_status, \ + releases.repository_url, \ + releases.homepage_url, \ + releases.keywords, \ + releases.have_examples, \ + releases.target_name, \ + crates.versions, \ + authors.name, \ + authors.slug \ + FROM author_rels \ + LEFT OUTER JOIN authors ON authors.id = author_rels.aid \ + LEFT OUTER JOIN releases ON releases.id = author_rels.rid \ + LEFT OUTER JOIN crates ON crates.id = releases.crate_id \ + WHERE crates.name = $1 AND releases.version = $2;"; + + let rows = conn.query(query, &[&name, &version]).unwrap(); + + if rows.len() == 0 { + return None; + } + + // sort versions with semver + let versions = { + let mut versions: Vec = Vec::new(); + let versions_from_db: Json = rows.get(0).get(15); + + versions_from_db.as_array().map(|vers| { + for version in vers { + version.as_string().map(|version| { + if let Ok(sem_ver) = semver::Version::parse(&version) { + versions.push(sem_ver); + }; + }); + } + }); + + versions.sort(); + versions.reverse(); + versions.iter().map(|semver| format!("{}", semver)).collect() + }; + + let metadata = MetaData { + name: rows.get(0).get(0), + version: rows.get(0).get(1), + description: rows.get(0).get(2), + rustdoc_status: rows.get(0).get(9), + target_name: rows.get(0).get(14), + }; + + let mut crate_details = CrateDetails { + name: rows.get(0).get(0), + version: rows.get(0).get(1), + description: rows.get(0).get(2), + authors: Vec::new(), + authors_json: rows.get(0).get(3), + dependencies: rows.get(0).get(4), + readme: rows.get(0).get(5), + rustdoc: rows.get(0).get(6), + release_time: rows.get(0).get(7), + build_status: rows.get(0).get(8), + rustdoc_status: rows.get(0).get(9), + repository_url: rows.get(0).get(10), + homepage_url: rows.get(0).get(11), + keywords: rows.get(0).get(12), + have_examples: rows.get(0).get(13), + target_name: rows.get(0).get(14), + versions: versions, + metadata: metadata, + }; + + // Insert authors with name and slug + for row in &rows { + crate_details.authors.push((row.get(16), row.get(17))); + } + + Some(crate_details) + } +} + + + +pub fn crate_details_handler(req: &mut Request) -> IronResult { + // this handler must always called with a crate name + let name = req.extensions.get::().unwrap().find("name").unwrap(); + let req_version = req.extensions.get::().unwrap().find("version"); + + let conn = connect_db().unwrap(); + + match_version(&conn, &name, req_version) + .and_then(|version| CrateDetails::new(&conn, &name, &version)) + .ok_or(IronError::new(NoCrate, status::NotFound)) + .and_then(|details| { + Page::new(details) + .set_true("show_package_navigation") + .set_true("package_navigation_crate_tab") + .to_resp("crate_details") + }) +} diff --git a/src/web/file.rs b/src/web/file.rs new file mode 100644 index 000000000..2a47dbff0 --- /dev/null +++ b/src/web/file.rs @@ -0,0 +1,87 @@ +//! Database based file handler + +use super::pool::Pool; +use time; +use postgres::Connection; +use iron::{Handler, Request, IronResult, Response, IronError}; +use iron::status; + + +pub struct File { + pub path: String, + pub mime: String, + pub date_added: time::Timespec, + pub date_updated: time::Timespec, + pub content: Vec, +} + + +impl File { + /// Gets file from database + pub fn from_path(conn: &Connection, path: &str) -> Option { + + let rows = conn.query("SELECT path, mime, date_added, date_updated, content FROM files \ + WHERE path = $1", + &[&path]) + .unwrap(); + + if rows.len() == 0 { + None + } else { + let row = rows.get(0); + Some(File { + path: row.get(0), + mime: row.get(1), + date_added: row.get(2), + date_updated: row.get(3), + content: row.get(4), + }) + } + } + + + /// Consumes File and creates a iron response + pub fn serve(self) -> Response { + use iron::headers::{CacheControl, LastModified, CacheDirective, HttpDate, ContentType}; + + let mut response = Response::with((status::Ok, self.content)); + let cache = vec![CacheDirective::Public, + CacheDirective::MaxAge(super::STATIC_FILE_CACHE_DURATION as u32)]; + response.headers.set(ContentType(self.mime.parse().unwrap())); + response.headers.set(CacheControl(cache)); + response.headers.set(LastModified(HttpDate(time::at(self.date_updated)))); + response + } + + + /// Checks if mime type of file is "application/x-empty" + pub fn is_empty(&self) -> bool { + self.mime == "application/x-empty" + } +} + + +/// Database based file handler for iron +/// +/// This is similar to staticfile crate, but its using getting files from database. +pub struct DatabaseFileHandler; + +impl Handler for DatabaseFileHandler { + fn handle(&self, req: &mut Request) -> IronResult { + + let path = { + let mut path = req.url.path.clone().join("/"); + if path.ends_with("/") { + path.push_str("index.html"); + } + path + }; + + let conn = req.extensions.get::().unwrap(); + if let Some(file) = File::from_path(&conn, &path) { + Ok(file.serve()) + } else { + Err(IronError::new(super::NoCrate, status::NotFound)) + } + } +} diff --git a/src/web/mod.rs b/src/web/mod.rs new file mode 100644 index 000000000..bebe2dedb --- /dev/null +++ b/src/web/mod.rs @@ -0,0 +1,312 @@ +//! Web interface of cratesfyi + + +mod rustdoc; +mod releases; +mod page; +mod crate_details; +mod source; +mod pool; +mod file; +mod builds; + +use std::fmt; +use std::error::Error; +use std::time::Duration; +use std::path::Path; +use iron::prelude::*; +use iron::{status, Handler}; +use router::Router; +use staticfile::Static; +use handlebars_iron::{HandlebarsEngine, DirectorySource}; +use time; +use postgres::Connection; +use semver::{Version, VersionReq}; +use rustc_serialize::json::{Json, ToJson}; +use std::collections::BTreeMap; + + + +/// Duration of static files for staticfile and DatabaseFileHandler (in seconds) +const STATIC_FILE_CACHE_DURATION: u64 = 60 * 60 * 24 * 3; // 3 days + + +// This is a generic error used in routes +#[derive(Debug)] +struct NoCrate; + +impl fmt::Display for NoCrate { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("Crate not found.") + } +} + +impl Error for NoCrate { + fn description(&self) -> &str { "No Crate" } +} + + + +struct CratesfyiHandler { + router_handler: Box, + database_file_handler: Box, + static_handler: Box, +} + + +impl CratesfyiHandler { + pub fn new() -> CratesfyiHandler { + let mut router = Router::new(); + router.get("/", releases::home_page); + router.get("/releases", releases::releases_handler); + router.get("/releases/:page", releases::releases_handler); + router.get("/rustdoc/:crate", rustdoc::rustdoc_redirector_handler); + router.get("/rustdoc/:crate/", rustdoc::rustdoc_redirector_handler); + router.get("/rustdoc/:crate/:version", rustdoc::rustdoc_redirector_handler); + router.get("/rustdoc/:crate/:version/", rustdoc::rustdoc_redirector_handler); + router.get("/rustdoc/:crate/:version/*", rustdoc::rustdoc_html_server_handler); + router.get("/crates/:name", crate_details::crate_details_handler); + router.get("/crates/:name/", crate_details::crate_details_handler); + router.get("/crates/:name/:version", crate_details::crate_details_handler); + router.get("/crates/:name/:version/", crate_details::crate_details_handler); + router.get("/crates/:name/:version/builds", builds::build_list_handler); + router.get("/crates/:name/:version/builds/:id", builds::build_list_handler); + router.get("/source/:name/:version/", source::source_browser_handler); + router.get("/source/:name/:version/*", source::source_browser_handler); + router.get("/search", releases::search_handler); + + // TODO: Use DocBuilderOptions for paths + let mut hbse = HandlebarsEngine::new(); + hbse.add(Box::new(DirectorySource::new("./templates", ".hbs"))); + + // load templates + if let Err(e) = hbse.reload() { + panic!("Failed to load handlebar templates: {}", e.description()); + } + + let mut router_chain = Chain::new(router); + router_chain.link_before(pool::Pool::new()); + router_chain.link_after(hbse); + + // FIXME: URGENT use correct path + let static_handler = Static::new(Path::new("../cratesfyi-prefix/public_html")) + .cache(Duration::from_secs(STATIC_FILE_CACHE_DURATION)); + + CratesfyiHandler { + router_handler: Box::new(router_chain), + database_file_handler: Box::new(file::DatabaseFileHandler), + static_handler: Box::new(static_handler), + } + } +} + + +impl Handler for CratesfyiHandler { + fn handle(&self, req: &mut Request) -> IronResult { + // try router first then staticfile handler + // return 404 if none of them return Ok + match self.router_handler.handle(req) { + Ok(res) => return Ok(res), + Err(e) => debug!("{}", e.description()) + }; + + // if router fails try to serve files from database first + // and then try static handler. if all of them fails, return 404 + // TODO: Add a custom 404 page + if let Ok(res) = self.database_file_handler.handle(req) { + Ok(res) + } else if let Ok(res) = self.static_handler.handle(req) { + Ok(res) + } else { + Ok(Response::with((status::NotFound, "404 FAILED IN CRATESFYIHANDLER"))) + } + } +} + + + +fn match_version(conn: &Connection, name: &str, version: Option<&str>) -> Option { + + // version is an Option<&str> from router::Router::get + // need to decode first + use url::percent_encoding::percent_decode; + let req_version = version.and_then(|v| { + match percent_decode(v.as_bytes()).decode_utf8() { + Ok(p) => Some(p.into_owned()), + Err(_) => None, + } + }).unwrap_or("*".to_string()); + + let versions = { + + let mut versions = Vec::new(); + // get every version of a crate + for row in &conn.query("SELECT version \ + FROM releases \ + INNER JOIN crates ON crates.id = releases.crate_id \ + WHERE crates.name = $1", + &[&name]) + .unwrap() { + let version: String = row.get(0); + versions.push(version); + } + + // FIXME: Need to sort versions with semver, database is not keeping them sorted + versions + }; + + // first check for exact match + // we can't expect users to use semver in query + for version in &versions { + if version == &req_version { + return Some(version.clone()) + } + } + + // Now try to match with semver + let req_sem_ver = VersionReq::parse(&req_version).unwrap(); + + // we need to sort versions first + let versions_sem = { + let mut versions_sem: Vec = Vec::new(); + + for version in &versions { + versions_sem.push(Version::parse(&version).unwrap()); + } + + versions_sem.sort(); + versions_sem.reverse(); + versions_sem + }; + + for version in &versions_sem { + if req_sem_ver.matches(&version) { + return Some(format!("{}", version)) + } + } + + None +} + + + + + +/// Wrapper around the pulldown-cmark parser and renderer to render markdown +fn render_markdown(text: &str) -> String { + // I got this from mdBook::src::utils + use pulldown_cmark::{Parser, html, Options, OPTION_ENABLE_TABLES, OPTION_ENABLE_FOOTNOTES}; + let mut s = String::with_capacity(text.len() * 3 / 2); + + let mut opts = Options::empty(); + opts.insert(OPTION_ENABLE_TABLES); + opts.insert(OPTION_ENABLE_FOOTNOTES); + + let p = Parser::new_ext(&text, opts); + html::push_html(&mut s, p); + s +} + + + +/// Starts cratesfyi web server +pub fn start_web_server() { + let cratesfyi = CratesfyiHandler::new(); + Iron::new(cratesfyi).http("localhost:3000").unwrap(); +} + + + +/// Converts Timespec to nice readable relative time string +fn duration_to_str(ts: time::Timespec) -> String { + + let tm = time::at(ts); + let delta = time::now() - tm; + + if delta.num_days() > 5 { + format!("{}", tm.strftime("%b %d, %Y").unwrap()) + } else if delta.num_days() > 1 { + format!("{} days ago", delta.num_days()) + } else if delta.num_days() == 1 { + "one day ago".to_string() + } else if delta.num_hours() > 1 { + format!("{} hours ago", delta.num_hours()) + } else if delta.num_hours() == 1 { + "an hour ago".to_string() + } else if delta.num_minutes() > 1 { + format!("{} minutes ago", delta.num_minutes()) + } else if delta.num_minutes() == 1 { + "one minute ago".to_string() + } else if delta.num_seconds() > 0 { + format!("{} seconds ago", delta.num_seconds()) + } else { + "just now".to_string() + } + +} + + + +/// MetaData used in header +#[derive(Debug)] +pub struct MetaData { + pub name: String, + pub version: String, + pub description: Option, + pub target_name: Option, + pub rustdoc_status: bool, +} + + +impl MetaData { + pub fn from_crate(conn: &Connection, name: &str, version: &str) -> Option { + for row in &conn.query("SELECT crates.name, + releases.version, + releases.description, + releases.target_name, + releases.rustdoc_status + FROM releases + INNER JOIN crates ON crates.id = releases.crate_id + WHERE crates.name = $1 AND releases.version = $2", + &[&name, &version]).unwrap() { + + return Some(MetaData { + name: row.get(0), + version: row.get(1), + description: row.get(2), + target_name: row.get(3), + rustdoc_status: row.get(4), + }); + } + + None + } +} + + +impl ToJson for MetaData { + fn to_json(&self) -> Json { + let mut m: BTreeMap = BTreeMap::new(); + m.insert("name".to_owned(), self.name.to_json()); + m.insert("version".to_owned(), self.version.to_json()); + m.insert("description".to_owned(), self.description.to_json()); + m.insert("target_name".to_owned(), self.target_name.to_json()); + m.insert("rustdoc_status".to_owned(), self.rustdoc_status.to_json()); + m.to_json() + } +} + + +#[cfg(test)] +mod test { + extern crate env_logger; + use super::*; + + #[test] + #[ignore] + fn test_start_web_server() { + // FIXME: This test is doing nothing + let _ = env_logger::init(); + start_web_server(); + } +} diff --git a/src/web/page.rs b/src/web/page.rs new file mode 100644 index 000000000..d57b9e607 --- /dev/null +++ b/src/web/page.rs @@ -0,0 +1,89 @@ +//! Generic page struct + +use std::collections::BTreeMap; +use rustc_serialize::json::{Json, ToJson}; +use iron::{IronResult, Set, status}; +use iron::response::Response; +use handlebars_iron::Template; + + +pub struct Page { + title: Option, + content: T, + varss: BTreeMap, + varsb: BTreeMap, + varsi: BTreeMap, +} + + +impl Page { + pub fn new(content: T) -> Page { + Page { + title: None, + content: content, + varss: BTreeMap::new(), + varsb: BTreeMap::new(), + varsi: BTreeMap::new(), + } + } + + /// Sets a string variable + pub fn set(mut self, var: &str, val: &str) -> Page { + &self.varss.insert(var.to_owned(), val.to_owned()); + self + } + + + /// Sets a boolean variable + pub fn set_bool(mut self, var: &str, val: bool) -> Page { + &self.varsb.insert(var.to_owned(), val); + self + } + + + /// Sets a boolean variable to true + pub fn set_true(mut self, var: &str) -> Page { + &self.varsb.insert(var.to_owned(), true); + self + } + + + /// Sets an integer variable + pub fn set_int(mut self, var: &str, val: i64) -> Page { + &self.varsi.insert(var.to_owned(), val); + self + } + + + /// Sets title of page + pub fn title(mut self, title: &str) -> Page { + self.title = Some(title.to_owned()); + self + } + + + pub fn to_resp(self, template: &str) -> IronResult { + let mut resp = Response::new(); + let temp = Template::new(template, self); + resp.set_mut(temp).set_mut(status::Ok); + Ok(resp) + } +} + + +impl ToJson for Page { + fn to_json(&self) -> Json { + let mut tree = BTreeMap::new(); + + if let Some(ref title) = self.title { + tree.insert("title".to_owned(), title.to_json()); + } + + tree.insert("content".to_owned(), self.content.to_json()); + tree.insert("cratesfyi_version".to_owned(), ::BUILD_VERSION.to_json()); + tree.insert("varss".to_owned(), self.varss.to_json()); + tree.insert("varsb".to_owned(), self.varsb.to_json()); + tree.insert("varsi".to_owned(), self.varsi.to_json()); + Json::Object(tree) + } +} diff --git a/src/web/pool.rs b/src/web/pool.rs new file mode 100644 index 000000000..cbd5c2b3b --- /dev/null +++ b/src/web/pool.rs @@ -0,0 +1,29 @@ + + +use iron::prelude::*; +use iron::{BeforeMiddleware, typemap}; +use r2d2; +use r2d2_postgres; +use db::create_pool; + + +pub struct Pool { + pool: r2d2::Pool, +} + +impl typemap::Key for Pool { + type Value = r2d2::PooledConnection; +} + +impl Pool { + pub fn new() -> Pool { + Pool { pool: create_pool() } + } +} + +impl BeforeMiddleware for Pool { + fn before(&self, req: &mut Request) -> IronResult<()> { + req.extensions.insert::(self.pool.get().unwrap()); + Ok(()) + } +} diff --git a/src/web/releases.rs b/src/web/releases.rs new file mode 100644 index 000000000..fce0b5c7f --- /dev/null +++ b/src/web/releases.rs @@ -0,0 +1,201 @@ +//! Releases web handlers + + +use super::{NoCrate, duration_to_str}; +use super::page::Page; +use super::pool::Pool; +use iron::prelude::*; +use iron::status; +use router::Router; +use rustc_serialize::json::{Json, ToJson}; +use std::collections::BTreeMap; +use time; +use postgres::Connection; + + +/// Number of release in home page +const RELEASES_IN_HOME: i64 = 15; +/// Releases in /releases page +const RELEASES_IN_RELEASES: i64 = 30; + + +struct Release { + name: String, + version: String, + description: Option, + target_name: Option, + rustdoc_status: bool, + release_time: time::Timespec, +} + + +impl ToJson for Release { + fn to_json(&self) -> Json { + let mut m: BTreeMap = BTreeMap::new(); + m.insert("name".to_string(), self.name.to_json()); + m.insert("version".to_string(), self.version.to_json()); + m.insert("description".to_string(), self.description.to_json()); + m.insert("target_name".to_string(), self.target_name.to_json()); + m.insert("rustdoc_status".to_string(), self.rustdoc_status.to_json()); + m.insert("release_time".to_string(), + duration_to_str(self.release_time).to_json()); + m.to_json() + } +} + + +fn get_releases(conn: &Connection, page: i64, limit: i64) -> Vec { + let mut packages = Vec::new(); + + let offset = (page - 1) * limit; + + for row in &conn.query("SELECT crates.name, \ + releases.version, \ + releases.description, \ + releases.target_name, \ + releases.release_time, \ + releases.rustdoc_status \ + FROM crates \ + INNER JOIN releases ON crates.id = releases.crate_id \ + ORDER BY releases.release_time DESC \ + LIMIT $1 OFFSET $2", + &[&limit, &offset]) + .unwrap() { + + let package = Release { + name: row.get(0), + version: row.get(1), + description: row.get(2), + target_name: row.get(3), + release_time: row.get(4), + rustdoc_status: row.get(5), + }; + + packages.push(package); + } + + packages +} + + + +fn get_search_results(conn: &Connection, + query: &str, + page: i64, + limit: i64) + -> Option<(i64, Vec)> { + + let offset = (page - 1) * limit; + let mut packages = Vec::new(); + + for row in &conn.query("SELECT crates.name, \ + releases.version, \ + releases.description, \ + releases.target_name, \ + releases.release_time, \ + releases.rustdoc_status, \ + ts_rank_cd(crates.content, to_tsquery($1)) AS rank \ + FROM crates \ + INNER JOIN releases ON crates.latest_version_id = releases.id \ + WHERE crates.content @@ to_tsquery($1) \ + ORDER BY rank DESC \ + LIMIT $2 OFFSET $3", + &[&query, &limit, &offset]) + .unwrap() { + + let package = Release { + name: row.get(0), + version: row.get(1), + description: row.get(2), + target_name: row.get(3), + release_time: row.get(4), + rustdoc_status: row.get(5), + }; + + packages.push(package); + } + + if !packages.is_empty() { + // get count of total results + let rows = conn.query("SELECT COUNT(*) FROM crates WHERE content @@ to_tsquery($1)", + &[&query]) + .unwrap(); + + Some((rows.get(0).get(0), packages)) + } else { + None + } +} + + + +pub fn home_page(req: &mut Request) -> IronResult { + let conn = req.extensions.get::().unwrap(); + let packages = get_releases(conn, 1, RELEASES_IN_HOME); + Page::new(packages) + .set_true("show_search_form") + .set_true("hide_package_navigation") + .to_resp("releases") +} + + +pub fn releases_handler(req: &mut Request) -> IronResult { + + // page number of releases + let page_number: i64 = req.extensions + .get::() + .unwrap() + .find("page") + .unwrap_or("1") + .parse() + .unwrap_or(1); + + let conn = req.extensions.get::().unwrap(); + let packages = get_releases(conn, page_number, RELEASES_IN_RELEASES); + let page = { + let page = Page::new(packages) + .title("Recent Releases") + .set_int("next_page", page_number + 1); + + // Set previous page if we are not in first page + // TODO: Currently, there is no way to know we are on the last page. + // TBH I kinda don't care. COUNT(*) is expensive, and there is more than + // 25k release anyway, I don't think people will check last page. I can cache + // result and use this value for approximation. But since I don't know how to + // do it yet, I will just skip page checking. I can also assume if Package count + // is less than RELEASES_IN_RELEASES, we are on last page. + if page_number == 1 { + page + } else { + page.set_int("previous_page", page_number - 1) + } + }; + + + page.set_int("next_page", page_number + 1).to_resp("releases") +} + + + +pub fn search_handler(req: &mut Request) -> IronResult { + use params::{Params, Value}; + + let params = req.get::().unwrap(); + let query = params.find(&["query"]); + + let conn = req.extensions.get::().unwrap(); + if let Some(&Value::String(ref query)) = query { + let search_query = query.replace(" ", " & "); + get_search_results(&conn, &search_query, 1, RELEASES_IN_RELEASES) + .ok_or(IronError::new(NoCrate, status::NotFound)) + .and_then(|(_, results)| { + // FIXME: There is no pagination + Page::new(results) + .set("search_query", &query) + .title(&format!("Search results for '{}'", query)) + .to_resp("releases") + }) + } else { + Err(IronError::new(NoCrate, status::NotFound)) + } +} diff --git a/src/web/rustdoc.rs b/src/web/rustdoc.rs new file mode 100644 index 000000000..b195c04b6 --- /dev/null +++ b/src/web/rustdoc.rs @@ -0,0 +1,176 @@ +//! rustdoc handler + + +use super::pool::Pool; +use super::file::File; +use super::MetaData; +use iron::prelude::*; +use iron::{status, Url}; +use iron::modifiers::Redirect; +use router::Router; +use super::{NoCrate, match_version}; +use super::page::Page; +use rustc_serialize::json::{Json, ToJson}; +use std::collections::BTreeMap; + + + +#[derive(Debug)] +struct RustdocPage { + pub head: String, + pub body: String, + pub name: String, + pub version: String, + pub description: Option, + pub metadata: Option, +} + + +impl Default for RustdocPage { + fn default() -> RustdocPage { + RustdocPage { + head: String::new(), + body: String::new(), + name: String::new(), + version: String::new(), + description: None, + metadata: None, + } + } +} + + +impl ToJson for RustdocPage { + fn to_json(&self) -> Json { + let mut m: BTreeMap = BTreeMap::new(); + m.insert("rustdoc_head".to_string(), self.head.to_json()); + m.insert("rustdoc_body".to_string(), self.body.to_json()); + m.insert("rustdoc_status".to_string(), true.to_json()); + m.insert("name".to_string(), self.name.to_json()); + m.insert("version".to_string(), self.version.to_json()); + m.insert("description".to_string(), self.description.to_json()); + m.insert("metadata".to_string(), self.metadata.to_json()); + m.to_json() + } +} + + + +pub fn rustdoc_redirector_handler(req: &mut Request) -> IronResult { + + fn redirect_to_doc(req: &Request, + name: &str, + vers: &str, + target_name: &str) + -> IronResult { + let url = Url::parse(&format!("{}://{}:{}/rustdoc/{}/{}/{}/", + req.url.scheme, + req.url.host, + req.url.port, + name, + vers, + target_name)[..]) + .unwrap(); + let mut resp = Response::with((status::Found, Redirect(url))); + + use iron::headers::{Expires, HttpDate}; + use time; + resp.headers.set(Expires(HttpDate(time::now()))); + + Ok(resp) + } + + // this handler should never called without crate pattern + let crate_name = req.extensions.get::().unwrap().find("crate").unwrap(); + let req_version = req.extensions.get::().unwrap().find("version"); + + let conn = req.extensions.get::().unwrap(); + + let version = match match_version(&conn, &crate_name, req_version) { + Some(v) => v, + None => return Err(IronError::new(NoCrate, status::NotFound)), + }; + + // get target name + // FIXME: This is a bit inefficient but allowing us to use less code in general + let target_name: String = conn.query("SELECT target_name FROM releases INNER JOIN crates ON \ + crates.id = releases.crate_id WHERE crates.name = $1 \ + AND releases.version = $2", + &[&crate_name, &version]) + .unwrap() + .get(0) + .get(0); + + redirect_to_doc(req, &crate_name, &version, &target_name) +} + + +pub fn rustdoc_html_server_handler(req: &mut Request) -> IronResult { + + let path = { + let mut path = req.url.path.clone().join("/"); + + if path.ends_with("/") { + path.push_str("index.html") + } + + path + }; + + // don't touch anything other than html files + if !path.ends_with(".html") { + return Err(IronError::new(NoCrate, status::NotFound)); + } + + let conn = req.extensions.get::().unwrap(); + + let file = match File::from_path(&conn, &path) { + Some(f) => f, + None => return Err(IronError::new(NoCrate, status::NotFound)), + }; + + let (mut in_head, mut in_body) = (false, false); + + let mut content = RustdocPage::default(); + + let file_content = String::from_utf8(file.content).unwrap(); + + for line in file_content.lines() { + + if line.starts_with("().unwrap().find("crate").unwrap_or("").to_string(); + let version = req.extensions + .get::() + .unwrap() + .find("version") + .unwrap_or("") + .to_string(); + + content.metadata = MetaData::from_crate(&conn, &name, &version); + + Page::new(content) + .set_true("show_package_navigation") + .set_true("package_navigation_documentation_tab") + .to_resp("rustdoc") +} diff --git a/src/web/source.rs b/src/web/source.rs new file mode 100644 index 000000000..e63127611 --- /dev/null +++ b/src/web/source.rs @@ -0,0 +1,245 @@ +//! Source code browser + + +use std::collections::BTreeMap; +use std::cmp::Ordering; +use super::MetaData; +use super::page::Page; +use super::pool::Pool; +use super::file::File as DbFile; +use iron::prelude::*; +use router::Router; +use rustc_serialize::json::{Json, ToJson}; +use postgres::Connection; + + +#[derive(PartialEq, PartialOrd)] +enum FileType { + Dir, + Text, + Binary, + RustSource, +} + + +#[derive(PartialEq, PartialOrd)] +struct File { + name: String, + file_type: FileType, +} + + +struct FileList { + metadata: MetaData, + files: Vec, +} + + +impl ToJson for FileList { + fn to_json(&self) -> Json { + let mut m: BTreeMap = BTreeMap::new(); + + m.insert("metadata".to_string(), self.metadata.to_json()); + + let mut file_vec: Vec = Vec::new(); + + for file in &self.files { + let mut file_m: BTreeMap = BTreeMap::new(); + file_m.insert("name".to_string(), file.name.to_json()); + + file_m.insert(match file.file_type { + FileType::Dir => "file_type_dir".to_string(), + FileType::Text => "file_type_text".to_string(), + FileType::Binary => "file_type_binary".to_string(), + FileType::RustSource => "file_type_rust_source".to_string(), + }, + true.to_json()); + + file_vec.push(file_m.to_json()); + } + + m.insert("files".to_string(), file_vec.to_json()); + m.to_json() + } +} + + +impl FileList { + /// Gets FileList from a request path + /// + /// All paths stored in database have this format: + /// + /// ```text + /// [ + /// ["text/plain",".gitignore"], + /// ["text/x-c","src/reseeding.rs"], + /// ["text/x-c","src/lib.rs"], + /// ["text/x-c","README.md"], + /// ... + /// ] + /// ``` + /// + /// This function is only returning FileList for requested directory. If is empty, + /// it will return list of files (and dirs) for root directory. req_path must be a + /// directory or empty for root directory. + pub fn from_path(conn: &Connection, + name: &str, + version: &str, + req_path: &str) + -> Option { + + let rows = conn.query("SELECT crates.name, + releases.version, + releases.description, + releases.target_name, + releases.rustdoc_status, + releases.files + FROM releases + LEFT OUTER JOIN crates ON crates.id = releases.crate_id + WHERE crates.name = $1 AND releases.version = $2", + &[&name, &version]) + .unwrap(); + + if rows.len() == 0 { + return None; + } + + let files: Json = rows.get(0).get(5); + + let mut file_list: Vec = Vec::new(); + + files.as_array().map(|files| { + for file in files { + file.as_array().map(|file| { + let mime = file[0].as_string().unwrap(); + let path = file[1].as_string().unwrap(); + + // skip .cargo-ok generated by cargo + if path == ".cargo-ok" { + return; + } + + // look only files for req_path + if path.starts_with(&req_path) { + // remove req_path from path to reach files in this directory + let path = path.replace(&req_path, ""); + let path_splited: Vec<&str> = path.split("/").collect(); + + // if path have '/' it is a directory + let ftype = if path_splited.len() > 1 { + FileType::Dir + } else if mime.starts_with("text") && + path_splited[0].ends_with(".rs") { + FileType::RustSource + } else if mime.starts_with("text") { + FileType::Text + } else { + FileType::Binary + }; + + let file = File { + name: path_splited[0].to_owned(), + file_type: ftype, + }; + + // avoid adding duplicates, a directory may occur more than once + if !file_list.contains(&file) { + file_list.push(file); + } + } + }); + } + }); + + if file_list.is_empty() { + return None; + } + + file_list.sort_by(|a, b| { + // directories must be listed first + if a.file_type == FileType::Dir && b.file_type != FileType::Dir { + Ordering::Less + } else if a.file_type != FileType::Dir && b.file_type == FileType::Dir { + Ordering::Greater + } else { + a.name.to_lowercase().cmp(&b.name.to_lowercase()) + } + }); + + Some(FileList { + metadata: MetaData { + name: rows.get(0).get(0), + version: rows.get(0).get(1), + description: rows.get(0).get(2), + target_name: rows.get(0).get(3), + rustdoc_status: rows.get(0).get(4), + }, + files: file_list, + }) + } +} + + +pub fn source_browser_handler(req: &mut Request) -> IronResult { + let name = req.extensions.get::().unwrap().find("name").unwrap(); + let version = req.extensions.get::().unwrap().find("version").unwrap(); + + // get path (req_path) for FileList::from_path and actual path for super::file::File::from_path + let (req_path, file_path) = { + let mut req_path = req.url.path.clone(); + // remove first element from path which is /source + req_path.remove(0); + let file_path = format!("sources/{}", req_path.join("/")); + + // FileList::from_path is only working for directories + // remove file name if it's not a directory + req_path.last_mut().map(|last| { + if !last.is_empty() { + last.clear(); + } + }); + + // remove crate name and version from req_path + let path = req_path.join("/").replace(&format!("{}/{}/", name, version), ""); + + (path, file_path) + }; + + + let conn = req.extensions.get::().unwrap(); + + // try to get actual file first + // skip if request is a directory + let file = if !file_path.ends_with("/") { + DbFile::from_path(&conn, &file_path) + } else { + None + }; + + + let content = if let Some(file) = file { + // serve the file with DatabaseFileHandler if file isn't text and not empty + if !file.mime.starts_with("text") && !file.is_empty() { + return Ok(file.serve()); + } else if file.mime.starts_with("text") && !file.is_empty() { + String::from_utf8(file.content).ok() + } else { + None + } + } else { + None + }; + + let list = FileList::from_path(&conn, &name, &version, &req_path); + + let page = Page::new(list) + .set_bool("show_parent_link", !req_path.is_empty()) + .set_true("show_package_navigation") + .set_true("package_source_tab"); + + if let Some(content) = content { + page.set("file_content", &content).to_resp("source") + } else { + page.to_resp("source") + } +} diff --git a/templates/builds.hbs b/templates/builds.hbs new file mode 100644 index 000000000..8444c838d --- /dev/null +++ b/templates/builds.hbs @@ -0,0 +1,44 @@ +{{> header}} + +{{#with content}} +
+
+ + {{#if build_details}} +
+ Build #{{build_details.id}} {{build_details.build_time}} +
+ 
+$ rustc --version
+{{build_details.rustc_version}}
+$ cratesfyi --version
+{{build_details.cratesfyi_version}}
+$ cratesfyi ...
+{{build_details.output}}
+
+ {{/if}} + +
+ Builds +
+ + + +
+
+{{/with}} + +{{> footer}} diff --git a/templates/crate_details.hbs b/templates/crate_details.hbs new file mode 100644 index 000000000..6dabaaefb --- /dev/null +++ b/templates/crate_details.hbs @@ -0,0 +1,57 @@ +{{> header}} + + +{{#with content}} +
+
+
+
+ +
+
+
+ {{#unless build_status}} +
cratesfyi failed to build {{name}}-{{version}}
Please check build logs and if you believe this is cratesfyi's fault, report into this issue report.
+ {{else}} + {{#unless rustdoc_status}} +
{{name}}-{{version}} doesn't have any documentation.
+ {{/unless}} + {{/unless}} + {{#if readme}} + {{{readme}}} + {{else}} + {{{rustdoc}}} + {{/if}} +
+
+ +
+{{/with}} + + +{{> footer}} diff --git a/templates/footer.hbs b/templates/footer.hbs new file mode 100644 index 000000000..308b1d01b --- /dev/null +++ b/templates/footer.hbs @@ -0,0 +1,2 @@ + + diff --git a/templates/header.hbs b/templates/header.hbs new file mode 100644 index 000000000..e45630a23 --- /dev/null +++ b/templates/header.hbs @@ -0,0 +1,17 @@ + + + + + + + + + + + + + {{#if title}}{{title}} - {{/if}}{{#if content.metadata.name}}{{content.metadata.name}} {{content.metadata.version}} - {{/if}}Cratesfyi + + + + {{> navigation}} diff --git a/templates/navigation.hbs b/templates/navigation.hbs new file mode 100644 index 000000000..430b253c9 --- /dev/null +++ b/templates/navigation.hbs @@ -0,0 +1,40 @@ +
+
+
+
+ {{#unless varsb.show_search_form}} + + {{/unless}} + Cratesfyi + +
+
+
+
+ + {{#unless varsb.hide_package_navigation}} + {{#with content}} +
+
+

{{#if ../title}}{{../title}}{{else}}{{metadata.name}} {{metadata.version}}{{/if}}

+
{{#if metadata.description }}{{metadata.description}}{{else}}{{../varss.description}}{{/if}}
+ {{#if ../varsb.show_package_navigation}} + {{#with metadata}} +
+ +
+ {{/with}} + {{/if}} +
+
+ {{/with}} + {{/unless}} diff --git a/templates/releases.hbs b/templates/releases.hbs new file mode 100644 index 000000000..d307587a2 --- /dev/null +++ b/templates/releases.hbs @@ -0,0 +1,50 @@ +{{> header}} + +{{#if varsb.show_search_form}} +
+

Cratesfyi

+ +
+
+
+ + +
+
+
+{{/if}} + +
+
+ {{#if varsb.show_search_form}} +
+ Recent Releases +
+ {{/if}} + + + + {{#if varsi.previous_page}} + Newest Releases + {{/if}} + {{#unless varsb.show_search_form}} + {{#if varsi.next_page}} + Oldest Releases + {{/if}} + {{/unless}} +
+
+ +{{> footer}} diff --git a/templates/rustdoc.hbs b/templates/rustdoc.hbs new file mode 100644 index 000000000..79ad37470 --- /dev/null +++ b/templates/rustdoc.hbs @@ -0,0 +1,17 @@ + + + + {{{content.rustdoc_head}}} + + + + + + + + {{> navigation}} +
+ {{{content.rustdoc_body}}} +
+ + diff --git a/templates/source.hbs b/templates/source.hbs new file mode 100644 index 000000000..ed2e059f2 --- /dev/null +++ b/templates/source.hbs @@ -0,0 +1,38 @@ +{{> header}} + + +{{#with content}} +
+
+
+
+ +
+
+ {{#if ../varss.file_content}} +
+ 
{{ ../varss.file_content }}
+
+ {{/if}} +
+
+{{/with}} + +{{> footer}} diff --git a/templates/style.scss b/templates/style.scss new file mode 100644 index 000000000..b786d7c12 --- /dev/null +++ b/templates/style.scss @@ -0,0 +1,390 @@ + +// FONTS +$font-family-sans: "Fira Sans", "Helvetica Neue", Helvetica, Arial, sans-serif; +$font-family-serif: "Source Serif Pro",Georgia,Times,"Times New Roman",serif; +$font-family-mono: "Source Code Pro", Menlo, Monaco, Consolas, "DejaVu Sans Mono", Inconsolata, monospace; + + +// COLORS - Guess what? +$color-standard: #000; // pure black +$color-url: #4d76ae; // blue +$color-macro: #068000; // green +$color-struct: #df3600; // red +$color-enum: #5e9766; // light green +$color-type: #e57300; // orange +$color-keyword: #8959A8; // purple +$color-string: #718C00; // greenish +$color-macro-in-code: #3E999F; // blueish +$color-lifetime-incode: #B76514; // orangish +$color-comment-in-code: #8E908C; // light gray +$color-background-code: #F5F5F5; // lighter gray +$color-border: #ddd; // gray + + + +// pure compatible media queries +$media-sm: "screen and (min-width: 35.5em)"; +$media-md: "screen and (min-width: 48em)"; +$media-lg: "screen and (min-width: 64em)"; +$media-xl: "screen and (min-width: 80em)"; +// usage: +// @media #{$media-sm} { ... } + + +html, button, input, select, textarea, +.pure-g [class *= "pure-u"] { + font-family: $font-family-sans; + color: $color-standard; +} + +strong { + font-weight: 500; +} + +pre { + background-color: $color-background-code; + padding: 14px; +} + +.pure-button-normal { + background-color: #fff; + box-sizing: border-box !important; + border: 1px solid $color-border; +} + +.description { + font-family: $font-family-serif; +} + +// rustdoc overrides +div.rustdoc { + font-family: $font-family-serif; + padding-right: 15px; +} + +.sidebar { + top: 150px; + left: auto; +} + +body { + padding: 0; + margin: 0; +} + + +// rustdocs have 200px sidebar and +// max-width 960px main pane +// BUT I really want to make the website centered + +body { + text-align: center; + font: 16px/1.4 $font-family-sans; +} + +div.container { + max-width: 1160px; + margin: 0 auto; + text-align: left; +} + +div.nav-container { + border-bottom: 1px solid $color-border; + + a { + font-weight: 400; + } + + form.landing-search-form-nav { + input.search-input { + float: right; + max-width: 200px; + border: none; + margin: 0; + font-size: 1em; + text-align: right; + box-shadow: none; + height: 38px; + } + } +} + +div.landing { + text-align: center; + padding-top: 30px; + padding-bottom: 60px; + + h1.brand { + font-size: 3em; + margin-bottom: 10px; + } + + form.landing-search-form { + max-width: 400px; + margin: 0 auto; + + div.buttons { + margin-top: 20px; + } + + } + +} + + + +div.recent-releases-container { + text-align: left; + + ul, li { + list-style-type: none; + margin: 0; + padding: 0; + } + + .release { + display: block; + border-bottom: 1px solid $color-border; + padding: .4em 1em; + + @media #{$media-lg} { + padding: .4em 0; + margin: 0 1em; + } + } + + .release:hover { + background-color: $color-background-code; + } + + .name { + color: $color-url; + font-weight: 500; + } + + .description { + font-family: $font-family-serif; + font-weight: normal; + @media #{$media-sm} { + font-size: 1em; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + } + } + + .description:hover { + @media #{$media-sm} { + overflow: visible; + white-space: normal; + } + } + + .date { + font-weight: normal; + @media #{$media-sm} { + text-align: right; + } + } + + .pure-button { + margin: .4em 1em; + } +} + + +div.package-container { + background-color: $color-url; + color: $color-background-code;; + + h1 { + margin: 0; + padding: 20px 0 0 16px; + } + p { + margin: 0; + padding: 0 0 20px 16px; + } + + .pure-menu { + + .pure-menu-link { + background-color: #fff; + border-top: 1px solid $color-border; + border-left: 1px solid $color-border; + border-right: 1px solid $color-border; + border-top-left-radius: 4px; + border-top-right-radius: 4px; + border-bottom: 2px solid $color-border; + padding: .4em 1em; + } + + .pure-menu-active { + border-bottom: 2px solid #fff; + color: $color-standard; + } + + .pure-menu-link:hover { + color: $color-standard; + } + } +} + + +div.package-sheet-container { + margin-top: 10px; + margin-bottom: 20px; + + .pure-menu-link { + border-radius: 4px; + padding: .2em .8em; + font-weight: 400; + } + + .build-success { + color: $color-macro; + } + + .build-fail { + color: $color-struct; + } +} + +div.package-page-container { + div.package-menu { + padding: 0 10px; + + li.pure-menu-heading { + font-size: 1.3em; + color: #000; + font-weight: 500; + text-align: center; + border-bottom: 1px solid lighten($color-border, 5%); + text-transform: none; + padding-bottom: 6px; + margin: 20px 0 15px 0; + } + + li.pure-menu-heading:first-child { + margin-top: 0; + } + + li i.fa { + display: inline-block; + width: 20px; + } + + a.pure-menu-link { + font-size: 14px; + color: $color-standard; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + padding: 7px 5px; + } + + a.pure-menu-link:hover { + background-color: $color-background-code; + } + + div.sub-menu { + max-height: 135px; + overflow-y: auto; + + ul.pure-menu-list { + border-top: none; + } + + li.pure-menu-item:last-child { + border-bottom: none; + } + } + } + + div.package-details p { + font-family: $font-family-serif; + } +} + + + +div.cratesfyi-package-container { + text-align: left; + background-color: $color-background-code; + border-bottom: 1px solid $color-border; + margin-bottom: 20px; + + h1 { + margin: 0; + padding: 20px 0 0 16px; + } + div.description { + font-family: $font-family-serif; + margin: 0; + padding: 0 0 20px 16px; + + @media #{$media-sm} { + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + } + } + + .pure-menu { + margin-bottom: -1px; + padding-left: 15px; + + .pure-menu-link { + color: #666; + font-size: 14px; + padding: .4em 1em .3em 1em; + } + + .pure-menu-active { + color: $color-standard; + + background-color: #fff; + border-top: 1px solid $color-border; + border-left: 1px solid $color-border; + border-right: 1px solid $color-border; + border-top-left-radius: 4px; + border-top-right-radius: 4px; + border-bottom: 2px solid #fff; + } + + .pure-menu-active:hover { + background-color: #fff !important; + } + + .pure-menu-link:hover { + color: #000; + background-color: inherit; + } + } +} + + +div.warning { + border-radius: 4px; + background-color: lighten($color-type, 45%); + padding: .4em 1em; + text-align: center; + a { + color: $color-url; + text-decoration: underline; + } +} + + +div.search-page-search-form { + padding: .4em 1em; + text-align: center; + + input.search-input { + display: inline-block; + max-width: 300px; + padding: .4em 1em; + } + + +} From 8b0e94335d4f53e337e639ab05d3bd08166deb15 Mon Sep 17 00:00:00 2001 From: Onur Aslan Date: Thu, 23 Jun 2016 17:31:28 +0300 Subject: [PATCH 02/79] Use different class than search-input rustdoc is using search-input class for searches. Using search-input is breaking rustdoc searches. Use search-input-nav instead of search-input for search input in navigation. --- templates/navigation.hbs | 2 +- templates/style.scss | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/templates/navigation.hbs b/templates/navigation.hbs index 430b253c9..b544ba31f 100644 --- a/templates/navigation.hbs +++ b/templates/navigation.hbs @@ -3,7 +3,7 @@
{{#unless varsb.show_search_form}} - + {{/unless}} Cratesfyi
    diff --git a/templates/style.scss b/templates/style.scss index b786d7c12..3a1ad09e5 100644 --- a/templates/style.scss +++ b/templates/style.scss @@ -96,7 +96,7 @@ div.nav-container { } form.landing-search-form-nav { - input.search-input { + input.search-input-nav { float: right; max-width: 200px; border: none; @@ -106,6 +106,9 @@ div.nav-container { box-shadow: none; height: 38px; } + input.search-input-nav:focus { + outline: unset; + } } } From 04ff2f815c78de7e5fadc1f8def83078d41645bf Mon Sep 17 00:00:00 2001 From: Onur Aslan Date: Fri, 24 Jun 2016 00:25:04 +0300 Subject: [PATCH 03/79] Syntax highlighting with highlightjs Also remove padding of first header in README, and remove some trailing whitespaces in style.scss --- src/web/crate_details.rs | 1 + src/web/source.rs | 13 ++++++++----- templates/footer.hbs | 1 + templates/header.hbs | 5 +++++ templates/source.hbs | 2 +- templates/style.scss | 40 ++++++++++++++++++++++++++++------------ 6 files changed, 44 insertions(+), 18 deletions(-) diff --git a/src/web/crate_details.rs b/src/web/crate_details.rs index 3f2c9551f..e96df5ad9 100644 --- a/src/web/crate_details.rs +++ b/src/web/crate_details.rs @@ -178,6 +178,7 @@ pub fn crate_details_handler(req: &mut Request) -> IronResult { .and_then(|details| { Page::new(details) .set_true("show_package_navigation") + .set_true("javascript_highlightjs") .set_true("package_navigation_crate_tab") .to_resp("crate_details") }) diff --git a/src/web/source.rs b/src/web/source.rs index e63127611..f5470b918 100644 --- a/src/web/source.rs +++ b/src/web/source.rs @@ -217,28 +217,31 @@ pub fn source_browser_handler(req: &mut Request) -> IronResult { }; - let content = if let Some(file) = file { + let (content, is_rust_source) = if let Some(file) = file { // serve the file with DatabaseFileHandler if file isn't text and not empty if !file.mime.starts_with("text") && !file.is_empty() { return Ok(file.serve()); } else if file.mime.starts_with("text") && !file.is_empty() { - String::from_utf8(file.content).ok() + (String::from_utf8(file.content).ok(), file.path.ends_with(".rs")) } else { - None + (None, false) } } else { - None + (None, false) }; let list = FileList::from_path(&conn, &name, &version, &req_path); let page = Page::new(list) .set_bool("show_parent_link", !req_path.is_empty()) + .set_true("javascript_highlightjs") .set_true("show_package_navigation") .set_true("package_source_tab"); if let Some(content) = content { - page.set("file_content", &content).to_resp("source") + page.set("file_content", &content) + .set_bool("file_content_rust_source", is_rust_source) + .to_resp("source") } else { page.to_resp("source") } diff --git a/templates/footer.hbs b/templates/footer.hbs index 308b1d01b..aca7519a0 100644 --- a/templates/footer.hbs +++ b/templates/footer.hbs @@ -1,2 +1,3 @@ +{{#if varsb.javascript_highlightjs}}{{/if}} diff --git a/templates/header.hbs b/templates/header.hbs index e45630a23..bcec57fa4 100644 --- a/templates/header.hbs +++ b/templates/header.hbs @@ -10,6 +10,11 @@ + {{#if varsb.javascript_highlightjs}} + + + + {{/if}} {{#if title}}{{title}} - {{/if}}{{#if content.metadata.name}}{{content.metadata.name}} {{content.metadata.version}} - {{/if}}Cratesfyi diff --git a/templates/source.hbs b/templates/source.hbs index ed2e059f2..8a554d498 100644 --- a/templates/source.hbs +++ b/templates/source.hbs @@ -28,7 +28,7 @@
{{#if ../varss.file_content}}
- 
{{ ../varss.file_content }}
+ 
{{ ../varss.file_content }}
{{/if}} diff --git a/templates/style.scss b/templates/style.scss index 3a1ad09e5..825965fcd 100644 --- a/templates/style.scss +++ b/templates/style.scss @@ -41,11 +41,6 @@ strong { font-weight: 500; } -pre { - background-color: $color-background-code; - padding: 14px; -} - .pure-button-normal { background-color: #fff; box-sizing: border-box !important; @@ -149,7 +144,7 @@ div.recent-releases-container { display: block; border-bottom: 1px solid $color-border; padding: .4em 1em; - + @media #{$media-lg} { padding: .4em 0; margin: 0 1em; @@ -168,7 +163,7 @@ div.recent-releases-container { .description { font-family: $font-family-serif; font-weight: normal; - @media #{$media-sm} { + @media #{$media-sm} { font-size: 1em; white-space: nowrap; overflow: hidden; @@ -177,7 +172,7 @@ div.recent-releases-container { } .description:hover { - @media #{$media-sm} { + @media #{$media-sm} { overflow: visible; white-space: normal; } @@ -304,8 +299,29 @@ div.package-page-container { } } - div.package-details p { - font-family: $font-family-serif; + div.package-details { + p { + font-family: $font-family-serif; + } + + h1:first-child, + h2:first-child, + h3:first-child, + h4:first-child, + h5:first-child, + h6:first-child { + margin-top: 0; + } + } + + pre { + background-color: inherit; + margin: 0; + padding: 0; + + code { + white-space: pre; + } } } @@ -326,7 +342,7 @@ div.cratesfyi-package-container { margin: 0; padding: 0 0 20px 16px; - @media #{$media-sm} { + @media #{$media-sm} { white-space: nowrap; overflow: hidden; text-overflow: ellipsis; @@ -388,6 +404,6 @@ div.search-page-search-form { max-width: 300px; padding: .4em 1em; } - + } From f833e0fdfd3616f33d7773cb9743bcf942fc515a Mon Sep 17 00:00:00 2001 From: Onur Aslan Date: Sun, 26 Jun 2016 12:55:14 +0300 Subject: [PATCH 04/79] Add most starred crates --- src/web/mod.rs | 4 +- src/web/releases.rs | 148 +++++++++++++++++++++++++++++---------- templates/navigation.hbs | 9 +++ templates/releases.hbs | 19 +++-- templates/style.scss | 13 +++- 5 files changed, 147 insertions(+), 46 deletions(-) diff --git a/src/web/mod.rs b/src/web/mod.rs index bebe2dedb..7e9726778 100644 --- a/src/web/mod.rs +++ b/src/web/mod.rs @@ -59,7 +59,9 @@ impl CratesfyiHandler { let mut router = Router::new(); router.get("/", releases::home_page); router.get("/releases", releases::releases_handler); - router.get("/releases/:page", releases::releases_handler); + router.get("/releases/recent/:page", releases::releases_handler); + router.get("/releases/stars", releases::stars_handler); + router.get("/releases/stars/:page", releases::stars_handler); router.get("/rustdoc/:crate", rustdoc::rustdoc_redirector_handler); router.get("/rustdoc/:crate/", rustdoc::rustdoc_redirector_handler); router.get("/rustdoc/:crate/:version", rustdoc::rustdoc_redirector_handler); diff --git a/src/web/releases.rs b/src/web/releases.rs index fce0b5c7f..57ae4e8f1 100644 --- a/src/web/releases.rs +++ b/src/web/releases.rs @@ -26,6 +26,22 @@ struct Release { target_name: Option, rustdoc_status: bool, release_time: time::Timespec, + stars: i32, +} + + +impl Default for Release { + fn default() -> Release { + Release { + name: String::new(), + version: String::new(), + description: None, + target_name: None, + rustdoc_status: false, + release_time: time::get_time(), + stars: 0, + } + } } @@ -39,29 +55,51 @@ impl ToJson for Release { m.insert("rustdoc_status".to_string(), self.rustdoc_status.to_json()); m.insert("release_time".to_string(), duration_to_str(self.release_time).to_json()); + m.insert("stars".to_string(), self.stars.to_json()); m.to_json() } } -fn get_releases(conn: &Connection, page: i64, limit: i64) -> Vec { - let mut packages = Vec::new(); +enum Order { + ReleaseTime, // this is default order + GithubStars, +} + + +fn get_releases(conn: &Connection, page: i64, limit: i64, order: Order) -> Vec { let offset = (page - 1) * limit; - for row in &conn.query("SELECT crates.name, \ - releases.version, \ - releases.description, \ - releases.target_name, \ - releases.release_time, \ - releases.rustdoc_status \ - FROM crates \ - INNER JOIN releases ON crates.id = releases.crate_id \ - ORDER BY releases.release_time DESC \ - LIMIT $1 OFFSET $2", - &[&limit, &offset]) - .unwrap() { + // TODO: This function changed so much during development and current version have code + // repeats for queries. There is definitely room for improvements. + let query = match order { + Order::ReleaseTime => "SELECT crates.name, \ + releases.version, \ + releases.description, \ + releases.target_name, \ + releases.release_time, \ + releases.rustdoc_status, \ + crates.github_stars \ + FROM crates \ + INNER JOIN releases ON crates.id = releases.crate_id \ + ORDER BY releases.release_time DESC \ + LIMIT $1 OFFSET $2", + Order::GithubStars => "SELECT crates.name, \ + releases.version, \ + releases.description, \ + releases.target_name, \ + releases.release_time, \ + releases.rustdoc_status, \ + crates.github_stars \ + FROM crates \ + INNER JOIN releases ON releases.id = crates.latest_version_id \ + ORDER BY crates.github_stars DESC \ + LIMIT $1 OFFSET $2", + }; + let mut packages = Vec::new(); + for row in &conn.query(&query, &[&limit, &offset]).unwrap() { let package = Release { name: row.get(0), version: row.get(1), @@ -69,6 +107,7 @@ fn get_releases(conn: &Connection, page: i64, limit: i64) -> Vec { target_name: row.get(3), release_time: row.get(4), rustdoc_status: row.get(5), + stars: row.get(6), }; packages.push(package); @@ -110,6 +149,7 @@ fn get_search_results(conn: &Connection, target_name: row.get(3), release_time: row.get(4), rustdoc_status: row.get(5), + ..Release::default() }; packages.push(package); @@ -131,7 +171,7 @@ fn get_search_results(conn: &Connection, pub fn home_page(req: &mut Request) -> IronResult { let conn = req.extensions.get::().unwrap(); - let packages = get_releases(conn, 1, RELEASES_IN_HOME); + let packages = get_releases(conn, 1, RELEASES_IN_HOME, Order::ReleaseTime); Page::new(packages) .set_true("show_search_form") .set_true("hide_package_navigation") @@ -140,7 +180,6 @@ pub fn home_page(req: &mut Request) -> IronResult { pub fn releases_handler(req: &mut Request) -> IronResult { - // page number of releases let page_number: i64 = req.extensions .get::() @@ -151,31 +190,68 @@ pub fn releases_handler(req: &mut Request) -> IronResult { .unwrap_or(1); let conn = req.extensions.get::().unwrap(); - let packages = get_releases(conn, page_number, RELEASES_IN_RELEASES); - let page = { - let page = Page::new(packages) - .title("Recent Releases") - .set_int("next_page", page_number + 1); - - // Set previous page if we are not in first page - // TODO: Currently, there is no way to know we are on the last page. - // TBH I kinda don't care. COUNT(*) is expensive, and there is more than - // 25k release anyway, I don't think people will check last page. I can cache - // result and use this value for approximation. But since I don't know how to - // do it yet, I will just skip page checking. I can also assume if Package count - // is less than RELEASES_IN_RELEASES, we are on last page. - if page_number == 1 { - page - } else { - page.set_int("previous_page", page_number - 1) - } - }; + let packages = get_releases(conn, page_number, RELEASES_IN_RELEASES, Order::ReleaseTime); + if packages.is_empty() { + return Err(IronError::new(NoCrate, status::NotFound)); + } - page.set_int("next_page", page_number + 1).to_resp("releases") + // Show next and previous page buttons + // This is a temporary solution to avoid expensive COUNT(*) + let (show_next_page, show_previous_page) = (packages.len() == RELEASES_IN_RELEASES as usize, + page_number != 1); + + Page::new(packages) + .title("Releases") + .set("description", "Recently uploaded crates") + .set("release_type", "recent") + .set_true("show_releases_navigation") + .set_true("releases_navigation_recent_tab") + .set_bool("show_next_page_button", show_next_page) + .set_int("next_page", page_number + 1) + .set_bool("show_previous_page_button", show_previous_page) + .set_int("previous_page", page_number - 1) + .to_resp("releases") } +// TODO: This function is almost identical to previous one +pub fn stars_handler(req: &mut Request) -> IronResult { + // page number of releases + let page_number: i64 = req.extensions + .get::() + .unwrap() + .find("page") + .unwrap_or("1") + .parse() + .unwrap_or(1); + + let conn = req.extensions.get::().unwrap(); + let packages = get_releases(conn, page_number, RELEASES_IN_RELEASES, Order::GithubStars); + + if packages.is_empty() { + return Err(IronError::new(NoCrate, status::NotFound)); + } + + // Show next and previous page buttons + // This is a temporary solution to avoid expensive COUNT(*) + let (show_next_page, show_previous_page) = (packages.len() == RELEASES_IN_RELEASES as usize, + page_number != 1); + + Page::new(packages) + .title("Releases") + .set("description", "Most starred crates") + .set("release_type", "stars") + .set_true("show_releases_navigation") + .set_true("releases_navigation_stars_tab") + .set_true("show_stars") + .set_bool("show_next_page_button", show_next_page) + .set_int("next_page", page_number + 1) + .set_bool("show_previous_page_button", show_previous_page) + .set_int("previous_page", page_number - 1) + .to_resp("releases") +} + pub fn search_handler(req: &mut Request) -> IronResult { use params::{Params, Value}; diff --git a/templates/navigation.hbs b/templates/navigation.hbs index b544ba31f..8cc2ccb02 100644 --- a/templates/navigation.hbs +++ b/templates/navigation.hbs @@ -34,6 +34,15 @@ {{/with}} {{/if}} + {{#if ../varsb.show_releases_navigation}} +
+ +
+ {{/if}} {{/with}} diff --git a/templates/releases.hbs b/templates/releases.hbs index d307587a2..72a4fe58a 100644 --- a/templates/releases.hbs +++ b/templates/releases.hbs @@ -29,20 +29,25 @@
{{name}}-{{version}}
{{description}}
+ {{#unless ../../varsb.show_stars}}
{{release_time}}
+ {{else}} +
{{stars}}
+ {{/unless}}
{{/each}} - - {{#if varsi.previous_page}} - Newest Releases - {{/if}} {{#unless varsb.show_search_form}} - {{#if varsi.next_page}} - Oldest Releases - {{/if}} + {{/unless}} diff --git a/templates/style.scss b/templates/style.scss index 825965fcd..05bdb0e7d 100644 --- a/templates/style.scss +++ b/templates/style.scss @@ -155,6 +155,10 @@ div.recent-releases-container { background-color: $color-background-code; } + li:last-child .release { + border-bottom: none; + } + .name { color: $color-url; font-weight: 500; @@ -185,8 +189,13 @@ div.recent-releases-container { } } - .pure-button { - margin: .4em 1em; + div.pagination { + text-align: center; + margin: 1em; + + .pure-button { + margin: 0; + } } } From e2e7ef9e09431c99b2023ec488bf34011fe93919 Mon Sep 17 00:00:00 2001 From: Onur Aslan Date: Sun, 3 Jul 2016 01:14:07 +0300 Subject: [PATCH 05/79] Add author handler --- src/web/mod.rs | 2 + src/web/releases.rs | 104 ++++++++++++++++++++++++++++++++++-- templates/crate_details.hbs | 2 +- templates/navigation.hbs | 4 +- 4 files changed, 106 insertions(+), 6 deletions(-) diff --git a/src/web/mod.rs b/src/web/mod.rs index 7e9726778..12cf91066 100644 --- a/src/web/mod.rs +++ b/src/web/mod.rs @@ -62,6 +62,8 @@ impl CratesfyiHandler { router.get("/releases/recent/:page", releases::releases_handler); router.get("/releases/stars", releases::stars_handler); router.get("/releases/stars/:page", releases::stars_handler); + router.get("/releases/:author", releases::author_handler); + router.get("/releases/:author/:page", releases::author_handler); router.get("/rustdoc/:crate", rustdoc::rustdoc_redirector_handler); router.get("/rustdoc/:crate/", rustdoc::rustdoc_redirector_handler); router.get("/rustdoc/:crate/:version", rustdoc::rustdoc_redirector_handler); diff --git a/src/web/releases.rs b/src/web/releases.rs index 57ae4e8f1..3d1a32449 100644 --- a/src/web/releases.rs +++ b/src/web/releases.rs @@ -74,7 +74,8 @@ fn get_releases(conn: &Connection, page: i64, limit: i64, order: Order) -> Vec "SELECT crates.name, \ + Order::ReleaseTime => { + "SELECT crates.name, \ releases.version, \ releases.description, \ releases.target_name, \ @@ -84,8 +85,10 @@ fn get_releases(conn: &Connection, page: i64, limit: i64, order: Order) -> Vec "SELECT crates.name, \ + LIMIT $1 OFFSET $2" + } + Order::GithubStars => { + "SELECT crates.name, \ releases.version, \ releases.description, \ releases.target_name, \ @@ -95,7 +98,8 @@ fn get_releases(conn: &Connection, page: i64, limit: i64, order: Order) -> Vec Vec (String, Vec) { + + let offset = (page - 1) * limit; + + let query = "SELECT crates.name, + releases.version, + releases.description, + releases.target_name, + releases.release_time, + releases.rustdoc_status, + crates.github_stars, + authors.name + FROM crates + INNER JOIN releases ON releases.id = crates.latest_version_id + INNER JOIN author_rels ON releases.id = author_rels.rid + INNER JOIN authors ON authors.id = author_rels.aid + WHERE authors.slug = $1 + ORDER BY crates.github_stars DESC + LIMIT $2 OFFSET $3"; + + let mut author_name = String::new(); + let mut packages = Vec::new(); + for row in &conn.query(&query, &[&author, &limit, &offset]).unwrap() { + let package = Release { + name: row.get(0), + version: row.get(1), + description: row.get(2), + target_name: row.get(3), + release_time: row.get(4), + rustdoc_status: row.get(5), + stars: row.get(6), + }; + + author_name = row.get(7); + packages.push(package); + } + + (author_name, packages) +} + + + fn get_search_results(conn: &Connection, query: &str, page: i64, @@ -253,6 +303,52 @@ pub fn stars_handler(req: &mut Request) -> IronResult { } +pub fn author_handler(req: &mut Request) -> IronResult { + // page number of releases + let page_number: i64 = req.extensions + .get::() + .unwrap() + .find("page") + .unwrap_or("1") + .parse() + .unwrap_or(1); + + let conn = req.extensions.get::().unwrap(); + let author = req.extensions.get::().unwrap().find("author"); + + if author.is_none() { + return Err(IronError::new(NoCrate, status::NotFound)); + } + + let (author_name, packages) = get_releases_by_author(conn, + page_number, + RELEASES_IN_RELEASES, + author.unwrap()); + + if packages.is_empty() { + return Err(IronError::new(NoCrate, status::NotFound)); + } + + // Show next and previous page buttons + // This is a temporary solution to avoid expensive COUNT(*) + let (show_next_page, show_previous_page) = (packages.len() == RELEASES_IN_RELEASES as usize, + page_number != 1); + + Page::new(packages) + .title("Releases") + .set("description", &format!("Crates from {}", author_name)) + .set("author", &author_name) + .set("release_type", &author.unwrap()) + .set_true("show_releases_navigation") + .set_true("show_stars") + .set_bool("show_next_page_button", show_next_page) + .set_int("next_page", page_number + 1) + .set_bool("show_previous_page_button", show_previous_page) + .set_int("previous_page", page_number - 1) + .to_resp("releases") +} + + pub fn search_handler(req: &mut Request) -> IronResult { use params::{Params, Value}; diff --git a/templates/crate_details.hbs b/templates/crate_details.hbs index 6dabaaefb..8102c2dae 100644 --- a/templates/crate_details.hbs +++ b/templates/crate_details.hbs @@ -9,7 +9,7 @@
  • Authors
    • {{#each authors}} -
  • {{this.[0]}}
    • +
  • {{this.[0]}}
    • {{/each}}
  • Dependencies
  • diff --git a/templates/navigation.hbs b/templates/navigation.hbs index 8cc2ccb02..2cd16953d 100644 --- a/templates/navigation.hbs +++ b/templates/navigation.hbs @@ -39,7 +39,9 @@ {{/if}} From 1b91f743c047cff604a83e89caa04acd1abc3581 Mon Sep 17 00:00:00 2001 From: Onur Aslan Date: Sun, 3 Jul 2016 01:48:21 +0300 Subject: [PATCH 06/79] Fix grid size --- templates/builds.hbs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/templates/builds.hbs b/templates/builds.hbs index 8444c838d..a0f8ad64f 100644 --- a/templates/builds.hbs +++ b/templates/builds.hbs @@ -27,10 +27,10 @@ $ cratesfyi ...
  • -
    -
    {{rustc_version}}
    -
    {{cratesfyi_version}}
    -
    {{build_time}}
    +
    +
    {{rustc_version}}
    +
    {{cratesfyi_version}}
    +
    {{build_time}}
    • From ee96a6600aced23c76c34cd5665da152b443ec49 Mon Sep 17 00:00:00 2001 From: Onur Aslan Date: Sun, 3 Jul 2016 01:50:38 +0300 Subject: [PATCH 07/79] Add right margin to search input in navigation --- templates/style.scss | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/style.scss b/templates/style.scss index 05bdb0e7d..5ac0caf67 100644 --- a/templates/style.scss +++ b/templates/style.scss @@ -95,7 +95,7 @@ div.nav-container { float: right; max-width: 200px; border: none; - margin: 0; + margin: 0 1em 0 0; font-size: 1em; text-align: right; box-shadow: none; From 5707a24de1694d5cdcdca86461bc7591099cc2d5 Mon Sep 17 00:00:00 2001 From: Onur Aslan Date: Sun, 3 Jul 2016 13:09:05 +0300 Subject: [PATCH 08/79] Show github fields (stars, forks, issue count) --- src/web/crate_details.rs | 22 +++++++++++++++++++++- templates/crate_details.hbs | 9 +++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/src/web/crate_details.rs b/src/web/crate_details.rs index e96df5ad9..c685aa738 100644 --- a/src/web/crate_details.rs +++ b/src/web/crate_details.rs @@ -36,6 +36,10 @@ struct CrateDetails { have_examples: bool, // need to check this manually target_name: Option, versions: Vec, + github: bool, // is crate hosted in github + github_stars: Option, + github_forks: Option, + github_issues: Option, metadata: MetaData, } @@ -65,6 +69,10 @@ impl ToJson for CrateDetails { m.insert("have_examples".to_string(), self.have_examples.to_json()); m.insert("target_name".to_string(), self.target_name.to_json()); m.insert("versions".to_string(), self.versions.to_json()); + m.insert("github".to_string(), self.github.to_json()); + m.insert("github_stars".to_string(), self.github_stars.to_json()); + m.insert("github_forks".to_string(), self.github_forks.to_json()); + m.insert("github_issues".to_string(), self.github_forks.to_json()); m.insert("metadata".to_string(), self.metadata.to_json()); m.to_json() } @@ -92,7 +100,10 @@ impl CrateDetails { releases.target_name, \ crates.versions, \ authors.name, \ - authors.slug \ + authors.slug, \ + crates.github_stars, \ + crates.github_forks, \ + crates.github_issues \ FROM author_rels \ LEFT OUTER JOIN authors ON authors.id = author_rels.aid \ LEFT OUTER JOIN releases ON releases.id = author_rels.rid \ @@ -151,9 +162,18 @@ impl CrateDetails { have_examples: rows.get(0).get(13), target_name: rows.get(0).get(14), versions: versions, + github: false, + github_stars: rows.get(0).get(18), + github_forks: rows.get(0).get(19), + github_issues: rows.get(0).get(20), metadata: metadata, }; + if let Some(repository_url) = crate_details.repository_url.clone() { + crate_details.github = repository_url.starts_with("http://github.com") || + repository_url.starts_with("https://github.com"); + } + // Insert authors with name and slug for row in &rows { crate_details.authors.push((row.get(16), row.get(17))); diff --git a/templates/crate_details.hbs b/templates/crate_details.hbs index 8102c2dae..eedd091b8 100644 --- a/templates/crate_details.hbs +++ b/templates/crate_details.hbs @@ -11,6 +11,15 @@ {{#each authors}}
  • {{this.[0]}}
    • {{/each}} +
  • Links
    • + {{#if homepage_url}}
  • Homepage
    • {{/if}} + {{#if github}} +
  • + {{github_stars}} {{github_forks}} {{github_issues}} +
    • + {{else}} + {{#if repository_url}}
  • Repository
    • {{/if}} + {{/if}}
  • Dependencies
  • From fe757036b25aa18bcb244edaefd6d0594d4b6cee Mon Sep 17 00:00:00 2001 From: Onur Aslan Date: Sun, 3 Jul 2016 13:21:52 +0300 Subject: [PATCH 09/79] Improve responsive layout --- templates/crate_details.hbs | 4 ++-- templates/source.hbs | 4 ++-- templates/style.scss | 16 ++++++++++++---- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/templates/crate_details.hbs b/templates/crate_details.hbs index eedd091b8..d8981c614 100644 --- a/templates/crate_details.hbs +++ b/templates/crate_details.hbs @@ -4,7 +4,7 @@ {{#with content}}
    -
    +
    • Authors
      • @@ -43,7 +43,7 @@
    -
    +
    {{#unless build_status}}
    cratesfyi failed to build {{name}}-{{version}}
    Please check build logs and if you believe this is cratesfyi's fault, report into this issue report.
    {{else}} diff --git a/templates/source.hbs b/templates/source.hbs index 8a554d498..8ab5a72ad 100644 --- a/templates/source.hbs +++ b/templates/source.hbs @@ -4,7 +4,7 @@ {{#with content}}
    -
    +
      {{#if ../varsb.show_parent_link}} @@ -27,7 +27,7 @@
    {{#if ../varss.file_content}} -
    + 
    {{ ../varss.file_content }}
    {{/if}} diff --git a/templates/style.scss b/templates/style.scss index 5ac0caf67..c03333ffa 100644 --- a/templates/style.scss +++ b/templates/style.scss @@ -54,11 +54,15 @@ strong { // rustdoc overrides div.rustdoc { font-family: $font-family-serif; - padding-right: 15px; + padding: 0 1em; + + @media #{$media-sm} { + padding-left: 0; + } } .sidebar { - top: 150px; + top: 160px; left: auto; } @@ -309,8 +313,12 @@ div.package-page-container { } div.package-details { - p { - font-family: $font-family-serif; + padding: 0 1em; + + font-family: $font-family-serif; + + h1, h2, h3, h4, h5, h6 { + font-family: $font-family-sans; } h1:first-child, From 16c447c39217446fb01175182c74b8a9a66bf85e Mon Sep 17 00:00:00 2001 From: Onur Aslan Date: Sun, 3 Jul 2016 13:34:23 +0300 Subject: [PATCH 10/79] Fix github issue count --- src/web/crate_details.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/web/crate_details.rs b/src/web/crate_details.rs index c685aa738..8efd47aa1 100644 --- a/src/web/crate_details.rs +++ b/src/web/crate_details.rs @@ -72,7 +72,7 @@ impl ToJson for CrateDetails { m.insert("github".to_string(), self.github.to_json()); m.insert("github_stars".to_string(), self.github_stars.to_json()); m.insert("github_forks".to_string(), self.github_forks.to_json()); - m.insert("github_issues".to_string(), self.github_forks.to_json()); + m.insert("github_issues".to_string(), self.github_issues.to_json()); m.insert("metadata".to_string(), self.metadata.to_json()); m.to_json() } From a8c15ac2e617c414a63275ccd7af3af023fae2b9 Mon Sep 17 00:00:00 2001 From: Onur Aslan Date: Sun, 10 Jul 2016 14:46:33 +0300 Subject: [PATCH 11/79] Add colors for build status icons --- templates/builds.hbs | 2 +- templates/style.scss | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/templates/builds.hbs b/templates/builds.hbs index a0f8ad64f..eca4df711 100644 --- a/templates/builds.hbs +++ b/templates/builds.hbs @@ -27,7 +27,7 @@ $ cratesfyi ...
  • -
    +
    {{rustc_version}}
    {{cratesfyi_version}}
    {{build_time}}
    diff --git a/templates/style.scss b/templates/style.scss index c03333ffa..6b1334a0f 100644 --- a/templates/style.scss +++ b/templates/style.scss @@ -168,6 +168,17 @@ div.recent-releases-container { font-weight: 500; } + .build { + font-weight: 500; + i.fa-check { + color: $color-macro; + } + i.fa-close { + color: $color-struct; + + } + } + .description { font-family: $font-family-serif; font-weight: normal; From e3581d9ab89838bcca91540925276d78edbc869f Mon Sep 17 00:00:00 2001 From: Onur Aslan Date: Thu, 21 Jul 2016 21:21:39 +0300 Subject: [PATCH 12/79] Activate I am feeling lucky button --- src/web/releases.rs | 26 +++++++++++++++++++++++++- templates/releases.hbs | 13 ++++++++++++- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/src/web/releases.rs b/src/web/releases.rs index 3d1a32449..b94f015fc 100644 --- a/src/web/releases.rs +++ b/src/web/releases.rs @@ -1,7 +1,7 @@ //! Releases web handlers -use super::{NoCrate, duration_to_str}; +use super::{NoCrate, duration_to_str, match_version}; use super::page::Page; use super::pool::Pool; use iron::prelude::*; @@ -357,6 +357,30 @@ pub fn search_handler(req: &mut Request) -> IronResult { let conn = req.extensions.get::().unwrap(); if let Some(&Value::String(ref query)) = query { + + // check if I am feeling lucky button pressed and redirect user to crate page + // if there is a match + // TODO: Redirecting to latest doc might be more useful + if params.find(&["i-am-feeling-lucky"]).is_some() { + if let Some(version) = match_version(&conn, &query, None) { + use iron::Url; + use iron::modifiers::Redirect; + let url = Url::parse(&format!("{}://{}:{}/crates/{}/{}", + req.url.scheme, + req.url.host, + req.url.port, + query, + version)[..]).unwrap(); + let mut resp = Response::with((status::Found, Redirect(url))); + + use iron::headers::{Expires, HttpDate}; + use time; + resp.headers.set(Expires(HttpDate(time::now()))); + return Ok(resp); + } + } + + let search_query = query.replace(" ", " & "); get_search_results(&conn, &search_query, 1, RELEASES_IN_RELEASES) .ok_or(IronError::new(NoCrate, status::NotFound)) diff --git a/templates/releases.hbs b/templates/releases.hbs index 72a4fe58a..fe4dccc23 100644 --- a/templates/releases.hbs +++ b/templates/releases.hbs @@ -8,9 +8,20 @@
    - +
    +
    {{/if}} From cfb957384d5c665613353cdd74349655a8152261 Mon Sep 17 00:00:00 2001 From: Onur Aslan Date: Thu, 21 Jul 2016 23:06:51 +0300 Subject: [PATCH 13/79] Add Rust submenu to navigation --- templates/navigation.hbs | 15 +++++++++++++++ templates/style.scss | 11 +++++++++++ 2 files changed, 26 insertions(+) diff --git a/templates/navigation.hbs b/templates/navigation.hbs index 2cd16953d..736301399 100644 --- a/templates/navigation.hbs +++ b/templates/navigation.hbs @@ -8,6 +8,21 @@     Cratesfyi diff --git a/templates/style.scss b/templates/style.scss index 6b1334a0f..14f19c292 100644 --- a/templates/style.scss +++ b/templates/style.scss @@ -435,3 +435,14 @@ div.search-page-search-form { } + +.menu-item-divided { + border-bottom: 1px solid $color-border; +} + +.rust-navigation-item { + background: url(/rust-logo.png) no-repeat; + background-position: 15px 45%; + background-size: 14px; + padding-left: 35px; +} From 0c05d5d6386f9d1208bbcf77d2aba1b22cf90f5e Mon Sep 17 00:00:00 2001 From: Onur Aslan Date: Thu, 21 Jul 2016 23:11:24 +0300 Subject: [PATCH 14/79] Set warning font-family to sans --- templates/style.scss | 1 + 1 file changed, 1 insertion(+) diff --git a/templates/style.scss b/templates/style.scss index 14f19c292..100012ba0 100644 --- a/templates/style.scss +++ b/templates/style.scss @@ -412,6 +412,7 @@ div.cratesfyi-package-container { div.warning { + font-family: $font-family-sans; border-radius: 4px; background-color: lighten($color-type, 45%); padding: .4em 1em; From 77c9e5aa72b737c0257e3d66476309a262a5d0ed Mon Sep 17 00:00:00 2001 From: Onur Aslan Date: Fri, 22 Jul 2016 10:33:54 +0300 Subject: [PATCH 15/79] Use pure-min.css in rustdoc --- templates/rustdoc.hbs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/templates/rustdoc.hbs b/templates/rustdoc.hbs index 79ad37470..684d9ba04 100644 --- a/templates/rustdoc.hbs +++ b/templates/rustdoc.hbs @@ -2,9 +2,8 @@ {{{content.rustdoc_head}}} - - - + + From c9c0da8b2c4d1d68399fe01160898f88ee9ea807 Mon Sep 17 00:00:00 2001 From: Onur Aslan Date: Fri, 22 Jul 2016 11:54:46 +0300 Subject: [PATCH 16/79] Revert "Use pure-min.css in rustdoc" This reverts commit 77c9e5aa72b737c0257e3d66476309a262a5d0ed. This commit caused some issues in rustdocs --- templates/rustdoc.hbs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/templates/rustdoc.hbs b/templates/rustdoc.hbs index 684d9ba04..79ad37470 100644 --- a/templates/rustdoc.hbs +++ b/templates/rustdoc.hbs @@ -2,8 +2,9 @@ {{{content.rustdoc_head}}} - - + + + From 444ff0cb43d915cf0c212a56812e51440cf42172 Mon Sep 17 00:00:00 2001 From: Onur Aslan Date: Fri, 22 Jul 2016 11:57:42 +0300 Subject: [PATCH 17/79] Use menus-min.css to get all purecss menu bundle --- templates/rustdoc.hbs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/templates/rustdoc.hbs b/templates/rustdoc.hbs index 79ad37470..77f8da1c0 100644 --- a/templates/rustdoc.hbs +++ b/templates/rustdoc.hbs @@ -2,9 +2,7 @@ {{{content.rustdoc_head}}} - - - + From 6b5b125cb8c44d5be1e24109cc39abb658c294d8 Mon Sep 17 00:00:00 2001 From: Onur Aslan Date: Fri, 22 Jul 2016 11:58:11 +0300 Subject: [PATCH 18/79] Serve static pages from CRATESFYI_PREFIX env var --- src/web/mod.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/web/mod.rs b/src/web/mod.rs index 12cf91066..3d8eaaa42 100644 --- a/src/web/mod.rs +++ b/src/web/mod.rs @@ -10,10 +10,10 @@ mod pool; mod file; mod builds; -use std::fmt; +use std::{env, fmt}; use std::error::Error; use std::time::Duration; -use std::path::Path; +use std::path::PathBuf; use iron::prelude::*; use iron::{status, Handler}; use router::Router; @@ -92,8 +92,8 @@ impl CratesfyiHandler { router_chain.link_before(pool::Pool::new()); router_chain.link_after(hbse); - // FIXME: URGENT use correct path - let static_handler = Static::new(Path::new("../cratesfyi-prefix/public_html")) + let prefix = PathBuf::from(env::var("CRATESFYI_PREFIX").unwrap()).join("public_html"); + let static_handler = Static::new(prefix) .cache(Duration::from_secs(STATIC_FILE_CACHE_DURATION)); CratesfyiHandler { From 559aa815ea48d93f1f06b8f2acd38aa66f7770ab Mon Sep 17 00:00:00 2001 From: Onur Aslan Date: Fri, 22 Jul 2016 12:39:23 +0300 Subject: [PATCH 19/79] Add link colors to crates readme --- templates/style.scss | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/templates/style.scss b/templates/style.scss index 100012ba0..4561d63be 100644 --- a/templates/style.scss +++ b/templates/style.scss @@ -328,6 +328,14 @@ div.package-page-container { font-family: $font-family-serif; + a { + color: $color-url; + } + + a:hover { + text-decoration: underline; + } + h1, h2, h3, h4, h5, h6 { font-family: $font-family-sans; } From 9412409f4d43c616d32aa8f48f9d9bd225632d79 Mon Sep 17 00:00:00 2001 From: Jon Gjengset Date: Fri, 22 Jul 2016 15:04:03 -0400 Subject: [PATCH 20/79] Add set_status to Page --- src/web/page.rs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/web/page.rs b/src/web/page.rs index d57b9e607..b5a4c39db 100644 --- a/src/web/page.rs +++ b/src/web/page.rs @@ -10,6 +10,7 @@ use handlebars_iron::Template; pub struct Page { title: Option, content: T, + status: status::Status, varss: BTreeMap, varsb: BTreeMap, varsi: BTreeMap, @@ -21,6 +22,7 @@ impl Page { Page { title: None, content: content, + status: status::Ok, varss: BTreeMap::new(), varsb: BTreeMap::new(), varsi: BTreeMap::new(), @@ -62,10 +64,18 @@ impl Page { } - pub fn to_resp(self, template: &str) -> IronResult { + /// Sets status code for response + pub fn set_status(mut self, s: status::Status) -> Page { + self.status = s; + self + } + + + pub fn to_resp(self, template: &str) -> IronResult { let mut resp = Response::new(); + let status = self.status; let temp = Template::new(template, self); - resp.set_mut(temp).set_mut(status::Ok); + resp.set_mut(temp).set_mut(status); Ok(resp) } } From ae38172d156b3398545f56987afc963822594452 Mon Sep 17 00:00:00 2001 From: Jon Gjengset Date: Fri, 22 Jul 2016 15:05:47 -0400 Subject: [PATCH 21/79] Add a less generic error type --- src/web/crate_details.rs | 5 ++-- src/web/error.rs | 64 ++++++++++++++++++++++++++++++++++++++++ src/web/file.rs | 2 +- src/web/mod.rs | 55 +++++++++++++--------------------- src/web/releases.rs | 17 ++++++----- src/web/rustdoc.rs | 9 +++--- templates/error.hbs | 2 ++ 7 files changed, 105 insertions(+), 49 deletions(-) create mode 100644 src/web/error.rs create mode 100644 templates/error.hbs diff --git a/src/web/crate_details.rs b/src/web/crate_details.rs index 8efd47aa1..7bdaed476 100644 --- a/src/web/crate_details.rs +++ b/src/web/crate_details.rs @@ -1,7 +1,8 @@ -use super::{NoCrate, MetaData, duration_to_str, match_version, render_markdown}; +use super::{MetaData, duration_to_str, match_version, render_markdown}; +use super::error::Nope; use super::page::Page; use db::connect_db; use iron::prelude::*; @@ -194,7 +195,7 @@ pub fn crate_details_handler(req: &mut Request) -> IronResult { match_version(&conn, &name, req_version) .and_then(|version| CrateDetails::new(&conn, &name, &version)) - .ok_or(IronError::new(NoCrate, status::NotFound)) + .ok_or(IronError::new(Nope::CrateNotFound, status::NotFound)) .and_then(|details| { Page::new(details) .set_true("show_package_navigation") diff --git a/src/web/error.rs b/src/web/error.rs new file mode 100644 index 000000000..341b31d8a --- /dev/null +++ b/src/web/error.rs @@ -0,0 +1,64 @@ +use std::error::Error; +use iron::prelude::*; +use iron::Handler; +use iron::status; +use web::page::Page; +use std::fmt; + +#[derive(Debug)] +pub enum Nope { + ResourceNotFound, + CrateNotFound, + NoResults, +} + +impl fmt::Display for Nope { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str(self.description()) + } +} + +impl Error for Nope { + fn description(&self) -> &str { + match *self { + Nope::ResourceNotFound => "Requested resource not found", + Nope::CrateNotFound => "Requested crate not found", + Nope::NoResults => "Search yielded no results", + } + } +} + +impl Handler for Nope { + fn handle(&self, req: &mut Request) -> IronResult { + match *self { + Nope::ResourceNotFound => { + // user tried to navigate to a resource (doc page/file) that doesn't exist + Page::new("no such resource".to_owned()) + .set_status(status::NotFound) + .title("The requested resource does not exist") + .to_resp("error") + } + Nope::CrateNotFound => { + // user tried to navigate to a crate that doesn't exist + Page::new("no such crate".to_owned()) + .set_status(status::NotFound) + .title("The requested crate does not exist") + .to_resp("error") + } + Nope::NoResults => { + use params::{Params, Value}; + let params = req.get::().unwrap(); + if let Some(&Value::String(ref query)) = params.find(&["query"]) { + // this used to be a search + Page::new(Vec::::new()) + .set_status(status::NotFound) + .set("search_query", &query) + .title(&format!("No crates found matching '{}'", query)) + .to_resp("releases") + } else { + unimplemented!(); + } + } + } + } +} diff --git a/src/web/file.rs b/src/web/file.rs index 2a47dbff0..9a81ef6a9 100644 --- a/src/web/file.rs +++ b/src/web/file.rs @@ -81,7 +81,7 @@ impl Handler for DatabaseFileHandler { if let Some(file) = File::from_path(&conn, &path) { Ok(file.serve()) } else { - Err(IronError::new(super::NoCrate, status::NotFound)) + Err(IronError::new(super::error::Nope::CrateNotFound, status::NotFound)) } } } diff --git a/src/web/mod.rs b/src/web/mod.rs index 3d8eaaa42..795cff47a 100644 --- a/src/web/mod.rs +++ b/src/web/mod.rs @@ -9,13 +9,14 @@ mod source; mod pool; mod file; mod builds; +mod error; -use std::{env, fmt}; +use std::env; use std::error::Error; use std::time::Duration; use std::path::PathBuf; use iron::prelude::*; -use iron::{status, Handler}; +use iron::Handler; use router::Router; use staticfile::Static; use handlebars_iron::{HandlebarsEngine, DirectorySource}; @@ -31,22 +32,6 @@ use std::collections::BTreeMap; const STATIC_FILE_CACHE_DURATION: u64 = 60 * 60 * 24 * 3; // 3 days -// This is a generic error used in routes -#[derive(Debug)] -struct NoCrate; - -impl fmt::Display for NoCrate { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - f.write_str("Crate not found.") - } -} - -impl Error for NoCrate { - fn description(&self) -> &str { "No Crate" } -} - - - struct CratesfyiHandler { router_handler: Box, database_file_handler: Box, @@ -107,23 +92,25 @@ impl CratesfyiHandler { impl Handler for CratesfyiHandler { fn handle(&self, req: &mut Request) -> IronResult { - // try router first then staticfile handler + // try router first then db/static file handler // return 404 if none of them return Ok - match self.router_handler.handle(req) { - Ok(res) => return Ok(res), - Err(e) => debug!("{}", e.description()) - }; - - // if router fails try to serve files from database first - // and then try static handler. if all of them fails, return 404 - // TODO: Add a custom 404 page - if let Ok(res) = self.database_file_handler.handle(req) { - Ok(res) - } else if let Ok(res) = self.static_handler.handle(req) { - Ok(res) - } else { - Ok(Response::with((status::NotFound, "404 FAILED IN CRATESFYIHANDLER"))) - } + self.router_handler + .handle(req) + .or_else(|e| { + // if router fails try to serve files from database first + self.database_file_handler.handle(req).or(Err(e)) + }) + .or_else(|e| { + // and then try static handler. if all of them fails, return 404 + self.static_handler.handle(req).or(Err(e)) + }) + .or_else(|e| { + debug!("{}", e.description()); + let err: &error::Nope = e.error + .downcast::() + .expect("all cratesfyi errors should be of type Nope"); + err.handle(req) + }) } } diff --git a/src/web/releases.rs b/src/web/releases.rs index b94f015fc..0ec912268 100644 --- a/src/web/releases.rs +++ b/src/web/releases.rs @@ -1,7 +1,8 @@ //! Releases web handlers -use super::{NoCrate, duration_to_str, match_version}; +use super::{duration_to_str, match_version}; +use super::error::Nope; use super::page::Page; use super::pool::Pool; use iron::prelude::*; @@ -19,7 +20,7 @@ const RELEASES_IN_HOME: i64 = 15; const RELEASES_IN_RELEASES: i64 = 30; -struct Release { +pub struct Release { name: String, version: String, description: Option, @@ -243,7 +244,7 @@ pub fn releases_handler(req: &mut Request) -> IronResult { let packages = get_releases(conn, page_number, RELEASES_IN_RELEASES, Order::ReleaseTime); if packages.is_empty() { - return Err(IronError::new(NoCrate, status::NotFound)); + return Err(IronError::new(Nope::CrateNotFound, status::NotFound)); } // Show next and previous page buttons @@ -280,7 +281,7 @@ pub fn stars_handler(req: &mut Request) -> IronResult { let packages = get_releases(conn, page_number, RELEASES_IN_RELEASES, Order::GithubStars); if packages.is_empty() { - return Err(IronError::new(NoCrate, status::NotFound)); + return Err(IronError::new(Nope::CrateNotFound, status::NotFound)); } // Show next and previous page buttons @@ -317,7 +318,7 @@ pub fn author_handler(req: &mut Request) -> IronResult { let author = req.extensions.get::().unwrap().find("author"); if author.is_none() { - return Err(IronError::new(NoCrate, status::NotFound)); + return Err(IronError::new(Nope::CrateNotFound, status::NotFound)); } let (author_name, packages) = get_releases_by_author(conn, @@ -326,7 +327,7 @@ pub fn author_handler(req: &mut Request) -> IronResult { author.unwrap()); if packages.is_empty() { - return Err(IronError::new(NoCrate, status::NotFound)); + return Err(IronError::new(Nope::CrateNotFound, status::NotFound)); } // Show next and previous page buttons @@ -383,7 +384,7 @@ pub fn search_handler(req: &mut Request) -> IronResult { let search_query = query.replace(" ", " & "); get_search_results(&conn, &search_query, 1, RELEASES_IN_RELEASES) - .ok_or(IronError::new(NoCrate, status::NotFound)) + .ok_or(IronError::new(Nope::NoResults, status::NotFound)) .and_then(|(_, results)| { // FIXME: There is no pagination Page::new(results) @@ -392,6 +393,6 @@ pub fn search_handler(req: &mut Request) -> IronResult { .to_resp("releases") }) } else { - Err(IronError::new(NoCrate, status::NotFound)) + Err(IronError::new(Nope::NoResults, status::NotFound)) } } diff --git a/src/web/rustdoc.rs b/src/web/rustdoc.rs index b195c04b6..ccc784cb8 100644 --- a/src/web/rustdoc.rs +++ b/src/web/rustdoc.rs @@ -8,7 +8,8 @@ use iron::prelude::*; use iron::{status, Url}; use iron::modifiers::Redirect; use router::Router; -use super::{NoCrate, match_version}; +use super::match_version; +use super::error::Nope; use super::page::Page; use rustc_serialize::json::{Json, ToJson}; use std::collections::BTreeMap; @@ -88,7 +89,7 @@ pub fn rustdoc_redirector_handler(req: &mut Request) -> IronResult { let version = match match_version(&conn, &crate_name, req_version) { Some(v) => v, - None => return Err(IronError::new(NoCrate, status::NotFound)), + None => return Err(IronError::new(Nope::CrateNotFound, status::NotFound)), }; // get target name @@ -119,14 +120,14 @@ pub fn rustdoc_html_server_handler(req: &mut Request) -> IronResult { // don't touch anything other than html files if !path.ends_with(".html") { - return Err(IronError::new(NoCrate, status::NotFound)); + return Err(IronError::new(Nope::ResourceNotFound, status::NotFound)); } let conn = req.extensions.get::().unwrap(); let file = match File::from_path(&conn, &path) { Some(f) => f, - None => return Err(IronError::new(NoCrate, status::NotFound)), + None => return Err(IronError::new(Nope::ResourceNotFound, status::NotFound)), }; let (mut in_head, mut in_body) = (false, false); diff --git a/templates/error.hbs b/templates/error.hbs new file mode 100644 index 000000000..bf2fd7d93 --- /dev/null +++ b/templates/error.hbs @@ -0,0 +1,2 @@ +{{> header}} +{{> footer}} From 7ec10ad96fbb18942edf1fc25734376a1abc0272 Mon Sep 17 00:00:00 2001 From: Jon Gjengset Date: Fri, 22 Jul 2016 15:06:03 -0400 Subject: [PATCH 22/79] Give error rendering access to templates --- src/web/error.rs | 2 +- src/web/mod.rs | 34 +++++++++++++++++++--------------- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/src/web/error.rs b/src/web/error.rs index 341b31d8a..791c66fd9 100644 --- a/src/web/error.rs +++ b/src/web/error.rs @@ -5,7 +5,7 @@ use iron::status; use web::page::Page; use std::fmt; -#[derive(Debug)] +#[derive(Debug, Copy, Clone)] pub enum Nope { ResourceNotFound, CrateNotFound, diff --git a/src/web/mod.rs b/src/web/mod.rs index 795cff47a..1a599d343 100644 --- a/src/web/mod.rs +++ b/src/web/mod.rs @@ -40,6 +40,22 @@ struct CratesfyiHandler { impl CratesfyiHandler { + fn chain(base: H) -> Chain { + // TODO: Use DocBuilderOptions for paths + let mut hbse = HandlebarsEngine::new(); + hbse.add(Box::new(DirectorySource::new("./templates", ".hbs"))); + + // load templates + if let Err(e) = hbse.reload() { + panic!("Failed to load handlebar templates: {}", e.description()); + } + + let mut chain = Chain::new(base); + chain.link_before(pool::Pool::new()); + chain.link_after(hbse); + chain + } + pub fn new() -> CratesfyiHandler { let mut router = Router::new(); router.get("/", releases::home_page); @@ -64,19 +80,7 @@ impl CratesfyiHandler { router.get("/source/:name/:version/*", source::source_browser_handler); router.get("/search", releases::search_handler); - // TODO: Use DocBuilderOptions for paths - let mut hbse = HandlebarsEngine::new(); - hbse.add(Box::new(DirectorySource::new("./templates", ".hbs"))); - - // load templates - if let Err(e) = hbse.reload() { - panic!("Failed to load handlebar templates: {}", e.description()); - } - - let mut router_chain = Chain::new(router); - router_chain.link_before(pool::Pool::new()); - router_chain.link_after(hbse); - + let router_chain = Self::chain(router); let prefix = PathBuf::from(env::var("CRATESFYI_PREFIX").unwrap()).join("public_html"); let static_handler = Static::new(prefix) .cache(Duration::from_secs(STATIC_FILE_CACHE_DURATION)); @@ -106,10 +110,10 @@ impl Handler for CratesfyiHandler { }) .or_else(|e| { debug!("{}", e.description()); - let err: &error::Nope = e.error + let err: error::Nope = *e.error .downcast::() .expect("all cratesfyi errors should be of type Nope"); - err.handle(req) + Self::chain(err).handle(req) }) } } From 76717143b21877fe08982f3987ec0031e55d1cf2 Mon Sep 17 00:00:00 2001 From: Jon Gjengset Date: Fri, 22 Jul 2016 15:06:34 -0400 Subject: [PATCH 23/79] Give more consistent rendering when content is missing --- templates/navigation.hbs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/templates/navigation.hbs b/templates/navigation.hbs index 736301399..937f2dfb3 100644 --- a/templates/navigation.hbs +++ b/templates/navigation.hbs @@ -30,13 +30,13 @@ {{#unless varsb.hide_package_navigation}} - {{#with content}}
    -

    {{#if ../title}}{{../title}}{{else}}{{metadata.name}} {{metadata.version}}{{/if}}

    -
    {{#if metadata.description }}{{metadata.description}}{{else}}{{../varss.description}}{{/if}}
    +

    {{#if title}}{{title}}{{else}}{{content.metadata.name}} {{content.metadata.version}}{{/if}}

    +
    {{#if content.metadata.description }}{{content.metadata.description}}{{else}}{{varss.description}}{{/if}}
    + {{#if ../varsb.show_package_navigation}} - {{#with metadata}} + {{#with content.metadata}}
      {{#if rustdoc_status}} @@ -49,18 +49,18 @@
    {{/with}} {{/if}} - {{#if ../varsb.show_releases_navigation}} + + {{#if varsb.show_releases_navigation}}
    {{/if}}
    - {{/with}} {{/unless}} From 92061edf7237ea1eb77e836fc3d8c5eec863b9a9 Mon Sep 17 00:00:00 2001 From: Jon Gjengset Date: Fri, 22 Jul 2016 15:08:48 -0400 Subject: [PATCH 24/79] Handle searches with no query --- src/web/error.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/web/error.rs b/src/web/error.rs index 791c66fd9..f7ed1857b 100644 --- a/src/web/error.rs +++ b/src/web/error.rs @@ -56,7 +56,11 @@ impl Handler for Nope { .title(&format!("No crates found matching '{}'", query)) .to_resp("releases") } else { - unimplemented!(); + // user did a search with no search terms + Page::new(Vec::::new()) + .set_status(status::NotFound) + .title("No results given for empty search query") + .to_resp("releases") } } } From 4a0e000bf7326baf595af5f0da77665ea0d5d524 Mon Sep 17 00:00:00 2001 From: Onur Aslan Date: Mon, 25 Jul 2016 23:46:11 +0300 Subject: [PATCH 25/79] Fix query --- src/db/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/db/mod.rs b/src/db/mod.rs index d9c646d2d..7470179db 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -84,7 +84,7 @@ pub fn create_tables(conn: &Connection) -> Result<(), Error> { github_issues INT DEFAULT 0, \ github_last_commit TIMESTAMP, \ github_last_update TIMESTAMP, \ - content tsvector, \ + content tsvector \ )", "CREATE TABLE releases ( \ id SERIAL PRIMARY KEY, \ From 15d746c95337f5c340920c33276a088b57eab281 Mon Sep 17 00:00:00 2001 From: Onur Aslan Date: Tue, 26 Jul 2016 01:39:37 +0300 Subject: [PATCH 26/79] Bring start-web-server subcommand back And add socket address parameter --- src/bin/cratesfyi.rs | 10 ++++++++-- src/web/mod.rs | 6 +++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/bin/cratesfyi.rs b/src/bin/cratesfyi.rs index 44fd4c883..e3e187d97 100644 --- a/src/bin/cratesfyi.rs +++ b/src/bin/cratesfyi.rs @@ -89,6 +89,12 @@ pub fn main() { .index(2) .required(true) .help("Version of crate")))) + .subcommand(SubCommand::with_name("start-web-server") + .about("Starts web server") + .arg(Arg::with_name("SOCKET_ADDR") + .index(1) + .required(false) + .help("Socket address to listen to"))) .subcommand(SubCommand::with_name("database") .about("Database operations") .subcommand(SubCommand::with_name("init") @@ -185,8 +191,8 @@ pub fn main() { matches.value_of("DIRECTORY").unwrap()) .unwrap(); } - } else if let Some(_) = matches.subcommand_matches("start-web-server") { - start_web_server(); + } else if let Some(matches) = matches.subcommand_matches("start-web-server") { + start_web_server(matches.value_of("SOCKET_ADDR")); } else { println!("{}", matches.usage()); } diff --git a/src/web/mod.rs b/src/web/mod.rs index 1a599d343..e39e59212 100644 --- a/src/web/mod.rs +++ b/src/web/mod.rs @@ -205,9 +205,9 @@ fn render_markdown(text: &str) -> String { /// Starts cratesfyi web server -pub fn start_web_server() { +pub fn start_web_server(sock_addr: Option<&str>) { let cratesfyi = CratesfyiHandler::new(); - Iron::new(cratesfyi).http("localhost:3000").unwrap(); + Iron::new(cratesfyi).http(sock_addr.unwrap_or("localhost:3000")).unwrap(); } @@ -302,6 +302,6 @@ mod test { fn test_start_web_server() { // FIXME: This test is doing nothing let _ = env_logger::init(); - start_web_server(); + start_web_server(None); } } From 0916258463b468d063683ce8b7286dd792ed1bb6 Mon Sep 17 00:00:00 2001 From: Matthew Hall Date: Thu, 28 Jul 2016 22:21:08 +0100 Subject: [PATCH 27/79] Don't 500 on a route with no match Previously a bad route (eg. /crate/regex) would be a 500 because the router returned it's own error, not a Nope. --- src/web/mod.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/web/mod.rs b/src/web/mod.rs index e39e59212..db9025716 100644 --- a/src/web/mod.rs +++ b/src/web/mod.rs @@ -17,7 +17,7 @@ use std::time::Duration; use std::path::PathBuf; use iron::prelude::*; use iron::Handler; -use router::Router; +use router::{Router, NoRoute}; use staticfile::Static; use handlebars_iron::{HandlebarsEngine, DirectorySource}; use time; @@ -110,9 +110,13 @@ impl Handler for CratesfyiHandler { }) .or_else(|e| { debug!("{}", e.description()); - let err: error::Nope = *e.error - .downcast::() - .expect("all cratesfyi errors should be of type Nope"); + let err = if let Some(err) = e.error.downcast::() { + *err + } else if e.error.downcast::().is_some() { + error::Nope::ResourceNotFound + } else { + panic!("all cratesfyi errors should be of type Nope"); + }; Self::chain(err).handle(req) }) } From be6c7d6c81e41897c89df7c8db9a490f212a7a47 Mon Sep 17 00:00:00 2001 From: Onur Aslan Date: Sat, 23 Jul 2016 11:04:25 +0300 Subject: [PATCH 28/79] Add release activity chart --- src/utils/mod.rs | 2 + src/utils/release_activity_updater.rs | 60 ++++++++++++++++++++++ src/web/mod.rs | 1 + src/web/releases.rs | 71 +++++++++++++++++---------- templates/header.hbs | 3 ++ templates/navigation.hbs | 1 + templates/releases_activity.hbs | 38 ++++++++++++++ 7 files changed, 149 insertions(+), 27 deletions(-) create mode 100644 src/utils/release_activity_updater.rs create mode 100644 templates/releases_activity.hbs diff --git a/src/utils/mod.rs b/src/utils/mod.rs index 34c854be2..cc9a80a78 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -4,7 +4,9 @@ pub use self::build_doc::{build_doc, get_package, source_path, update_sources}; pub use self::copy::{copy_dir, copy_doc_dir}; pub use self::github_updater::github_updater; +pub use self::release_activity_updater::update_release_activity; mod github_updater; mod build_doc; mod copy; +mod release_activity_updater; diff --git a/src/utils/release_activity_updater.rs b/src/utils/release_activity_updater.rs new file mode 100644 index 000000000..9c3885c69 --- /dev/null +++ b/src/utils/release_activity_updater.rs @@ -0,0 +1,60 @@ + +use db::connect_db; +use DocBuilderError; +use time::{now, Duration}; +use std::collections::BTreeMap; +use rustc_serialize::json::ToJson; + + +pub fn update_release_activity() -> Result<(), DocBuilderError> { + + let conn = try!(connect_db()); + let mut dates = Vec::new(); + let mut crate_counts = Vec::new(); + + for day in 1..31 { + let rows = try!(conn.query(&format!("SELECT COUNT(*) + FROM releases + WHERE release_time < NOW() - INTERVAL '{} day' AND + release_time > NOW() - INTERVAL '{} day'", + day, + day + 1), + &[])); + let release_count: i64 = rows.get(0).get(0); + let now = now(); + let date = now - Duration::days(day); + dates.push(format!("{}", date.strftime("%d %b").unwrap())); + // unwrap is fine here, ~~~~~~~~~~~~^ our date format is always valid + crate_counts.push(release_count); + } + + let map = { + let mut map = BTreeMap::new(); + map.insert("dates".to_owned(), dates.to_json()); + map.insert("counts".to_owned(), crate_counts.to_json()); + map.to_json() + }; + + try!(conn.query("INSERT INTO config (name, value) VALUES ('release_activity', $1)", + &[&map]) + .or_else(|_| { + conn.query("UPDATE config SET value = $1 WHERE name = 'release_activity'", + &[&map]) + })); + + Ok(()) +} + + +#[cfg(test)] +mod test { + extern crate env_logger; + use super::update_release_activity; + + #[test] + #[ignore] + fn test_update_release_activity() { + let _ = env_logger::init(); + assert!(update_release_activity().is_ok()); + } +} diff --git a/src/web/mod.rs b/src/web/mod.rs index db9025716..65eb87993 100644 --- a/src/web/mod.rs +++ b/src/web/mod.rs @@ -65,6 +65,7 @@ impl CratesfyiHandler { router.get("/releases/stars/:page", releases::stars_handler); router.get("/releases/:author", releases::author_handler); router.get("/releases/:author/:page", releases::author_handler); + router.get("/releases/activity", releases::activity_handler); router.get("/rustdoc/:crate", rustdoc::rustdoc_redirector_handler); router.get("/rustdoc/:crate/", rustdoc::rustdoc_redirector_handler); router.get("/rustdoc/:crate/:version", rustdoc::rustdoc_redirector_handler); diff --git a/src/web/releases.rs b/src/web/releases.rs index 0ec912268..c7a6fd29e 100644 --- a/src/web/releases.rs +++ b/src/web/releases.rs @@ -190,8 +190,8 @@ fn get_search_results(conn: &Connection, WHERE crates.content @@ to_tsquery($1) \ ORDER BY rank DESC \ LIMIT $2 OFFSET $3", - &[&query, &limit, &offset]) - .unwrap() { + &[&query, &limit, &offset]) + .unwrap() { let package = Release { name: row.get(0), @@ -209,8 +209,8 @@ fn get_search_results(conn: &Connection, if !packages.is_empty() { // get count of total results let rows = conn.query("SELECT COUNT(*) FROM crates WHERE content @@ to_tsquery($1)", - &[&query]) - .unwrap(); + &[&query]) + .unwrap(); Some((rows.get(0).get(0), packages)) } else { @@ -233,12 +233,12 @@ pub fn home_page(req: &mut Request) -> IronResult { pub fn releases_handler(req: &mut Request) -> IronResult { // page number of releases let page_number: i64 = req.extensions - .get::() - .unwrap() - .find("page") - .unwrap_or("1") - .parse() - .unwrap_or(1); + .get::() + .unwrap() + .find("page") + .unwrap_or("1") + .parse() + .unwrap_or(1); let conn = req.extensions.get::().unwrap(); let packages = get_releases(conn, page_number, RELEASES_IN_RELEASES, Order::ReleaseTime); @@ -270,12 +270,12 @@ pub fn releases_handler(req: &mut Request) -> IronResult { pub fn stars_handler(req: &mut Request) -> IronResult { // page number of releases let page_number: i64 = req.extensions - .get::() - .unwrap() - .find("page") - .unwrap_or("1") - .parse() - .unwrap_or(1); + .get::() + .unwrap() + .find("page") + .unwrap_or("1") + .parse() + .unwrap_or(1); let conn = req.extensions.get::().unwrap(); let packages = get_releases(conn, page_number, RELEASES_IN_RELEASES, Order::GithubStars); @@ -307,12 +307,12 @@ pub fn stars_handler(req: &mut Request) -> IronResult { pub fn author_handler(req: &mut Request) -> IronResult { // page number of releases let page_number: i64 = req.extensions - .get::() - .unwrap() - .find("page") - .unwrap_or("1") - .parse() - .unwrap_or(1); + .get::() + .unwrap() + .find("page") + .unwrap_or("1") + .parse() + .unwrap_or(1); let conn = req.extensions.get::().unwrap(); let author = req.extensions.get::().unwrap().find("author"); @@ -321,10 +321,8 @@ pub fn author_handler(req: &mut Request) -> IronResult { return Err(IronError::new(Nope::CrateNotFound, status::NotFound)); } - let (author_name, packages) = get_releases_by_author(conn, - page_number, - RELEASES_IN_RELEASES, - author.unwrap()); + let (author_name, packages) = + get_releases_by_author(conn, page_number, RELEASES_IN_RELEASES, author.unwrap()); if packages.is_empty() { return Err(IronError::new(Nope::CrateNotFound, status::NotFound)); @@ -371,7 +369,8 @@ pub fn search_handler(req: &mut Request) -> IronResult { req.url.host, req.url.port, query, - version)[..]).unwrap(); + version)[..]) + .unwrap(); let mut resp = Response::with((status::Found, Redirect(url))); use iron::headers::{Expires, HttpDate}; @@ -396,3 +395,21 @@ pub fn search_handler(req: &mut Request) -> IronResult { Err(IronError::new(Nope::NoResults, status::NotFound)) } } + + +pub fn activity_handler(req: &mut Request) -> IronResult { + let conn = req.extensions.get::().unwrap(); + let release_activity_data: Json = + conn.query("SELECT value FROM config WHERE name = 'release_activity'", + &[]) + .unwrap() + .get(0) + .get(0); + Page::new(release_activity_data) + .title("Releases") + .set("description", "Monthly release activity") + .set_true("show_releases_navigation") + .set_true("releases_navigation_activity_tab") + .set_true("javascript_highchartjs") + .to_resp("releases_activity") +} diff --git a/templates/header.hbs b/templates/header.hbs index bcec57fa4..5cbc61e56 100644 --- a/templates/header.hbs +++ b/templates/header.hbs @@ -15,6 +15,9 @@ {{/if}} + {{#if varsb.javascript_highchartjs}} + + {{/if}} {{#if title}}{{title}} - {{/if}}{{#if content.metadata.name}}{{content.metadata.name}} {{content.metadata.version}} - {{/if}}Cratesfyi diff --git a/templates/navigation.hbs b/templates/navigation.hbs index 937f2dfb3..c517aa564 100644 --- a/templates/navigation.hbs +++ b/templates/navigation.hbs @@ -55,6 +55,7 @@
    • Recent
    • Stars
      • +
    • Activity
      • {{#if varss.author}}
    • {{varss.author}}
      • {{/if}} diff --git a/templates/releases_activity.hbs b/templates/releases_activity.hbs new file mode 100644 index 000000000..1137aa989 --- /dev/null +++ b/templates/releases_activity.hbs @@ -0,0 +1,38 @@ +{{> header}} +{{#with content}} + +
      +
      +
      + + +{{/with}} +{{> footer}} From c37a187743327d9dd1a7c8cee70a8035e22e81e5 Mon Sep 17 00:00:00 2001 From: Onur Aslan Date: Sat, 30 Jul 2016 17:02:04 +0300 Subject: [PATCH 29/79] Reverse release activity vectors --- src/utils/release_activity_updater.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/utils/release_activity_updater.rs b/src/utils/release_activity_updater.rs index 9c3885c69..64b26c8ae 100644 --- a/src/utils/release_activity_updater.rs +++ b/src/utils/release_activity_updater.rs @@ -28,6 +28,9 @@ pub fn update_release_activity() -> Result<(), DocBuilderError> { crate_counts.push(release_count); } + dates.reverse(); + crate_counts.reverse(); + let map = { let mut map = BTreeMap::new(); map.insert("dates".to_owned(), dates.to_json()); From 2b97d6b6d139acc3d3747008d12e0bbef259c7d1 Mon Sep 17 00:00:00 2001 From: Onur Aslan Date: Sat, 30 Jul 2016 19:01:47 +0300 Subject: [PATCH 30/79] Update routes | Old route | New route | Desc | |---------------------|---------------------------|----------------| | /rustdoc/{crate}... | /{crate}... | Documentations | | /search | /releases/search | Search results | | /crates/{crate} | /crate/{crate} | Crate info | | /source/{crate} | /crate/{crate}/{v}/source | Source browser | --- src/web/file.rs | 8 +++++++- src/web/mod.rs | 30 ++++++++++++++++-------------- src/web/releases.rs | 2 +- src/web/rustdoc.rs | 7 +++++-- src/web/source.rs | 8 +++++--- templates/builds.hbs | 2 +- templates/crate_details.hbs | 6 +++--- templates/header.hbs | 4 ++-- templates/navigation.hbs | 10 +++++----- templates/releases.hbs | 4 ++-- 10 files changed, 47 insertions(+), 34 deletions(-) diff --git a/src/web/file.rs b/src/web/file.rs index 9a81ef6a9..a67a3ce3a 100644 --- a/src/web/file.rs +++ b/src/web/file.rs @@ -74,7 +74,13 @@ impl Handler for DatabaseFileHandler { if path.ends_with("/") { path.push_str("index.html"); } - path + // rustdoc javascripts have rustdoc prefix in database + // FIXME: this is kinda lame I have to save all javascripts with rustdoc prefix + if path.ends_with(".js") { + format!("rustdoc/{}", path) + } else { + path + } }; let conn = req.extensions.get::().unwrap(); diff --git a/src/web/mod.rs b/src/web/mod.rs index 65eb87993..725719fe8 100644 --- a/src/web/mod.rs +++ b/src/web/mod.rs @@ -66,20 +66,22 @@ impl CratesfyiHandler { router.get("/releases/:author", releases::author_handler); router.get("/releases/:author/:page", releases::author_handler); router.get("/releases/activity", releases::activity_handler); - router.get("/rustdoc/:crate", rustdoc::rustdoc_redirector_handler); - router.get("/rustdoc/:crate/", rustdoc::rustdoc_redirector_handler); - router.get("/rustdoc/:crate/:version", rustdoc::rustdoc_redirector_handler); - router.get("/rustdoc/:crate/:version/", rustdoc::rustdoc_redirector_handler); - router.get("/rustdoc/:crate/:version/*", rustdoc::rustdoc_html_server_handler); - router.get("/crates/:name", crate_details::crate_details_handler); - router.get("/crates/:name/", crate_details::crate_details_handler); - router.get("/crates/:name/:version", crate_details::crate_details_handler); - router.get("/crates/:name/:version/", crate_details::crate_details_handler); - router.get("/crates/:name/:version/builds", builds::build_list_handler); - router.get("/crates/:name/:version/builds/:id", builds::build_list_handler); - router.get("/source/:name/:version/", source::source_browser_handler); - router.get("/source/:name/:version/*", source::source_browser_handler); - router.get("/search", releases::search_handler); + router.get("/releases/search", releases::search_handler); + router.get("/crate/:name", crate_details::crate_details_handler); + router.get("/crate/:name/", crate_details::crate_details_handler); + router.get("/crate/:name/:version", crate_details::crate_details_handler); + router.get("/crate/:name/:version/", crate_details::crate_details_handler); + router.get("/crate/:name/:version/builds", builds::build_list_handler); + router.get("/crate/:name/:version/builds/:id", builds::build_list_handler); + router.get("/crate/:name/:version/source/", source::source_browser_handler); + router.get("/crate/:name/:version/source/*", source::source_browser_handler); + router.get("/:crate", rustdoc::rustdoc_redirector_handler); + router.get("/:crate/", rustdoc::rustdoc_redirector_handler); + router.get("/:crate/:version", rustdoc::rustdoc_redirector_handler); + router.get("/:crate/:version/search-index.js", rustdoc::rustdoc_html_server_handler); + router.get("/:crate/:version/:target", rustdoc::rustdoc_redirector_handler); + router.get("/:crate/:version/:target/", rustdoc::rustdoc_html_server_handler); + router.get("/:crate/:version/:target/*.html", rustdoc::rustdoc_html_server_handler); let router_chain = Self::chain(router); let prefix = PathBuf::from(env::var("CRATESFYI_PREFIX").unwrap()).join("public_html"); diff --git a/src/web/releases.rs b/src/web/releases.rs index c7a6fd29e..1842bcad0 100644 --- a/src/web/releases.rs +++ b/src/web/releases.rs @@ -364,7 +364,7 @@ pub fn search_handler(req: &mut Request) -> IronResult { if let Some(version) = match_version(&conn, &query, None) { use iron::Url; use iron::modifiers::Redirect; - let url = Url::parse(&format!("{}://{}:{}/crates/{}/{}", + let url = Url::parse(&format!("{}://{}:{}/crate/{}/{}", req.url.scheme, req.url.host, req.url.port, diff --git a/src/web/rustdoc.rs b/src/web/rustdoc.rs index ccc784cb8..8c0e26ab8 100644 --- a/src/web/rustdoc.rs +++ b/src/web/rustdoc.rs @@ -64,7 +64,7 @@ pub fn rustdoc_redirector_handler(req: &mut Request) -> IronResult { vers: &str, target_name: &str) -> IronResult { - let url = Url::parse(&format!("{}://{}:{}/rustdoc/{}/{}/{}/", + let url = Url::parse(&format!("{}://{}:{}/{}/{}/{}/", req.url.scheme, req.url.host, req.url.port, @@ -109,8 +109,11 @@ pub fn rustdoc_redirector_handler(req: &mut Request) -> IronResult { pub fn rustdoc_html_server_handler(req: &mut Request) -> IronResult { let path = { - let mut path = req.url.path.clone().join("/"); + let mut path = req.url.path.clone(); + // documentations have "rustdoc" prefix in database + path.insert(0, "rustdoc".to_owned()); + let mut path = path.join("/"); if path.ends_with("/") { path.push_str("index.html") } diff --git a/src/web/source.rs b/src/web/source.rs index f5470b918..a0ca99b20 100644 --- a/src/web/source.rs +++ b/src/web/source.rs @@ -187,9 +187,11 @@ pub fn source_browser_handler(req: &mut Request) -> IronResult { // get path (req_path) for FileList::from_path and actual path for super::file::File::from_path let (req_path, file_path) = { let mut req_path = req.url.path.clone(); - // remove first element from path which is /source - req_path.remove(0); - let file_path = format!("sources/{}", req_path.join("/")); + // remove first elements from path which is /crate/:name/:version/source + for _ in 0..4 { + req_path.remove(0); + } + let file_path = format!("sources/{}/{}/{}", name, version, req_path.join("/")); // FileList::from_path is only working for directories // remove file name if it's not a directory diff --git a/templates/builds.hbs b/templates/builds.hbs index eca4df711..9282070c8 100644 --- a/templates/builds.hbs +++ b/templates/builds.hbs @@ -25,7 +25,7 @@ $ cratesfyi ...
        {{#each builds}}
      • - +
        {{rustc_version}}
        diff --git a/templates/crate_details.hbs b/templates/crate_details.hbs index d8981c614..39406ab05 100644 --- a/templates/crate_details.hbs +++ b/templates/crate_details.hbs @@ -25,7 +25,7 @@
        @@ -35,7 +35,7 @@
        @@ -45,7 +45,7 @@
        {{#unless build_status}} -
        cratesfyi failed to build {{name}}-{{version}}
        Please check build logs and if you believe this is cratesfyi's fault, report into this issue report.
        +
        cratesfyi failed to build {{name}}-{{version}}
        Please check build logs and if you believe this is cratesfyi's fault, report into this issue report.
        {{else}} {{#unless rustdoc_status}}
        {{name}}-{{version}} doesn't have any documentation.
        diff --git a/templates/header.hbs b/templates/header.hbs index 5cbc61e56..673076cde 100644 --- a/templates/header.hbs +++ b/templates/header.hbs @@ -7,8 +7,8 @@ - - + + {{#if varsb.javascript_highlightjs}} diff --git a/templates/navigation.hbs b/templates/navigation.hbs index c517aa564..b7ac77016 100644 --- a/templates/navigation.hbs +++ b/templates/navigation.hbs @@ -1,7 +1,7 @@
        -
        + {{#unless varsb.show_search_form}} {{/unless}} @@ -40,11 +40,11 @@
        {{/with}} diff --git a/templates/releases.hbs b/templates/releases.hbs index fe4dccc23..9945a5e20 100644 --- a/templates/releases.hbs +++ b/templates/releases.hbs @@ -4,7 +4,7 @@

        Cratesfyi

        - +
        @@ -36,7 +36,7 @@
        + +{{#if varsb.show_search_form}} +
        + +
        +{{/if}} + {{> footer}} diff --git a/templates/style.scss b/templates/style.scss index 4561d63be..d560ae907 100644 --- a/templates/style.scss +++ b/templates/style.scss @@ -455,3 +455,25 @@ div.search-page-search-form { background-size: 14px; padding-left: 35px; } + + +footer { + margin-top: 60px; + margin-bottom: 10px; + text-align: center; + ul { + list-style-type: none; + li { + list-style-type: none; + margin-bottom: 10px; + + a { + font-weight: 500; + color: $color-url; + } + } + li.hosting { + font-size: .8em; + } + } +} From 1f3c0df5f5b37557919d954084cccd301ebb11b6 Mon Sep 17 00:00:00 2001 From: Onur Aslan Date: Sun, 31 Jul 2016 11:02:11 +0300 Subject: [PATCH 32/79] Show another warning if crate is not a library --- src/web/crate_details.rs | 6 +++++- templates/crate_details.hbs | 4 ++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/web/crate_details.rs b/src/web/crate_details.rs index 7bdaed476..2cc94f83e 100644 --- a/src/web/crate_details.rs +++ b/src/web/crate_details.rs @@ -42,6 +42,7 @@ struct CrateDetails { github_forks: Option, github_issues: Option, metadata: MetaData, + is_library: bool, } @@ -75,6 +76,7 @@ impl ToJson for CrateDetails { m.insert("github_forks".to_string(), self.github_forks.to_json()); m.insert("github_issues".to_string(), self.github_issues.to_json()); m.insert("metadata".to_string(), self.metadata.to_json()); + m.insert("is_library".to_string(), self.is_library.to_json()); m.to_json() } } @@ -104,7 +106,8 @@ impl CrateDetails { authors.slug, \ crates.github_stars, \ crates.github_forks, \ - crates.github_issues \ + crates.github_issues, \ + releases.is_library \ FROM author_rels \ LEFT OUTER JOIN authors ON authors.id = author_rels.aid \ LEFT OUTER JOIN releases ON releases.id = author_rels.rid \ @@ -168,6 +171,7 @@ impl CrateDetails { github_forks: rows.get(0).get(19), github_issues: rows.get(0).get(20), metadata: metadata, + is_library: rows.get(0).get(21), }; if let Some(repository_url) = crate_details.repository_url.clone() { diff --git a/templates/crate_details.hbs b/templates/crate_details.hbs index 39406ab05..3f42c5ced 100644 --- a/templates/crate_details.hbs +++ b/templates/crate_details.hbs @@ -44,6 +44,9 @@
        + {{#unless is_library}} +
        {{name}}-{{version}} is not a library.
        + {{else}} {{#unless build_status}}
        cratesfyi failed to build {{name}}-{{version}}
        Please check build logs and if you believe this is cratesfyi's fault, report into this issue report.
        {{else}} @@ -51,6 +54,7 @@
        {{name}}-{{version}} doesn't have any documentation.
        {{/unless}} {{/unless}} + {{/unless}} {{#if readme}} {{{readme}}} {{else}} From a0c46a45a8cf2de23949ae6c90b7a9c075c98a95 Mon Sep 17 00:00:00 2001 From: Onur Aslan Date: Sun, 31 Jul 2016 12:34:31 +0300 Subject: [PATCH 33/79] Replace cratesfyi with docs.rs in templates And use smaller font in navigation --- templates/crate_details.hbs | 2 +- templates/header.hbs | 4 ++-- templates/navigation.hbs | 6 +++--- templates/releases.hbs | 4 ++-- templates/style.scss | 28 +++++++++++++++++++++++++--- 5 files changed, 33 insertions(+), 11 deletions(-) diff --git a/templates/crate_details.hbs b/templates/crate_details.hbs index 3f42c5ced..ef8dbd57e 100644 --- a/templates/crate_details.hbs +++ b/templates/crate_details.hbs @@ -48,7 +48,7 @@
        {{name}}-{{version}} is not a library.
        {{else}} {{#unless build_status}} -
        cratesfyi failed to build {{name}}-{{version}}
        Please check build logs and if you believe this is cratesfyi's fault, report into this issue report.
        +
        docs.rs failed to build {{name}}-{{version}}
        Please check build logs and if you believe this is docs.rs' fault, report into this issue report.
        {{else}} {{#unless rustdoc_status}}
        {{name}}-{{version}} doesn't have any documentation.
        diff --git a/templates/header.hbs b/templates/header.hbs index 673076cde..8b5abb460 100644 --- a/templates/header.hbs +++ b/templates/header.hbs @@ -3,7 +3,7 @@ - + @@ -18,7 +18,7 @@ {{#if varsb.javascript_highchartjs}} {{/if}} - {{#if title}}{{title}} - {{/if}}{{#if content.metadata.name}}{{content.metadata.name}} {{content.metadata.version}} - {{/if}}Cratesfyi + {{#if title}}{{title}} - {{/if}}{{#if content.metadata.name}}{{content.metadata.name}} {{content.metadata.version}} - {{/if}}Docs.rs diff --git a/templates/navigation.hbs b/templates/navigation.hbs index b7ac77016..ffbb78600 100644 --- a/templates/navigation.hbs +++ b/templates/navigation.hbs @@ -5,11 +5,11 @@ {{#unless varsb.show_search_form}} {{/unless}} - Cratesfyi + Docs.rs
          -
        • Releases
          • +
        • Releases
        • - Rust + Rust
          • The Rust Programming Language
          • Standard Library API Reference
            • diff --git a/templates/releases.hbs b/templates/releases.hbs index 5c92c1c32..76a969aed 100644 --- a/templates/releases.hbs +++ b/templates/releases.hbs @@ -2,7 +2,7 @@ {{#if varsb.show_search_form}}
            -

            Cratesfyi

            +

            Docs.rs

            @@ -68,7 +68,7 @@
            diff --git a/templates/style.scss b/templates/style.scss index d560ae907..ae20150d7 100644 --- a/templates/style.scss +++ b/templates/style.scss @@ -90,25 +90,47 @@ div.container { div.nav-container { border-bottom: 1px solid $color-border; + li { + border-left: 1px solid $color-border; + } + + .pure-menu-has-children>.pure-menu-link:after { + font-size: 14px; + } + a { + font-size: 14px; font-weight: 400; } + .pure-menu-link:hover { + color: $color-standard; + background-color: inherit; + } + form.landing-search-form-nav { input.search-input-nav { float: right; max-width: 200px; border: none; margin: 0 1em 0 0; - font-size: 1em; + font-size: 14px; text-align: right; box-shadow: none; - height: 38px; + height: 33px; } input.search-input-nav:focus { outline: unset; } } + + .pure-menu-children { + border: 1px solid $color-border; + margin-left: -1px; + li { + border-left: none; + } + } } div.landing { @@ -452,7 +474,7 @@ div.search-page-search-form { .rust-navigation-item { background: url(/rust-logo.png) no-repeat; background-position: 15px 45%; - background-size: 14px; + background-size: 12px; padding-left: 35px; } From 1faf8dc283d61801eb36b68ea031e84e9e54a2e2 Mon Sep 17 00:00:00 2001 From: Onur Aslan Date: Sun, 31 Jul 2016 17:46:34 +0300 Subject: [PATCH 34/79] Add platform selector --- src/web/rustdoc.rs | 51 +++++++++++++++++++++++++++++++--------- templates/navigation.hbs | 14 ++++++++++- templates/style.scss | 13 ++++++++++ 3 files changed, 66 insertions(+), 12 deletions(-) diff --git a/src/web/rustdoc.rs b/src/web/rustdoc.rs index 8c0e26ab8..e3233762b 100644 --- a/src/web/rustdoc.rs +++ b/src/web/rustdoc.rs @@ -24,6 +24,7 @@ struct RustdocPage { pub version: String, pub description: Option, pub metadata: Option, + pub platforms: Option } @@ -36,6 +37,7 @@ impl Default for RustdocPage { version: String::new(), description: None, metadata: None, + platforms: None, } } } @@ -51,6 +53,7 @@ impl ToJson for RustdocPage { m.insert("version".to_string(), self.version.to_json()); m.insert("description".to_string(), self.description.to_json()); m.insert("metadata".to_string(), self.metadata.to_json()); + m.insert("platforms".to_string(), self.platforms.to_json()); m.to_json() } } @@ -71,7 +74,7 @@ pub fn rustdoc_redirector_handler(req: &mut Request) -> IronResult { name, vers, target_name)[..]) - .unwrap(); + .unwrap(); let mut resp = Response::with((status::Found, Redirect(url))); use iron::headers::{Expires, HttpDate}; @@ -97,10 +100,10 @@ pub fn rustdoc_redirector_handler(req: &mut Request) -> IronResult { let target_name: String = conn.query("SELECT target_name FROM releases INNER JOIN crates ON \ crates.id = releases.crate_id WHERE crates.name = $1 \ AND releases.version = $2", - &[&crate_name, &version]) - .unwrap() - .get(0) - .get(0); + &[&crate_name, &version]) + .unwrap() + .get(0) + .get(0); redirect_to_doc(req, &crate_name, &version, &target_name) } @@ -165,16 +168,42 @@ pub fn rustdoc_html_server_handler(req: &mut Request) -> IronResult { let name = req.extensions.get::().unwrap().find("crate").unwrap_or("").to_string(); let version = req.extensions - .get::() - .unwrap() - .find("version") - .unwrap_or("") - .to_string(); + .get::() + .unwrap() + .find("version") + .unwrap_or("") + .to_string(); + + // content.metadata = MetaData::from_crate(&conn, &name, &version); + let (metadata, platforms) = { + let rows = conn.query("SELECT crates.name, + releases.version, + releases.description, + releases.target_name, + releases.rustdoc_status, + doc_targets + FROM releases + INNER JOIN crates ON crates.id = releases.crate_id + WHERE crates.name = $1 AND releases.version = $2", + &[&name, &version]).unwrap(); + + let metadata = MetaData { + name: rows.get(0).get(0), + version: rows.get(0).get(1), + description: rows.get(0).get(2), + target_name: rows.get(0).get(3), + rustdoc_status: rows.get(0).get(4), + }; + let platforms: Json = rows.get(0).get(5); + (Some(metadata), platforms) + }; - content.metadata = MetaData::from_crate(&conn, &name, &version); + content.metadata = metadata; + content.platforms = Some(platforms); Page::new(content) .set_true("show_package_navigation") .set_true("package_navigation_documentation_tab") + .set_true("package_navigation_show_platforms_tab") .to_resp("rustdoc") } diff --git a/templates/navigation.hbs b/templates/navigation.hbs index ffbb78600..357f9bfcc 100644 --- a/templates/navigation.hbs +++ b/templates/navigation.hbs @@ -46,8 +46,20 @@
          • Source
          • Builds
          + {{/with}} + {{#if ../varsb.package_navigation_show_platforms_tab}} + + {{/if}}
        - {{/with}} {{/if}} {{#if varsb.show_releases_navigation}} diff --git a/templates/style.scss b/templates/style.scss index ae20150d7..6d05dd2e8 100644 --- a/templates/style.scss +++ b/templates/style.scss @@ -438,6 +438,19 @@ div.cratesfyi-package-container { background-color: inherit; } } + + ul.platforms-menu { + float: right; + display: none; + ul.pure-menu-children { + left: auto; + right: 0; + } + + @media #{$media-sm} { + display: inline-block; + } + } } From 2e4da752b966d7d371611efc486d0f6e51096aff Mon Sep 17 00:00:00 2001 From: Onur Aslan Date: Sun, 31 Jul 2016 17:59:18 +0300 Subject: [PATCH 35/79] Adjust platforms submenu --- templates/style.scss | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/templates/style.scss b/templates/style.scss index 6d05dd2e8..5201b9524 100644 --- a/templates/style.scss +++ b/templates/style.scss @@ -126,6 +126,7 @@ div.nav-container { .pure-menu-children { border: 1px solid $color-border; + border-radius: 0 0 2px 2px; margin-left: -1px; li { border-left: none; @@ -445,6 +446,12 @@ div.cratesfyi-package-container { ul.pure-menu-children { left: auto; right: 0; + border: 1px solid $color-border; + border-radius: 2px; + } + + .pure-menu-has-children>.pure-menu-link:after { + font-size: 14px; } @media #{$media-sm} { From b1bf7f17530a9e8030a6dbd65fd9612c1cbd7cca Mon Sep 17 00:00:00 2001 From: Onur Aslan Date: Sun, 31 Jul 2016 18:27:49 +0300 Subject: [PATCH 36/79] Add margin-bottom to warning container --- templates/style.scss | 1 + 1 file changed, 1 insertion(+) diff --git a/templates/style.scss b/templates/style.scss index 5201b9524..dfc45d253 100644 --- a/templates/style.scss +++ b/templates/style.scss @@ -467,6 +467,7 @@ div.warning { background-color: lighten($color-type, 45%); padding: .4em 1em; text-align: center; + margin-bottom: 10px; a { color: $color-url; text-decoration: underline; From b9589955e9f1996eead5b1249f23f6ca634f3b4c Mon Sep 17 00:00:00 2001 From: Onur Aslan Date: Sun, 31 Jul 2016 18:42:56 +0300 Subject: [PATCH 37/79] Show completion rate for beta --- src/web/releases.rs | 6 ++++++ templates/releases.hbs | 7 +++++++ 2 files changed, 13 insertions(+) diff --git a/src/web/releases.rs b/src/web/releases.rs index 1842bcad0..579a87f04 100644 --- a/src/web/releases.rs +++ b/src/web/releases.rs @@ -223,9 +223,15 @@ fn get_search_results(conn: &Connection, pub fn home_page(req: &mut Request) -> IronResult { let conn = req.extensions.get::().unwrap(); let packages = get_releases(conn, 1, RELEASES_IN_HOME, Order::ReleaseTime); + + // TODO: Remove this after beta + let compiled_releases: i64 = conn.query("SELECT COUNT(*) FROM releases", &[]).unwrap().get(0).get(0); + let percentage = compiled_releases * 100 / 30432; + Page::new(packages) .set_true("show_search_form") .set_true("hide_package_navigation") + .set_int("beta_percentage_complete", percentage) .to_resp("releases") } diff --git a/templates/releases.hbs b/templates/releases.hbs index 76a969aed..002867275 100644 --- a/templates/releases.hbs +++ b/templates/releases.hbs @@ -10,6 +10,13 @@
        + {{#if varsi.beta_percentage_complete}} +

        +
        + Initial build is in progress, completion: {{varsi.beta_percentage_complete}}%
        + Search and github stats may not work! +
        + {{/if}} From 8fd7261a49dbcee3adc95238104f4ce27ad6edaa Mon Sep 17 00:00:00 2001 From: Onur Aslan Date: Mon, 22 Aug 2016 17:54:04 +0300 Subject: [PATCH 61/79] Add update-release-activity subcommand --- src/bin/cratesfyi.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/bin/cratesfyi.rs b/src/bin/cratesfyi.rs index e3e187d97..29693e56d 100644 --- a/src/bin/cratesfyi.rs +++ b/src/bin/cratesfyi.rs @@ -112,7 +112,8 @@ pub fn main() { .arg(Arg::with_name("PREFIX") .index(2) .help("Prefix of files in \ - database")))) + database"))) + .subcommand(SubCommand::with_name("update-release-activity"))) .get_matches(); @@ -190,6 +191,9 @@ pub fn main() { matches.value_of("PREFIX").unwrap_or(""), matches.value_of("DIRECTORY").unwrap()) .unwrap(); + } else if let Some(_) = matches.subcommand_matches("update-release-activity") { + // FIXME: This is actually util command not database + cratesfyi::utils::update_release_activity().expect("Failed to update release activity"); } } else if let Some(matches) = matches.subcommand_matches("start-web-server") { start_web_server(matches.value_of("SOCKET_ADDR")); From cb40f6f0f0eed93169e300df6d791f5e240b030c Mon Sep 17 00:00:00 2001 From: Onur Aslan Date: Tue, 23 Aug 2016 21:24:40 +0300 Subject: [PATCH 62/79] Add simple daemon --- Cargo.lock | 1 + Cargo.toml | 1 + src/bin/cratesfyi.rs | 22 ++++++++- src/docbuilder/options.rs | 1 + src/lib.rs | 1 + src/utils/daemon.rs | 98 +++++++++++++++++++++++++++++++++++++++ src/utils/mod.rs | 2 + 7 files changed, 125 insertions(+), 1 deletion(-) create mode 100644 src/utils/daemon.rs diff --git a/Cargo.lock b/Cargo.lock index 9f9a1766d..523269c46 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9,6 +9,7 @@ dependencies = [ "handlebars-iron 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", "hyper 0.9.7 (registry+https://github.com/rust-lang/crates.io-index)", "iron 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", "magic 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", "params 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/Cargo.toml b/Cargo.toml index e0a406c31..56197d063 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,6 +29,7 @@ r2d2 = "0.7.0" r2d2_postgres = "0.10.1" url = "1.1.1" params = "0.2.2" +libc = "0.2" [dependencies.cargo] git = "https://github.com/rust-lang/cargo.git" diff --git a/src/bin/cratesfyi.rs b/src/bin/cratesfyi.rs index 29693e56d..507ad3fc7 100644 --- a/src/bin/cratesfyi.rs +++ b/src/bin/cratesfyi.rs @@ -5,6 +5,7 @@ extern crate clap; #[macro_use] extern crate log; extern crate env_logger; +extern crate time; use std::env; @@ -19,7 +20,7 @@ use cratesfyi::db::add_path_into_database; pub fn main() { - let _ = env_logger::init(); + logger_init(); let matches = App::new("cratesfyi") .version(cratesfyi::BUILD_VERSION) @@ -95,6 +96,8 @@ pub fn main() { .index(1) .required(false) .help("Socket address to listen to"))) + .subcommand(SubCommand::with_name("daemon") + .about("Starts cratesfyi daemon")) .subcommand(SubCommand::with_name("database") .about("Database operations") .subcommand(SubCommand::with_name("init") @@ -197,7 +200,24 @@ pub fn main() { } } else if let Some(matches) = matches.subcommand_matches("start-web-server") { start_web_server(matches.value_of("SOCKET_ADDR")); + } else if let Some(_) = matches.subcommand_matches("daemon") { + cratesfyi::utils::start_daemon(); } else { println!("{}", matches.usage()); } } + + + +fn logger_init() { + let format = |record: &log::LogRecord| { + format!("{} [{}] {}: {}", + time::now().strftime("%Y/%m/%d %H:%M:%S").unwrap(), + record.level(), record.target(), record.args()) + }; + + let mut builder = env_logger::LogBuilder::new(); + builder.format(format); + builder.parse(&env::var("RUST_LOG").unwrap_or("cratesfyi=info".to_owned())); + builder.init().unwrap(); +} diff --git a/src/docbuilder/options.rs b/src/docbuilder/options.rs index 8adf44c8a..020f45f95 100644 --- a/src/docbuilder/options.rs +++ b/src/docbuilder/options.rs @@ -4,6 +4,7 @@ use std::{env, fmt}; use std::path::PathBuf; +#[derive(Clone)] pub struct DocBuilderOptions { pub keep_build_directory: bool, pub prefix: PathBuf, diff --git a/src/lib.rs b/src/lib.rs index 6cf89a7a3..382455eb0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -20,6 +20,7 @@ extern crate r2d2; extern crate r2d2_postgres; extern crate url; extern crate params; +extern crate libc; pub use self::docbuilder::DocBuilder; pub use self::docbuilder::ChrootBuilderResult; diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs new file mode 100644 index 000000000..2b6034130 --- /dev/null +++ b/src/utils/daemon.rs @@ -0,0 +1,98 @@ +//! Simple daemon +//! +//! This daemon will start web server, track new packages and build them + + +use std::env; +use libc::fork; +use std::process::exit; +use std::fs::File; +use std::io::Write; +use std::thread; +use std::time::Duration; +use std::path::PathBuf; +use time; +use DocBuilderOptions; +use DocBuilder; + + +const DAEMON_PID_FILE_PATH: &'static str = "/var/run/cratesfyi.pid"; + + +pub fn start_daemon() { + // first check required environment variables + for v in ["CRATESFYI_PREFIX", + "CRATESFYI_PREFIX", + "CRATESFYI_GITHUB_USERNAME", + "CRATESFYI_GITHUB_ACCESSTOKEN"] + .iter() { + env::var(v).expect("Environment variable not found"); + } + + info!("Starting cratesfyi {} daemon", ::BUILD_VERSION); + + // fork the process + let pid = unsafe { fork() }; + if pid > 0 { + let mut file = File::create(DAEMON_PID_FILE_PATH).expect("Failed to create pid file"); + writeln!(&mut file, "{}", pid).expect("Failed to write pid"); + + info!("cratesfyi daemon started on: {}", pid); + exit(0); + } + + + fn opts() -> DocBuilderOptions { + let prefix = PathBuf::from(env::var("CRATESFYI_PREFIX") + .expect("CRATESFYI_PREFIX environment variable not found")); + let opts = DocBuilderOptions::from_prefix(prefix); + opts.check_paths().unwrap(); + opts + } + + // check new crates every 5 minutes + thread::spawn(move || { + loop { + thread::sleep(Duration::from_secs(300)); + debug!("Checking new crates"); + let doc_builder = DocBuilder::new(opts()); + if let Err(e) = doc_builder.get_new_crates() { + error!("Failed to get new crates: {}", e); + } + } + }); + + + // build new crates every 3 minutes + thread::spawn(move || { + loop { + thread::sleep(Duration::from_secs(180)); + debug!("Building new crates"); + let mut doc_builder = DocBuilder::new(opts()); + if let Err(e) = doc_builder.build_packages_queue() { + error!("Failed build new crates: {}", e); + } + } + }); + + + // update release activity everyday at 02:00 + thread::spawn(move || { + loop { + thread::sleep(Duration::from_secs(60)); + let now = time::now(); + if now.tm_hour == 2 && now.tm_min == 0 { + info!("Updating release activity"); + if let Err(e) = ::utils::update_release_activity() { + error!("Failed to update release activity: {}", e); + } + } + } + }); + + // TODO: update ssl certificate every 3 months + + // at least start web server + info!("Starting web server"); + ::start_web_server(None); +} diff --git a/src/utils/mod.rs b/src/utils/mod.rs index cc9a80a78..47c87f3bc 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -5,8 +5,10 @@ pub use self::build_doc::{build_doc, get_package, source_path, update_sources}; pub use self::copy::{copy_dir, copy_doc_dir}; pub use self::github_updater::github_updater; pub use self::release_activity_updater::update_release_activity; +pub use self::daemon::start_daemon; mod github_updater; mod build_doc; mod copy; mod release_activity_updater; +mod daemon; From a4e297395cfb0fe3f158878ac8a96960904e12d5 Mon Sep 17 00:00:00 2001 From: Onur Aslan Date: Tue, 23 Aug 2016 21:50:21 +0300 Subject: [PATCH 63/79] Show error detail if a package fails to build in que --- src/docbuilder/queue.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/docbuilder/queue.rs b/src/docbuilder/queue.rs index 737847ed5..7182a499f 100644 --- a/src/docbuilder/queue.rs +++ b/src/docbuilder/queue.rs @@ -93,11 +93,9 @@ impl DocBuilder { let name: String = row.get(1); let version: String = row.get(2); - if let Ok(_) = self.build_package(&name[..], &version[..]) { - // remove package from que - let _ = conn.execute("DELETE FROM queue WHERE id = $1", &[&id]); - } else { - warn!("Failed to build package {}-{} from queue", name, version); + match self.build_package(&name[..], &version[..]) { + Ok(_) => { let _ = conn.execute("DELETE FROM queue WHERE id = $1", &[&id]); }, + Err(e) => error!("Failed to build package {}-{} from queue: {}", name, version, e), } } From 0222cd38d5fb38552f0179056fe34d9ca3afc7e4 Mon Sep 17 00:00:00 2001 From: Onur Aslan Date: Wed, 24 Aug 2016 12:44:00 +0300 Subject: [PATCH 64/79] Update daemon --- src/utils/daemon.rs | 46 +++++++++++++++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 14 deletions(-) diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs index 2b6034130..cacdc7b45 100644 --- a/src/utils/daemon.rs +++ b/src/utils/daemon.rs @@ -3,14 +3,13 @@ //! This daemon will start web server, track new packages and build them -use std::env; -use libc::fork; +use std::{env, thread}; use std::process::exit; use std::fs::File; use std::io::Write; -use std::thread; use std::time::Duration; use std::path::PathBuf; +use libc::fork; use time; use DocBuilderOptions; use DocBuilder; @@ -29,7 +28,8 @@ pub fn start_daemon() { env::var(v).expect("Environment variable not found"); } - info!("Starting cratesfyi {} daemon", ::BUILD_VERSION); + // check paths once + opts().check_paths().unwrap(); // fork the process let pid = unsafe { fork() }; @@ -37,19 +37,11 @@ pub fn start_daemon() { let mut file = File::create(DAEMON_PID_FILE_PATH).expect("Failed to create pid file"); writeln!(&mut file, "{}", pid).expect("Failed to write pid"); - info!("cratesfyi daemon started on: {}", pid); + info!("cratesfyi {} daemon started on: {}", ::BUILD_VERSION, pid); exit(0); } - fn opts() -> DocBuilderOptions { - let prefix = PathBuf::from(env::var("CRATESFYI_PREFIX") - .expect("CRATESFYI_PREFIX environment variable not found")); - let opts = DocBuilderOptions::from_prefix(prefix); - opts.check_paths().unwrap(); - opts - } - // check new crates every 5 minutes thread::spawn(move || { loop { @@ -67,11 +59,29 @@ pub fn start_daemon() { thread::spawn(move || { loop { thread::sleep(Duration::from_secs(180)); + + let mut opts = opts(); + opts.skip_if_exists = true; + + // check lock file + if opts.prefix.join("cratesfyi.lock").exists() { + warn!("Lock file exits, skipping building new crates"); + } + + let mut doc_builder = DocBuilder::new(opts); + if let Err(e) = doc_builder.load_cache() { + error!("Failed to load cache: {}", e); + continue; + } + debug!("Building new crates"); - let mut doc_builder = DocBuilder::new(opts()); if let Err(e) = doc_builder.build_packages_queue() { error!("Failed build new crates: {}", e); } + + if let Err(e) = doc_builder.save_cache() { + error!("Failed to save cache: {}", e); + } } }); @@ -96,3 +106,11 @@ pub fn start_daemon() { info!("Starting web server"); ::start_web_server(None); } + + + +fn opts() -> DocBuilderOptions { + let prefix = PathBuf::from(env::var("CRATESFYI_PREFIX") + .expect("CRATESFYI_PREFIX environment variable not found")); + DocBuilderOptions::from_prefix(prefix) +} From f22280495e09e2e10fa29488e3aaf5f2cdb04f29 Mon Sep 17 00:00:00 2001 From: Onur Aslan Date: Wed, 24 Aug 2016 12:52:36 +0300 Subject: [PATCH 65/79] Update sources before build --- src/utils/daemon.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs index cacdc7b45..1077af162 100644 --- a/src/utils/daemon.rs +++ b/src/utils/daemon.rs @@ -13,6 +13,7 @@ use libc::fork; use time; use DocBuilderOptions; use DocBuilder; +use utils::{update_sources, update_release_activity}; const DAEMON_PID_FILE_PATH: &'static str = "/var/run/cratesfyi.pid"; @@ -66,6 +67,13 @@ pub fn start_daemon() { // check lock file if opts.prefix.join("cratesfyi.lock").exists() { warn!("Lock file exits, skipping building new crates"); + continue; + } + + // update index + if let Err(e) = update_sources() { + error!("Failed to update sources: {}", e); + continue; } let mut doc_builder = DocBuilder::new(opts); @@ -93,7 +101,7 @@ pub fn start_daemon() { let now = time::now(); if now.tm_hour == 2 && now.tm_min == 0 { info!("Updating release activity"); - if let Err(e) = ::utils::update_release_activity() { + if let Err(e) = update_release_activity() { error!("Failed to update release activity: {}", e); } } From efdf58893592b71204565a949967963969a8a18a Mon Sep 17 00:00:00 2001 From: Onur Aslan Date: Wed, 24 Aug 2016 18:04:10 +0300 Subject: [PATCH 66/79] Use plural form of platforms in navigation menu --- templates/navigation.hbs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/navigation.hbs b/templates/navigation.hbs index a63b7b839..d40c208a0 100644 --- a/templates/navigation.hbs +++ b/templates/navigation.hbs @@ -38,7 +38,7 @@ {{#if ../varsb.package_navigation_show_platforms_tab}}
        • - Platforms + Platform
            {{#each content.platforms}}
          • {{this}}
            • From 10d26cdb7951212a0e7c3499cfccf3e3b25876d0 Mon Sep 17 00:00:00 2001 From: Onur Aslan Date: Wed, 24 Aug 2016 18:09:15 +0300 Subject: [PATCH 67/79] Update search index every 3 hours --- src/utils/daemon.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs index 1077af162..48ee4a58c 100644 --- a/src/utils/daemon.rs +++ b/src/utils/daemon.rs @@ -14,6 +14,7 @@ use time; use DocBuilderOptions; use DocBuilder; use utils::{update_sources, update_release_activity}; +use db::{connect_db, update_search_index}; const DAEMON_PID_FILE_PATH: &'static str = "/var/run/cratesfyi.pid"; @@ -108,6 +109,18 @@ pub fn start_daemon() { } }); + + // update search index every 3 hours + thread::spawn(move || { + loop { + thread::sleep(Duration::from_secs(60*60*3)); + let conn = connect_db().expect("Failed to connect database"); + if let Err(e) = update_search_index(&conn) { + error!("Failed to update search index: {}", e); + } + } + }); + // TODO: update ssl certificate every 3 months // at least start web server From 82e1eb15ecc6aa24733bd5651927947c0f782682 Mon Sep 17 00:00:00 2001 From: Onur Aslan Date: Wed, 24 Aug 2016 18:39:35 +0300 Subject: [PATCH 68/79] Save pid file into prefix --- src/utils/daemon.rs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs index 48ee4a58c..9063e3551 100644 --- a/src/utils/daemon.rs +++ b/src/utils/daemon.rs @@ -17,8 +17,6 @@ use utils::{update_sources, update_release_activity}; use db::{connect_db, update_search_index}; -const DAEMON_PID_FILE_PATH: &'static str = "/var/run/cratesfyi.pid"; - pub fn start_daemon() { // first check required environment variables @@ -30,13 +28,16 @@ pub fn start_daemon() { env::var(v).expect("Environment variable not found"); } + let dbopts = opts(); + // check paths once - opts().check_paths().unwrap(); + dbopts.check_paths().unwrap(); // fork the process let pid = unsafe { fork() }; if pid > 0 { - let mut file = File::create(DAEMON_PID_FILE_PATH).expect("Failed to create pid file"); + let mut file = File::create(dbopts.prefix.join("cratesfyi.pid")) + .expect("Failed to create pid file"); writeln!(&mut file, "{}", pid).expect("Failed to write pid"); info!("cratesfyi {} daemon started on: {}", ::BUILD_VERSION, pid); @@ -113,7 +114,7 @@ pub fn start_daemon() { // update search index every 3 hours thread::spawn(move || { loop { - thread::sleep(Duration::from_secs(60*60*3)); + thread::sleep(Duration::from_secs(60 * 60 * 3)); let conn = connect_db().expect("Failed to connect database"); if let Err(e) = update_search_index(&conn) { error!("Failed to update search index: {}", e); @@ -132,6 +133,6 @@ pub fn start_daemon() { fn opts() -> DocBuilderOptions { let prefix = PathBuf::from(env::var("CRATESFYI_PREFIX") - .expect("CRATESFYI_PREFIX environment variable not found")); + .expect("CRATESFYI_PREFIX environment variable not found")); DocBuilderOptions::from_prefix(prefix) } From 64b42f4d5ae4ed9770386c535a106b6d27b3965f Mon Sep 17 00:00:00 2001 From: Onur Aslan Date: Wed, 24 Aug 2016 21:33:12 +0300 Subject: [PATCH 69/79] Fix syntax --- src/docbuilder/queue.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/docbuilder/queue.rs b/src/docbuilder/queue.rs index b2ea8fde0..24c246e88 100644 --- a/src/docbuilder/queue.rs +++ b/src/docbuilder/queue.rs @@ -116,7 +116,7 @@ impl DocBuilder { match self.build_package(&name[..], &version[..]) { Ok(_) => { let _ = conn.execute("DELETE FROM queue WHERE id = $1", &[&id]); - } + }, Err(e) => { error!("Failed to build package {}-{} from queue: {}", name, From 065a2f8cad06d9bafaface5e600b31e9d313facd Mon Sep 17 00:00:00 2001 From: Onur Aslan Date: Wed, 24 Aug 2016 18:37:11 +0300 Subject: [PATCH 70/79] Update README --- INSTALLATION.md | 172 ++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 153 +++++++++--------------------------------- src/lib.rs | 2 + 3 files changed, 204 insertions(+), 123 deletions(-) create mode 100644 INSTALLATION.md diff --git a/INSTALLATION.md b/INSTALLATION.md new file mode 100644 index 000000000..1f2016e2c --- /dev/null +++ b/INSTALLATION.md @@ -0,0 +1,172 @@ +# Installation Guide + +## Requirements + +`cratesfyi` is written in Rust and requires Rust compiler. It is developed under +Linux, it may not work or compile under other operating systems. List of +requirements to compile `cratesfyi`: + +* rustc compiler +* gcc +* cmake +* pkg-config +* openssl (development files) +* libmagic (development files) +* git +* postgresql +* libc (development files) +* lxc + +## `cratesfyi-prefix` directory + +`cratesfyi` is using a prefix directory for: + +* Clone of `crates.io-index` +* `documentations` directory to copy generated documentation +* `sources` directory +* `public_html` directory to serve some static files +* `cratesfyi-container` a Linux container (LXC) to build crates in a safe + environment + +An example script to create cratesfyi-prefix directory (note: you don't need +setup a lxc-container if you are gonna run only web interface, +skip to setting up database): + + +```sh +#!/bin/sh +# Creates cratesfyi-prefix directory for cratesfyi +# This script is designed to run on Debian based operating systems, +# and tested under Debian jessie and sid + +set -e + +PREFIX=$(pwd)/cratesfyi-prefix +DIST_TEMPLATE=debian +DIST_RELEASE=jessie +DIST_MIRROR=http://httpredir.debian.org/debian + +mkdir $PREFIX +mkdir -p $PREFIX/sources $PREFIX/documentations +git clone https://github.com/rust-lang/crates.io-index.git $PREFIX/crates.io-index + +# Create debian8 lxc container into cratesfyi-container directory +# Use your own distribution template and release name +sudo LANG=C MIRROR=$DIST_MIRROR \ + lxc-create -n cratesfyi-container -P $PREFIX \ + -t $DIST_TEMPLATE -- -r $DIST_RELEASE + +# Due to some bug in lxc-attach this container +# must have a symbolic link in /var/lib/lxc +sudo ln -s $PREFIX/cratesfyi-container /var/lib/lxc + +# Container directory must be accessible by current user +sudo chmod 755 $PREFIX/cratesfyi-container + +# Setup lxc network +echo 'USE_LXC_BRIDGE="true" +LXC_BRIDGE="lxcbr0" +LXC_ADDR="10.0.3.1" +LXC_NETMASK="255.255.255.0" +LXC_NETWORK="10.0.3.0/24" +LXC_DHCP_RANGE="10.0.3.2,10.0.3.254" +LXC_DHCP_MAX="253" +LXC_DHCP_CONFILE="" +LXC_DOMAIN=""' | sudo tee /etc/default/lxc-net + +# Start network interface +sudo service lxc-net restart + +# Setup network for container +sudo sed -i 's/lxc.network.type.*/lxc.network.type = veth\nlxc.network.link = lxcbr0/' \ + $PREFIX/cratesfyi-container/config + +# Start lxc container +sudo lxc-start -n cratesfyi-container + +# Add user accounts into container +# cratesfyi is using multiple user accounts to run cargo simultaneously +for user in $(whoami) cratesfyi updater; do + sudo lxc-attach -n cratesfyi-container -- \ + adduser --disabled-login --disabled-password --gecos "" $user +done + +# Install required packages for rust installation +sudo lxc-attach -n cratesfyi-container -- apt-get update +sudo lxc-attach -n cratesfyi-container -- apt-get install -y file git curl sudo ca-certificates + +# Install rust nightly into container +sudo lxc-attach -n cratesfyi-container -- \ + su - -c 'curl -sSf https://static.rust-lang.org/rustup.sh | sh -s -- --channel=nightly' +``` + + +The last step is to install cratesfyi into the guest machine +(or build in guest machine). If your host and guest +operating system is same simply build cratesfyi in release mode and copy into +`/usr/local/bin` directory of guest system: + +```sh +cargo build --release +cp target/release/cratesfyi CRATESFYI_PREFIX_DIR/rootfs/usr/local/bin/ +``` + +cratesfyi is only using `lxd-attach` command with sudo. Make sure your user +account can use this command without root password. Example `sudoers` entry: + +```text +cratesfyi ALL=(ALL) NOPASSWD: /usr/bin/lxc-attach +``` + +### Setting up database + +cratesfyi is using postgresql database to store crate and build +information. You need to set up database before using chroot builder. To do +this: + +```text +$ sudo su - postgres -c psql +# First create a user +postgres=# CREATE USER cratesfyi WITH PASSWORD 'password'; +postgres=# CREATE DATABASE cratesfyi OWNER cratesfyi; +postgres=# \q +# Initialize database with cratesfyi +$ CRATESFYI_DATABASE_URL=postgresql://cratesfyi:password@localhost cratesfyi database init +``` + +## Environment variables + +`cratesfyi` is using various environment variables to run. Those are: + +```sh +HOME=/srv/cratesfyi +PATH=/srv/cratesfyi/.cargo/bin:/usr/local/bin:/usr/bin:/bin +CRATESFYI_PREFIX=/srv/cratesfyi/cratesfyi-prefix +CRATESFYI_DATABASE_URL='postgresql://cratesfyi@localhost' +CRATESFYI_GITHUB_USERNAME='USERNAME' +CRATESFYI_GITHUB_ACCESSTOKEN='TOKEN' +``` + +You can also specify `RUST_LOG=cratesfyi` to see all log messages. It will only +show `INFO` or more leveled log messages without any `RUST_LOG` environment +variable. + +Example systemd service file for cratesfyi: + +```text +[Unit] +Description=Cratesfyi daemon +After=network.target postgresql.service + +[Service] +User=cratesfyi +Group=cratesfyi +Type=forking +PIDFile=/srv/cratesfyi/cratesfyi-prefix/cratesfyi.pid +EnvironmentFile=/srv/cratesfyi/env +ExecStart=/bin/sh -c '/srv/cratesfyi/cratesfyi daemon > /srv/cratesfyi/cratesfyi-prefix/cratesfyi.log 2>&1' +WorkingDirectory=/srv/cratesfyi + +[Install] +WantedBy=multi-user.target +``` diff --git a/README.md b/README.md index 25f4fa676..41b3b3e62 100644 --- a/README.md +++ b/README.md @@ -1,139 +1,46 @@ - -# docs.rs +# Docs.rs [![Build Status](https://secure.travis-ci.org/onur/docs.rs.svg?branch=master)](https://travis-ci.org/onur/docs.rs) [![License](https://img.shields.io/badge/license-MIT-blue.svg)](https://raw.githubusercontent.com/onur/docs.rs/master/LICENSE) -Documentation host for the Rust Programming Language crates. - - -## Installation - -cratesfyi needs `cratesfyi-prefix` directory and a postgresql server to run. -This directory must have: - -* Clone of `crates.io-index` repository. -* `sources` directory for crate sources. -* `cratesfyi-container` lxc container for building crates. This container - must use exact same operating system as host machine to avoid conflicts - (or you can build cratesfyi in guest system). -* `documentations` directory for crate documentations. - - -An example script to create cratesfyi-prefix directory. Make sure you have -`git` and `lxc` packages installed. **Run this script as a normal user**: - - -```sh -#!/bin/sh -# Creates cratesfyi-prefix directory for cratesfyi -# This script is designed to run on Debian based operating systems, -# and tested under Debian jessie and sid - -set -e - -PREFIX=$(pwd)/cratesfyi-prefix -DIST_TEMPLATE=debian -DIST_RELEASE=jessie -DIST_MIRROR=http://httpredir.debian.org/debian - -mkdir $PREFIX -mkdir -p $PREFIX/sources $PREFIX/documentations -git clone https://github.com/rust-lang/crates.io-index.git $PREFIX/crates.io-index - -# Create debian8 lxc container into cratesfyi-container directory -# Use your own distribution template and release name -sudo LANG=C MIRROR=$DIST_MIRROR \ - lxc-create -n cratesfyi-container -P $PREFIX \ - -t $DIST_TEMPLATE -- -r $DIST_RELEASE - -# Due to some bug in lxc-attach this container -# must have a symbolic link in /var/lib/lxc -sudo ln -s $PREFIX/cratesfyi-container /var/lib/lxc - -# Container directory must be accessible by current user -sudo chmod 755 $PREFIX/cratesfyi-container - -# Setup lxc network -echo 'USE_LXC_BRIDGE="true" -LXC_BRIDGE="lxcbr0" -LXC_ADDR="10.0.3.1" -LXC_NETMASK="255.255.255.0" -LXC_NETWORK="10.0.3.0/24" -LXC_DHCP_RANGE="10.0.3.2,10.0.3.254" -LXC_DHCP_MAX="253" -LXC_DHCP_CONFILE="" -LXC_DOMAIN=""' | sudo tee /etc/default/lxc-net - -# Start network interface -sudo service lxc-net restart - -# Setup network for container -sudo sed -i 's/lxc.network.type.*/lxc.network.type = veth\nlxc.network.link = lxcbr0/' \ - $PREFIX/cratesfyi-container/config - -# Start lxc container -sudo lxc-start -n cratesfyi-container - -# Add user accounts into container -# cratesfyi is using multiple user accounts to run cargo simultaneously -for user in $(whoami) cratesfyi updater; do - sudo lxc-attach -n cratesfyi-container -- \ - adduser --disabled-login --disabled-password --gecos "" $user -done - -# Install required packages for rust installation -sudo lxc-attach -n cratesfyi-container -- apt-get update -sudo lxc-attach -n cratesfyi-container -- apt-get install -y file git curl sudo ca-certificates - -# Install rust nightly into container -sudo lxc-attach -n cratesfyi-container -- \ - su - -c 'curl -sSf https://static.rust-lang.org/rustup.sh | sh -s -- --channel=nightly' -``` - - -The last step is to install cratesfyi into the guest machine -(or build in guest machine). If your host and guest -operating system is same simply build cratesfyi in release mode and copy into -`/usr/local/bin` directory of guest system: +Docs.rs (formerly cratesfyi) is an open source project to host documentation +of crates for the Rust Programming Language. -```sh -cargo build --release -cp target/release/cratesfyi CRATESFYI_PREFIX_DIR/rootfs/usr/local/bin/ -``` +Docs.rs automatically builds crates' documentation released on crates.io using +the nightly release of the Rust compiler. -cratesfyi is only using `lxd-attach` command with sudo. Make sure your user -account can use this command without root password. Example `sudoers` entry: +The README of a crate is taken from the readme field defined in Cargo.toml. +If a crate doesn't have this field, no README will be displayed. -```text -yourusername ALL=(ALL) NOPASSWD: /usr/sbin/chroot -``` +### Redirections +Docs.rs is using semver to parse URLs. You can use this feature to access +crates' documentation easily. Example of URL redirections for `clap` crate: -### Setting up database +| URL | Redirects to documentation of | +|------------------------------|------------------------------------------------| +| | Latest version of clap | +| | 2.* version | +| | 2.9.* version | +| | 2.9.3 version (you don't need = unlike semver) | -cratesfyi is using postgresql database to store crate and build -information. You need to set up database before using chroot builder. To do -this: +The crates.fyi domain will redirect to docs.rs, supporting all of the +redirects discussed above -```sh -$ sudo su - postgres -c psql -# First create a user -postgres=# CREATE USER cratesfyi WITH PASSWORD 'password'; -postgres=# CREATE DATABASE cratesfyi OWNER cratesfyi; -postgres=# \q -# Initialize database with cratesfyi -CRATESFYI_DATABASE_URL=postgresql://cratesfyi:password@localhost ./cratesfyi database init -``` +#### Contributors -Make sure to export `CRATESFYI_DATABASE_URL` environment variable before -using cratesfyi. +* [https://github.com/onur](Onur Aslan) +* [https://github.com/frewsxcv](Corey Farwell) +* [https://github.com/jonhoo](Jon Gjengset) +* [https://github.com/mattyhall](Matthew Hall) +* [https://github.com/GuillaumeGomez](Guillaume Gomez) +* [https://github.com/Mark-Simulacrum](Mark Simulacrum) +#### Sponsors -## Environment variables +Hosting generously provided by: -cratesfyi is using few environment variables: +![Leaseweb](https://docs.rs/leaseweb.gif) -* `CRATESFYI_PREFIX` Prefix directory for cratesfyi -* `CRATESFYI_DATABASE_URL` Postgresql database URL -* `RUST_LOG` Set this to desired log level to get log messages +If you are interested in sponsoring Docs.rs, please don't hesitate to +contact us at TODO. diff --git a/src/lib.rs b/src/lib.rs index 382455eb0..327fee5c1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,3 +1,5 @@ +//! [Docs.rs](https://docs.rs) (formerly cratesfyi) is an open source project to host +//! documentation of crates for the Rust Programming Language. #[macro_use] extern crate log; From 4038d5f56f8a926c5475fee4a286bd473adb12af Mon Sep 17 00:00:00 2001 From: Onur Aslan Date: Wed, 24 Aug 2016 21:52:17 +0300 Subject: [PATCH 71/79] Update README --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 41b3b3e62..4c0f95637 100644 --- a/README.md +++ b/README.md @@ -29,12 +29,12 @@ redirects discussed above #### Contributors -* [https://github.com/onur](Onur Aslan) -* [https://github.com/frewsxcv](Corey Farwell) -* [https://github.com/jonhoo](Jon Gjengset) -* [https://github.com/mattyhall](Matthew Hall) -* [https://github.com/GuillaumeGomez](Guillaume Gomez) -* [https://github.com/Mark-Simulacrum](Mark Simulacrum) +* [Onur Aslan](https://github.com/onur]) +* [Corey Farwell](https://github.com/frewsxcv) +* [Jon Gjengset](https://github.com/jonhoo) +* [Matthew Hall](https://github.com/mattyhall) +* [Guillaume Gomez](https://github.com/GuillaumeGomez) +* [Mark Simulacrum](https://github.com/Mark-Simulacrum) #### Sponsors From cf27c9e903f47988ddd8c562bed0edad05c43843 Mon Sep 17 00:00:00 2001 From: Onur Aslan Date: Wed, 24 Aug 2016 21:54:33 +0300 Subject: [PATCH 72/79] Add a bit margin to releases container We don't have a footer anymore, add some margin to make it look like better --- templates/style.scss | 1 + 1 file changed, 1 insertion(+) diff --git a/templates/style.scss b/templates/style.scss index 3f36385d9..b0e6cb96d 100644 --- a/templates/style.scss +++ b/templates/style.scss @@ -164,6 +164,7 @@ div.landing { div.recent-releases-container { text-align: left; + margin-bottom: 50px; ul, li { list-style-type: none; From cc41c5a2cc50a8a6861af5e1a0203a44ec87de72 Mon Sep 17 00:00:00 2001 From: Onur Aslan Date: Wed, 24 Aug 2016 22:00:28 +0300 Subject: [PATCH 73/79] Add github_updater to daemon --- src/utils/daemon.rs | 13 ++++++++++++- src/utils/github_updater.rs | 2 +- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs index 9063e3551..79cb2c487 100644 --- a/src/utils/daemon.rs +++ b/src/utils/daemon.rs @@ -13,7 +13,7 @@ use libc::fork; use time; use DocBuilderOptions; use DocBuilder; -use utils::{update_sources, update_release_activity}; +use utils::{update_sources, update_release_activity, github_updater}; use db::{connect_db, update_search_index}; @@ -122,6 +122,17 @@ pub fn start_daemon() { } }); + + // update github stats every 6 hours + thread::spawn(move || { + loop { + thread::sleep(Duration::from_secs(60 * 60 * 6)); + if let Err(e) = github_updater() { + error!("Failed to update github fields: {}", e); + } + } + }); + // TODO: update ssl certificate every 3 months // at least start web server diff --git a/src/utils/github_updater.rs b/src/utils/github_updater.rs index 52faac909..307ee2afa 100644 --- a/src/utils/github_updater.rs +++ b/src/utils/github_updater.rs @@ -54,7 +54,7 @@ pub fn github_updater() -> Result<(), DocBuilderError> { &crate_id]) .map_err(DocBuilderError::DatabaseError) }) { - error!("Failed to update github fields of: {} {}", crate_name, err); + debug!("Failed to update github fields of: {} {}", crate_name, err); } // sleep for rate limits From 188700e3f518db09de898453a644ded043d5b130 Mon Sep 17 00:00:00 2001 From: Onur Aslan Date: Fri, 26 Aug 2016 11:58:41 +0300 Subject: [PATCH 74/79] Increase delay between pull requests This only got stuck once but I belive its more safer to increase duration between pull requests --- src/utils/daemon.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs index 79cb2c487..a8d567a00 100644 --- a/src/utils/daemon.rs +++ b/src/utils/daemon.rs @@ -45,10 +45,11 @@ pub fn start_daemon() { } - // check new crates every 5 minutes + // check new crates every 15 minutes + // FIXME: why not just check new crates and build them if there is any? thread::spawn(move || { loop { - thread::sleep(Duration::from_secs(300)); + thread::sleep(Duration::from_secs(900)); debug!("Checking new crates"); let doc_builder = DocBuilder::new(opts()); if let Err(e) = doc_builder.get_new_crates() { @@ -58,10 +59,10 @@ pub fn start_daemon() { }); - // build new crates every 3 minutes + // build crates in que every 12 minutes thread::spawn(move || { loop { - thread::sleep(Duration::from_secs(180)); + thread::sleep(Duration::from_secs(720)); let mut opts = opts(); opts.skip_if_exists = true; From 6757c918cd77cc7a004c0eb9e5d614562a953a57 Mon Sep 17 00:00:00 2001 From: Onur Aslan Date: Fri, 26 Aug 2016 12:09:24 +0300 Subject: [PATCH 75/79] Update TODO's * Add report link for failed builds. * Add email for contact. --- templates/about.hbs | 2 +- templates/builds.hbs | 1 - templates/crate_details.hbs | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/templates/about.hbs b/templates/about.hbs index 3976568b0..ff466e760 100644 --- a/templates/about.hbs +++ b/templates/about.hbs @@ -80,7 +80,7 @@

            Sponsors

            Hosting generously provided by:

            Powered by LeaseWeb

            -

            If you are interested in sponsoring Docs.rs, please don't hesitate to contact us at TODO.

            +

            If you are interested in sponsoring Docs.rs, please don't hesitate to contact us at :onur@onur.im

        {{> footer}} diff --git a/templates/builds.hbs b/templates/builds.hbs index 9282070c8..843cce49e 100644 --- a/templates/builds.hbs +++ b/templates/builds.hbs @@ -36,7 +36,6 @@ $ cratesfyi ...
        • {{/each}}
      - {{/with}} diff --git a/templates/crate_details.hbs b/templates/crate_details.hbs index 4ad7e440c..e8570b89f 100644 --- a/templates/crate_details.hbs +++ b/templates/crate_details.hbs @@ -48,7 +48,7 @@
      {{name}}-{{version}} is not a library.
      {{else}} {{#unless build_status}} -
      docs.rs failed to build {{name}}-{{version}}
      Please check       build logs and if you believe this is docs.rs' fault, report into this issue report.
      +
      docs.rs failed to build {{name}}-{{version}}
      Please check build logs and if you believe this is docs.rs' fault, report into this issue report.
      {{else}} {{#unless rustdoc_status}}
      {{name}}-{{version}} doesn't have any documentation.
      From ea2a91e6066389f537c00a5efc79f20f56a20960 Mon Sep 17 00:00:00 2001 From: Onur Aslan Date: Fri, 26 Aug 2016 12:11:00 +0300 Subject: [PATCH 76/79] Fix title of about page --- src/web/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/web/mod.rs b/src/web/mod.rs index a7d72fcc2..7ee0d998e 100644 --- a/src/web/mod.rs +++ b/src/web/mod.rs @@ -60,7 +60,7 @@ impl CratesfyiHandler { let mut router = Router::new(); router.get("/", releases::home_page); router.get("/about", |_: &mut Request| { - page::Page::new(false).title("About docs.rs").to_resp("about") + page::Page::new(false).title("About Docs.rs").to_resp("about") }); router.get("/releases", releases::releases_handler); router.get("/releases/recent/:page", releases::releases_handler); From 2ef3ffae55132f937a70b36ecbba64b5d14b747e Mon Sep 17 00:00:00 2001 From: Onur Aslan Date: Fri, 26 Aug 2016 12:17:26 +0300 Subject: [PATCH 77/79] Fix email address --- templates/about.hbs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/about.hbs b/templates/about.hbs index ff466e760..66e6147eb 100644 --- a/templates/about.hbs +++ b/templates/about.hbs @@ -80,7 +80,7 @@

      Sponsors

      Hosting generously provided by:

      Powered by LeaseWeb

      -

      If you are interested in sponsoring Docs.rs, please don't hesitate to contact us at :onur@onur.im

      +

      If you are interested in sponsoring Docs.rs, please don't hesitate to contact us at onur@onur.im

      {{> footer}} From 48d341a6624fa6f7a99161fe103632aa061a2b07 Mon Sep 17 00:00:00 2001 From: Onur Aslan Date: Fri, 26 Aug 2016 12:36:15 +0300 Subject: [PATCH 78/79] Increase static file cache duration to 12 months --- src/web/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/web/mod.rs b/src/web/mod.rs index 7ee0d998e..71b750c6f 100644 --- a/src/web/mod.rs +++ b/src/web/mod.rs @@ -29,7 +29,7 @@ use std::collections::BTreeMap; /// Duration of static files for staticfile and DatabaseFileHandler (in seconds) -const STATIC_FILE_CACHE_DURATION: u64 = 60 * 60 * 24 * 3; // 3 days +const STATIC_FILE_CACHE_DURATION: u64 = 60 * 60 * 24 * 30 * 12; // 12 months struct CratesfyiHandler { From 62fccee1929e121eddf3b1c719dca504bf141d70 Mon Sep 17 00:00:00 2001 From: Onur Aslan Date: Fri, 26 Aug 2016 12:57:33 +0300 Subject: [PATCH 79/79] Fix top position of sidebar --- templates/style.scss | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/style.scss b/templates/style.scss index b0e6cb96d..729d1c22a 100644 --- a/templates/style.scss +++ b/templates/style.scss @@ -62,7 +62,7 @@ div.rustdoc { } .sidebar { - top: 138px; + top: 160px; left: auto; }