From 9ec2ad9778e868fd8fd8766b670f5da819df25ed Mon Sep 17 00:00:00 2001 From: Thomas Feiner Date: Mon, 15 May 2017 11:39:22 +0200 Subject: [PATCH 1/2] build method should return interface instead of implementation + formatting --- .../bots/KnownHostBotVerifierBuilder.java | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/optimaize/webcrawlerverifier/bots/KnownHostBotVerifierBuilder.java b/src/main/java/com/optimaize/webcrawlerverifier/bots/KnownHostBotVerifierBuilder.java index 4dd74da..f04b27c 100644 --- a/src/main/java/com/optimaize/webcrawlerverifier/bots/KnownHostBotVerifierBuilder.java +++ b/src/main/java/com/optimaize/webcrawlerverifier/bots/KnownHostBotVerifierBuilder.java @@ -19,7 +19,7 @@ public class KnownHostBotVerifierBuilder { public KnownHostBotVerifierBuilder crawlerData(@NotNull CrawlerData crawlerData) { - if (this.crawlerData!=null) throw new IllegalStateException("The crawlerData was set already!"); + if (this.crawlerData != null) throw new IllegalStateException("The crawlerData was set already!"); this.crawlerData = crawlerData; return this; } @@ -27,10 +27,11 @@ public KnownHostBotVerifierBuilder crawlerData(@NotNull CrawlerData crawlerData) /** */ public KnownHostBotVerifierBuilder dnsVerifier(@NotNull ReverseDnsVerifier dnsVerifier) { - if (this.dnsVerifier!=null) throw new IllegalStateException("The dnsVerifier was set already!"); + if (this.dnsVerifier != null) throw new IllegalStateException("The dnsVerifier was set already!"); this.dnsVerifier = dnsVerifier; return this; } + /** * Uses the {@link DnsjavaReverseDnsVerifier} with the default name server(s) provided by the system. */ @@ -43,26 +44,27 @@ public KnownHostBotVerifierBuilder dnsVerifierDefault() { * If you must, for testing, then pass in a dummy cache that drops all. */ public KnownHostBotVerifierBuilder dnsResultCache(@NotNull Cache dnsResultCache) { - if (this.dnsResultCache !=null) throw new IllegalStateException("The dnsResultCache was set already!"); + if (this.dnsResultCache != null) throw new IllegalStateException("The dnsResultCache was set already!"); this.dnsResultCache = dnsResultCache; return this; } + /** * Uses maximumSize(1_000) and expireAfterWrite(3*24, TimeUnit.HOURS) */ public KnownHostBotVerifierBuilder dnsResultCacheDefault() { Cache cache = CacheBuilder.newBuilder() .maximumSize(1_000) - .expireAfterWrite(3*24, TimeUnit.HOURS) + .expireAfterWrite(3 * 24, TimeUnit.HOURS) .build(); return dnsResultCache(cache); } @NotNull - public KnownHostBotVerifierImpl build() { - if (dnsVerifier==null) throw new IllegalArgumentException("No dnsVerifier provided!"); - if (dnsResultCache ==null) throw new IllegalArgumentException("No cache provided!"); + public KnownHostBotVerifier build() { + if (dnsVerifier == null) throw new IllegalArgumentException("No dnsVerifier provided!"); + if (dnsResultCache == null) throw new IllegalArgumentException("No cache provided!"); return new KnownHostBotVerifierImpl(crawlerData, dnsVerifier, dnsResultCache); } From 8280a4ca24e3e2587a5fa6fbf5dccb08cc030608 Mon Sep 17 00:00:00 2001 From: Thomas Feiner Date: Thu, 18 May 2017 09:30:51 +0200 Subject: [PATCH 2/2] add "google.com" to HOSTNAMES according to https://support.google.com/webmasters/answer/80553 --- .../com/optimaize/webcrawlerverifier/bots/GooglebotData.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/optimaize/webcrawlerverifier/bots/GooglebotData.java b/src/main/java/com/optimaize/webcrawlerverifier/bots/GooglebotData.java index 3e30804..291df85 100644 --- a/src/main/java/com/optimaize/webcrawlerverifier/bots/GooglebotData.java +++ b/src/main/java/com/optimaize/webcrawlerverifier/bots/GooglebotData.java @@ -22,7 +22,7 @@ public boolean apply(String userAgent) { } }; - private static final ImmutableSet HOSTNAMES = ImmutableSet.of("googlebot.com"); + private static final ImmutableSet HOSTNAMES = ImmutableSet.of("googlebot.com", "google.com"); private static final GooglebotData INSTANCE = new GooglebotData();