diff --git a/src/main/java/com/optimaize/webcrawlerverifier/bots/GooglebotData.java b/src/main/java/com/optimaize/webcrawlerverifier/bots/GooglebotData.java index 3e30804..291df85 100644 --- a/src/main/java/com/optimaize/webcrawlerverifier/bots/GooglebotData.java +++ b/src/main/java/com/optimaize/webcrawlerverifier/bots/GooglebotData.java @@ -22,7 +22,7 @@ public boolean apply(String userAgent) { } }; - private static final ImmutableSet HOSTNAMES = ImmutableSet.of("googlebot.com"); + private static final ImmutableSet HOSTNAMES = ImmutableSet.of("googlebot.com", "google.com"); private static final GooglebotData INSTANCE = new GooglebotData(); diff --git a/src/main/java/com/optimaize/webcrawlerverifier/bots/KnownHostBotVerifierBuilder.java b/src/main/java/com/optimaize/webcrawlerverifier/bots/KnownHostBotVerifierBuilder.java index 4dd74da..f04b27c 100644 --- a/src/main/java/com/optimaize/webcrawlerverifier/bots/KnownHostBotVerifierBuilder.java +++ b/src/main/java/com/optimaize/webcrawlerverifier/bots/KnownHostBotVerifierBuilder.java @@ -19,7 +19,7 @@ public class KnownHostBotVerifierBuilder { public KnownHostBotVerifierBuilder crawlerData(@NotNull CrawlerData crawlerData) { - if (this.crawlerData!=null) throw new IllegalStateException("The crawlerData was set already!"); + if (this.crawlerData != null) throw new IllegalStateException("The crawlerData was set already!"); this.crawlerData = crawlerData; return this; } @@ -27,10 +27,11 @@ public KnownHostBotVerifierBuilder crawlerData(@NotNull CrawlerData crawlerData) /** */ public KnownHostBotVerifierBuilder dnsVerifier(@NotNull ReverseDnsVerifier dnsVerifier) { - if (this.dnsVerifier!=null) throw new IllegalStateException("The dnsVerifier was set already!"); + if (this.dnsVerifier != null) throw new IllegalStateException("The dnsVerifier was set already!"); this.dnsVerifier = dnsVerifier; return this; } + /** * Uses the {@link DnsjavaReverseDnsVerifier} with the default name server(s) provided by the system. */ @@ -43,26 +44,27 @@ public KnownHostBotVerifierBuilder dnsVerifierDefault() { * If you must, for testing, then pass in a dummy cache that drops all. */ public KnownHostBotVerifierBuilder dnsResultCache(@NotNull Cache dnsResultCache) { - if (this.dnsResultCache !=null) throw new IllegalStateException("The dnsResultCache was set already!"); + if (this.dnsResultCache != null) throw new IllegalStateException("The dnsResultCache was set already!"); this.dnsResultCache = dnsResultCache; return this; } + /** * Uses maximumSize(1_000) and expireAfterWrite(3*24, TimeUnit.HOURS) */ public KnownHostBotVerifierBuilder dnsResultCacheDefault() { Cache cache = CacheBuilder.newBuilder() .maximumSize(1_000) - .expireAfterWrite(3*24, TimeUnit.HOURS) + .expireAfterWrite(3 * 24, TimeUnit.HOURS) .build(); return dnsResultCache(cache); } @NotNull - public KnownHostBotVerifierImpl build() { - if (dnsVerifier==null) throw new IllegalArgumentException("No dnsVerifier provided!"); - if (dnsResultCache ==null) throw new IllegalArgumentException("No cache provided!"); + public KnownHostBotVerifier build() { + if (dnsVerifier == null) throw new IllegalArgumentException("No dnsVerifier provided!"); + if (dnsResultCache == null) throw new IllegalArgumentException("No cache provided!"); return new KnownHostBotVerifierImpl(crawlerData, dnsVerifier, dnsResultCache); }