From cac4a2e310671eaaee482e3ba7415ba45539e12d Mon Sep 17 00:00:00 2001 From: komp15 <77535280+komp15@users.noreply.github.com> Date: Mon, 28 Nov 2022 19:43:50 +0100 Subject: [PATCH 1/5] :sparkles: Introduced support for news pages for faculties websites --- pom.xml | 7 +- src/main/java/dev/wms/pwrapi/api/NewsAPI.java | 32 ++++++++ .../wms/pwrapi/dao/news/GeneralNewsDAO.java | 79 +++++++++++++++++++ .../java/dev/wms/pwrapi/dto/news/Channel.java | 21 +++++ .../dev/wms/pwrapi/dto/news/FacultyType.java | 31 ++++++++ .../java/dev/wms/pwrapi/dto/news/Item.java | 21 +++++ .../java/dev/wms/pwrapi/dto/news/Rss.java | 14 ++++ .../wms/pwrapi/service/news/NewsService.java | 26 ++++++ .../utils/config/DeserializationConfig.java | 29 +++++++ .../GeneralAdvice.java | 1 + .../dev/wms/pwrapi/utils/http/HttpUtils.java | 17 +++- src/main/resources/application.properties | 3 +- .../wms/pwrapi/parking/ParkingDAOTest.java | 2 +- 13 files changed, 279 insertions(+), 4 deletions(-) create mode 100644 src/main/java/dev/wms/pwrapi/api/NewsAPI.java create mode 100644 src/main/java/dev/wms/pwrapi/dao/news/GeneralNewsDAO.java create mode 100644 src/main/java/dev/wms/pwrapi/dto/news/Channel.java create mode 100644 src/main/java/dev/wms/pwrapi/dto/news/FacultyType.java create mode 100644 src/main/java/dev/wms/pwrapi/dto/news/Item.java create mode 100644 src/main/java/dev/wms/pwrapi/dto/news/Rss.java create mode 100644 src/main/java/dev/wms/pwrapi/service/news/NewsService.java create mode 100644 src/main/java/dev/wms/pwrapi/utils/config/DeserializationConfig.java diff --git a/pom.xml b/pom.xml index ba9e27c..48ab24d 100644 --- a/pom.xml +++ b/pom.xml @@ -16,7 +16,12 @@ 11 - + + + com.fasterxml.jackson.dataformat + jackson-dataformat-xml + 2.14.1 + org.springframework.boot spring-boot-starter-web diff --git a/src/main/java/dev/wms/pwrapi/api/NewsAPI.java b/src/main/java/dev/wms/pwrapi/api/NewsAPI.java new file mode 100644 index 0000000..463864c --- /dev/null +++ b/src/main/java/dev/wms/pwrapi/api/NewsAPI.java @@ -0,0 +1,32 @@ +package dev.wms.pwrapi.api; + +import dev.wms.pwrapi.dto.news.Channel; +import dev.wms.pwrapi.dto.news.FacultyType; +import dev.wms.pwrapi.service.news.NewsService; +import lombok.AllArgsConstructor; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + + +@RestController +@RequestMapping(value = "/api/news", produces = "application/json") +@AllArgsConstructor +public class NewsAPI { + + private NewsService newsService; + + @GetMapping("/general") + public ResponseEntity fetchGeneralNews(){ + return ResponseEntity.ok(newsService.fetchGeneralNews()); + } + + @GetMapping("/faculty") + public ResponseEntity fetchFacultyNews( + @RequestParam FacultyType faculty){ + return ResponseEntity.ok(newsService.fetchNewsForFaculty(faculty)); + } + +} diff --git a/src/main/java/dev/wms/pwrapi/dao/news/GeneralNewsDAO.java b/src/main/java/dev/wms/pwrapi/dao/news/GeneralNewsDAO.java new file mode 100644 index 0000000..4dbe908 --- /dev/null +++ b/src/main/java/dev/wms/pwrapi/dao/news/GeneralNewsDAO.java @@ -0,0 +1,79 @@ +package dev.wms.pwrapi.dao.news; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.dataformat.xml.XmlMapper; +import dev.wms.pwrapi.dto.news.*; +import dev.wms.pwrapi.utils.http.HttpUtils; +import okhttp3.OkHttpClient; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; +import org.springframework.stereotype.Repository; + +import java.util.List; +import java.util.stream.Collectors; + +@Repository +public class GeneralNewsDAO { + + public Channel parsePwrRSS(String rssUrl) { + OkHttpClient client = new OkHttpClient(); + String response = HttpUtils.makeRequestWithClientAndGetString(client, rssUrl); + XmlMapper xmlMapper = new XmlMapper(); + xmlMapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, true); + xmlMapper.configure(DeserializationFeature.ACCEPT_EMPTY_ARRAY_AS_NULL_OBJECT, true); + xmlMapper.enable(DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT); + xmlMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + + try { + Rss items = xmlMapper.readValue(response, Rss.class); + return items.getChannel(); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + public Channel getFacultyNews(FacultyType faculty) { + if(faculty.isRss){ + return parsePwrRSS(faculty.url); + } else { + return parsePwrHTML(faculty.url); + } + } + + private Channel parsePwrHTML(String url) { + OkHttpClient client = new OkHttpClient(); + Document document = HttpUtils.makeRequestWithClientAndGetDocument(client, url); + + Elements newsBoxes = document.getElementsByClass("news-box"); + + List channelItems = newsBoxes.parallelStream() + .map(newsBox -> parseItem(newsBox, url)) + .toList(); + + + return Channel.builder() + .title(document.getElementsByClass("portal-title").first().text()) + .link(url) + .description("") + .item(channelItems) + .build(); + } + + private Item parseItem(Element element, String url){ + Element textDiv = element.getElementsByClass("col-text").first(); + return Item.builder() + .title(textDiv.getElementsByClass("title").first().attr("title")) + .link(getDomainFromNewsUrl(url) + textDiv.getElementsByClass("title").first().attr("href")) + .pubDate(textDiv.getElementsByClass("date").text().replace("Data: ","").strip() + .split("Kategoria:")[0].strip()) + .description(textDiv.getElementsByTag("p").get(1).text().replace("... więcej","") + .replace("więcej","").strip()) + .build(); + } + + private String getDomainFromNewsUrl(String url){ + return url.split(".pl")[0] + ".pl"; + } +} diff --git a/src/main/java/dev/wms/pwrapi/dto/news/Channel.java b/src/main/java/dev/wms/pwrapi/dto/news/Channel.java new file mode 100644 index 0000000..928d310 --- /dev/null +++ b/src/main/java/dev/wms/pwrapi/dto/news/Channel.java @@ -0,0 +1,21 @@ +package dev.wms.pwrapi.dto.news; + +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor +public class Channel { + public String title; + public String link; + public String description; + @JacksonXmlElementWrapper(useWrapping = false) + public List item; +} diff --git a/src/main/java/dev/wms/pwrapi/dto/news/FacultyType.java b/src/main/java/dev/wms/pwrapi/dto/news/FacultyType.java new file mode 100644 index 0000000..d77dcab --- /dev/null +++ b/src/main/java/dev/wms/pwrapi/dto/news/FacultyType.java @@ -0,0 +1,31 @@ +package dev.wms.pwrapi.dto.news; + +public enum FacultyType { + + INFORMATYKI_I_TELEKOMUNIKACJI("https://wit.pwr.edu.pl/rss/pl/189.xml", true), + ZARZADZANIA("https://wz.pwr.edu.pl/rss/pl/127.xml", true), + ELEKTRONIKI_MIKROSYSTEMOW_I_FOTONIKI("https://wefim.pwr.edu.pl/rss/pl/5.xml", true), + ARCHITEKTURY("https://wa.pwr.edu.pl/o-wydziale/aktualnosci", false), + BUDOWNICTWA_LADOWEGO_I_WODNEGO("https://wbliw.pwr.edu.pl/o-wydziale/aktualnosci", false), + CHEMICZNY("https://wch.pwr.edu.pl/o-wydziale/aktualnosci", false), + ELEKTRYCZNY("https://weny.pwr.edu.pl/o-wydziale/aktualnosci", false), + GEOINZYNIERII_GORNICTWA_I_GEOLOGII("https://wggg.pwr.edu.pl/o-wydziale/aktualnosci", false), + INZYNIERII_SRODOWISKA("https://wis.pwr.edu.pl/o-wydziale/aktualnosci", false), + MECHANICZNO_ENERGETYCZNY("https://wme.pwr.edu.pl/aktualnosci", false), + MECHANICZNY("https://wm.pwr.edu.pl/o-wydziale/aktualnosci", false), + PODSTAWOWYCH_PROBLEMOW_TECHNIKI("https://wppt.pwr.edu.pl/o-wydziale/aktualnosci", false), + MATEMATYKI("https://wmat.pwr.edu.pl/o-wydziale/aktualnosci", false), + FILIA_POLITECHNIKI_WROCLAWSKIEJ_W_JELENIEJ_GORZE("https://jelenia-gora.pwr.edu.pl/o-filii/aktualnosci", false), + FILIA_POLITECHNIKI_WROCLAWSKIEJ_W_WALBRZYCHU("https://walbrzych.pwr.edu.pl/o-filii/aktualnosci", false), + FILIA_POLITECHNIKI_WROCLAWSKIEJ_W_LEGNICY("https://legnica.pwr.edu.pl/o-wydziale/aktualnosci", false); + + + + public final String url; + public final boolean isRss; + + FacultyType(String url, boolean isRss) { + this.url = url; + this.isRss = isRss; + } +} diff --git a/src/main/java/dev/wms/pwrapi/dto/news/Item.java b/src/main/java/dev/wms/pwrapi/dto/news/Item.java new file mode 100644 index 0000000..14a2cbd --- /dev/null +++ b/src/main/java/dev/wms/pwrapi/dto/news/Item.java @@ -0,0 +1,21 @@ +package dev.wms.pwrapi.dto.news; + +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.extern.jackson.Jacksonized; + +@Data +@Jacksonized +@Builder +@AllArgsConstructor +@NoArgsConstructor +public class Item { + public String title; + public String link; + public String pubDate; + public String description; +} + diff --git a/src/main/java/dev/wms/pwrapi/dto/news/Rss.java b/src/main/java/dev/wms/pwrapi/dto/news/Rss.java new file mode 100644 index 0000000..2823ed9 --- /dev/null +++ b/src/main/java/dev/wms/pwrapi/dto/news/Rss.java @@ -0,0 +1,14 @@ +package dev.wms.pwrapi.dto.news; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class Rss { + public Channel channel; + public double version; + public String text; +} diff --git a/src/main/java/dev/wms/pwrapi/service/news/NewsService.java b/src/main/java/dev/wms/pwrapi/service/news/NewsService.java new file mode 100644 index 0000000..8001b79 --- /dev/null +++ b/src/main/java/dev/wms/pwrapi/service/news/NewsService.java @@ -0,0 +1,26 @@ +package dev.wms.pwrapi.service.news; + +import dev.wms.pwrapi.dao.news.GeneralNewsDAO; +import dev.wms.pwrapi.dto.news.Channel; +import dev.wms.pwrapi.dto.news.FacultyType; +import dev.wms.pwrapi.dto.news.Item; +import lombok.AllArgsConstructor; +import org.springframework.stereotype.Service; + +import java.util.Set; + +@Service +@AllArgsConstructor +public class NewsService { + + private GeneralNewsDAO generalNewsDAO; + + + public Channel fetchGeneralNews() { + return generalNewsDAO.parsePwrRSS("https://pwr.edu.pl/rss/pl/24.xml"); + } + + public Channel fetchNewsForFaculty(FacultyType faculty) { + return generalNewsDAO.getFacultyNews(faculty); + } +} diff --git a/src/main/java/dev/wms/pwrapi/utils/config/DeserializationConfig.java b/src/main/java/dev/wms/pwrapi/utils/config/DeserializationConfig.java new file mode 100644 index 0000000..b836b5a --- /dev/null +++ b/src/main/java/dev/wms/pwrapi/utils/config/DeserializationConfig.java @@ -0,0 +1,29 @@ +package dev.wms.pwrapi.utils.config; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.http.MediaType; +import org.springframework.http.converter.json.Jackson2ObjectMapperBuilder; +import org.springframework.web.servlet.config.annotation.ContentNegotiationConfigurer; +import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; + +@Configuration +public class DeserializationConfig { + + @Bean + Jackson2ObjectMapperBuilder getDefaultDeserializer(){ + return Jackson2ObjectMapperBuilder.json(); + } + + @Bean + public WebMvcConfigurer customWebMvcConfigurer(){ + return new WebMvcConfigurer() { + @Override + public void configureContentNegotiation(ContentNegotiationConfigurer configurer) { + configurer.defaultContentType(MediaType.APPLICATION_JSON); + } + }; + } + +} diff --git a/src/main/java/dev/wms/pwrapi/utils/generalExceptionHandling/GeneralAdvice.java b/src/main/java/dev/wms/pwrapi/utils/generalExceptionHandling/GeneralAdvice.java index 1cb90c2..a7620d3 100644 --- a/src/main/java/dev/wms/pwrapi/utils/generalExceptionHandling/GeneralAdvice.java +++ b/src/main/java/dev/wms/pwrapi/utils/generalExceptionHandling/GeneralAdvice.java @@ -32,6 +32,7 @@ public class GeneralAdvice { @Order(Ordered.LOWEST_PRECEDENCE) public ResponseEntity handleGeneralException(Throwable t){ log.info("Handling unexpected exception " + t); + t.printStackTrace(); return ResponseEntity.status(500).body(new ExceptionMessagingDTO(t.getMessage())); } diff --git a/src/main/java/dev/wms/pwrapi/utils/http/HttpUtils.java b/src/main/java/dev/wms/pwrapi/utils/http/HttpUtils.java index 01f8b7f..96fee87 100644 --- a/src/main/java/dev/wms/pwrapi/utils/http/HttpUtils.java +++ b/src/main/java/dev/wms/pwrapi/utils/http/HttpUtils.java @@ -20,6 +20,19 @@ public static String makeRequestWithClientAndGetString(OkHttpClient client, Requ } } + public static String makeRequestWithClientAndGetString(OkHttpClient client, String url){ + + Request request = new Request.Builder() + .url(url) + .build(); + + try(Response response = client.newCall(request).execute()){ + return response.body().string(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + /** * Makes request with OkHttp's client and parses it to Jsoup's Document. Needed for proper response closing * @param client OkHttp client which will execute the request @@ -27,7 +40,7 @@ public static String makeRequestWithClientAndGetString(OkHttpClient client, Requ * @return Jsoup's Document containing parsed html from OkHttp response * @throws IOException when parsing goes wrong */ - public static Document makeRequestWithClientAndGetDocument(OkHttpClient client, String url) throws IOException { + public static Document makeRequestWithClientAndGetDocument(OkHttpClient client, String url) { Request financeRequest = new Request.Builder() .url(url) @@ -39,6 +52,8 @@ public static Document makeRequestWithClientAndGetDocument(OkHttpClient client, responseString = response.body().string(); } catch (SocketTimeoutException e){ throw new SystemTimeoutException(); + } catch (IOException e) { + throw new RuntimeException(e); } return Jsoup.parse(responseString); diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index f4874f4..a9eb758 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -1,3 +1,4 @@ logging.file.path=./pwr-api-logs logging.file.name=pwr-api.log -logging.level.org.springframework.web.filter.CommonsRequestLoggingFilter=DEBUG \ No newline at end of file +logging.level.org.springframework.web.filter.CommonsRequestLoggingFilter=DEBUG +spring.data.rest.default-media-type=application/json \ No newline at end of file diff --git a/src/test/java/dev/wms/pwrapi/parking/ParkingDAOTest.java b/src/test/java/dev/wms/pwrapi/parking/ParkingDAOTest.java index dad611d..38713d6 100644 --- a/src/test/java/dev/wms/pwrapi/parking/ParkingDAOTest.java +++ b/src/test/java/dev/wms/pwrapi/parking/ParkingDAOTest.java @@ -8,7 +8,7 @@ import org.springframework.boot.test.context.SpringBootTest; import java.io.IOException; -import java.util.List; +import java.util.*; import static org.junit.jupiter.api.Assertions.assertEquals; From fa5ccac5bd27d8b2f881f39cdf9c74cb6ebcf695 Mon Sep 17 00:00:00 2001 From: komp15 <77535280+komp15@users.noreply.github.com> Date: Tue, 29 Nov 2022 00:38:47 +0100 Subject: [PATCH 2/5] :art: Improved date formatting for RSS parsed news --- .../wms/pwrapi/dao/news/GeneralNewsDAO.java | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/main/java/dev/wms/pwrapi/dao/news/GeneralNewsDAO.java b/src/main/java/dev/wms/pwrapi/dao/news/GeneralNewsDAO.java index 4dbe908..b1bc473 100644 --- a/src/main/java/dev/wms/pwrapi/dao/news/GeneralNewsDAO.java +++ b/src/main/java/dev/wms/pwrapi/dao/news/GeneralNewsDAO.java @@ -11,12 +11,21 @@ import org.jsoup.select.Elements; import org.springframework.stereotype.Repository; +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; import java.util.List; +import java.util.Locale; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import java.util.stream.Collectors; @Repository public class GeneralNewsDAO { + private final DateTimeFormatter rssFormatter = DateTimeFormatter.ofPattern("dd MMM yyyy", Locale.ENGLISH); + private final DateTimeFormatter goalFormatter = DateTimeFormatter.ofPattern("dd.MM.yyyy"); + private final Pattern datePattern = Pattern.compile("\\d{2} [a-zA-Z]{3} \\d{4}"); + public Channel parsePwrRSS(String rssUrl) { OkHttpClient client = new OkHttpClient(); String response = HttpUtils.makeRequestWithClientAndGetString(client, rssUrl); @@ -28,12 +37,21 @@ public Channel parsePwrRSS(String rssUrl) { try { Rss items = xmlMapper.readValue(response, Rss.class); + for(Item item : items.getChannel().getItem()) reformatDate(item); + return items.getChannel(); } catch (JsonProcessingException e) { throw new RuntimeException(e); } } + private void reformatDate(Item item){ + Matcher matcher = datePattern.matcher(item.getPubDate()); + matcher.find(); + LocalDate parsedDate = LocalDate.parse(matcher.group(), rssFormatter); + item.setPubDate(parsedDate.format(goalFormatter)); + } + public Channel getFacultyNews(FacultyType faculty) { if(faculty.isRss){ return parsePwrRSS(faculty.url); @@ -47,6 +65,7 @@ private Channel parsePwrHTML(String url) { Document document = HttpUtils.makeRequestWithClientAndGetDocument(client, url); Elements newsBoxes = document.getElementsByClass("news-box"); + newsBoxes.removeIf(box -> box.text().isEmpty()); List channelItems = newsBoxes.parallelStream() .map(newsBox -> parseItem(newsBox, url)) From 1950e084b53a92ba4a273e90282b5e1d1bb36702 Mon Sep 17 00:00:00 2001 From: komp15 <77535280+komp15@users.noreply.github.com> Date: Tue, 29 Nov 2022 11:25:23 +0100 Subject: [PATCH 3/5] :sparkles: Added caching for news fetching --- pom.xml | 4 +++ .../wms/pwrapi/service/news/NewsService.java | 6 +++- .../pwrapi/utils/config/CachingConfig.java | 35 +++++++++++++++++++ src/main/resources/application.properties | 4 ++- 4 files changed, 47 insertions(+), 2 deletions(-) create mode 100644 src/main/java/dev/wms/pwrapi/utils/config/CachingConfig.java diff --git a/pom.xml b/pom.xml index 48ab24d..f12dc2b 100644 --- a/pom.xml +++ b/pom.xml @@ -17,6 +17,10 @@ 11 + + org.springframework.boot + spring-boot-starter-cache + com.fasterxml.jackson.dataformat jackson-dataformat-xml diff --git a/src/main/java/dev/wms/pwrapi/service/news/NewsService.java b/src/main/java/dev/wms/pwrapi/service/news/NewsService.java index 8001b79..c7d0d4d 100644 --- a/src/main/java/dev/wms/pwrapi/service/news/NewsService.java +++ b/src/main/java/dev/wms/pwrapi/service/news/NewsService.java @@ -5,6 +5,7 @@ import dev.wms.pwrapi.dto.news.FacultyType; import dev.wms.pwrapi.dto.news.Item; import lombok.AllArgsConstructor; +import org.springframework.cache.annotation.Cacheable; import org.springframework.stereotype.Service; import java.util.Set; @@ -15,12 +16,15 @@ public class NewsService { private GeneralNewsDAO generalNewsDAO; - + @Cacheable("pwr-news") public Channel fetchGeneralNews() { + System.out.println("FETCHING NEW NEWSES"); return generalNewsDAO.parsePwrRSS("https://pwr.edu.pl/rss/pl/24.xml"); } + @Cacheable("pwr-news") public Channel fetchNewsForFaculty(FacultyType faculty) { + System.out.println("FETCHING NEW NEWSES"); return generalNewsDAO.getFacultyNews(faculty); } } diff --git a/src/main/java/dev/wms/pwrapi/utils/config/CachingConfig.java b/src/main/java/dev/wms/pwrapi/utils/config/CachingConfig.java new file mode 100644 index 0000000..c9ddafa --- /dev/null +++ b/src/main/java/dev/wms/pwrapi/utils/config/CachingConfig.java @@ -0,0 +1,35 @@ +package dev.wms.pwrapi.utils.config; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.cache.CacheManagerCustomizer; +import org.springframework.cache.CacheManager; +import org.springframework.cache.annotation.CacheEvict; +import org.springframework.cache.annotation.EnableCaching; +import org.springframework.cache.concurrent.ConcurrentMapCacheManager; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.env.ConfigurableEnvironment; +import org.springframework.scheduling.annotation.EnableScheduling; +import org.springframework.scheduling.annotation.Scheduled; + +import javax.annotation.PostConstruct; +import java.util.List; + +@Configuration +@EnableCaching +@EnableScheduling +@Slf4j +public class CachingConfig implements CacheManagerCustomizer { + + @Override + public void customize(ConcurrentMapCacheManager cacheManager) { + cacheManager.setCacheNames(List.of("pwr-news")); + } + + @CacheEvict(allEntries = true, cacheNames = "pwr-news") + @Scheduled(fixedDelayString = "${pwr-api.news.cacheTTL}") + public void reportCacheEvict(){ + } + +} diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index a9eb758..820b4af 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -1,4 +1,6 @@ logging.file.path=./pwr-api-logs logging.file.name=pwr-api.log logging.level.org.springframework.web.filter.CommonsRequestLoggingFilter=DEBUG -spring.data.rest.default-media-type=application/json \ No newline at end of file +spring.data.rest.default-media-type=application/json + +pwr-api.news.cacheTTL=900000 \ No newline at end of file From 34968e68e011d900bcc794b1cd77de0bcd8ab65b Mon Sep 17 00:00:00 2001 From: komp15 <77535280+komp15@users.noreply.github.com> Date: Tue, 29 Nov 2022 12:00:30 +0100 Subject: [PATCH 4/5] :white_check_mark: Added news API tests, improved class naming --- .../{GeneralNewsDAO.java => NewsDAO.java} | 3 +- .../wms/pwrapi/service/news/NewsService.java | 11 ++--- .../service/news/NewsServiceCachingTest.java | 43 +++++++++++++++++++ .../pwrapi/service/news/NewsServiceTest.java | 29 +++++++++++++ 4 files changed, 77 insertions(+), 9 deletions(-) rename src/main/java/dev/wms/pwrapi/dao/news/{GeneralNewsDAO.java => NewsDAO.java} (98%) create mode 100644 src/test/java/dev/wms/pwrapi/service/news/NewsServiceCachingTest.java create mode 100644 src/test/java/dev/wms/pwrapi/service/news/NewsServiceTest.java diff --git a/src/main/java/dev/wms/pwrapi/dao/news/GeneralNewsDAO.java b/src/main/java/dev/wms/pwrapi/dao/news/NewsDAO.java similarity index 98% rename from src/main/java/dev/wms/pwrapi/dao/news/GeneralNewsDAO.java rename to src/main/java/dev/wms/pwrapi/dao/news/NewsDAO.java index b1bc473..a885f0f 100644 --- a/src/main/java/dev/wms/pwrapi/dao/news/GeneralNewsDAO.java +++ b/src/main/java/dev/wms/pwrapi/dao/news/NewsDAO.java @@ -17,10 +17,9 @@ import java.util.Locale; import java.util.regex.Matcher; import java.util.regex.Pattern; -import java.util.stream.Collectors; @Repository -public class GeneralNewsDAO { +public class NewsDAO { private final DateTimeFormatter rssFormatter = DateTimeFormatter.ofPattern("dd MMM yyyy", Locale.ENGLISH); private final DateTimeFormatter goalFormatter = DateTimeFormatter.ofPattern("dd.MM.yyyy"); diff --git a/src/main/java/dev/wms/pwrapi/service/news/NewsService.java b/src/main/java/dev/wms/pwrapi/service/news/NewsService.java index c7d0d4d..6a6fd6b 100644 --- a/src/main/java/dev/wms/pwrapi/service/news/NewsService.java +++ b/src/main/java/dev/wms/pwrapi/service/news/NewsService.java @@ -1,30 +1,27 @@ package dev.wms.pwrapi.service.news; -import dev.wms.pwrapi.dao.news.GeneralNewsDAO; +import dev.wms.pwrapi.dao.news.NewsDAO; import dev.wms.pwrapi.dto.news.Channel; import dev.wms.pwrapi.dto.news.FacultyType; -import dev.wms.pwrapi.dto.news.Item; import lombok.AllArgsConstructor; import org.springframework.cache.annotation.Cacheable; import org.springframework.stereotype.Service; -import java.util.Set; - @Service @AllArgsConstructor public class NewsService { - private GeneralNewsDAO generalNewsDAO; + private NewsDAO newsDAO; @Cacheable("pwr-news") public Channel fetchGeneralNews() { System.out.println("FETCHING NEW NEWSES"); - return generalNewsDAO.parsePwrRSS("https://pwr.edu.pl/rss/pl/24.xml"); + return newsDAO.parsePwrRSS("https://pwr.edu.pl/rss/pl/24.xml"); } @Cacheable("pwr-news") public Channel fetchNewsForFaculty(FacultyType faculty) { System.out.println("FETCHING NEW NEWSES"); - return generalNewsDAO.getFacultyNews(faculty); + return newsDAO.getFacultyNews(faculty); } } diff --git a/src/test/java/dev/wms/pwrapi/service/news/NewsServiceCachingTest.java b/src/test/java/dev/wms/pwrapi/service/news/NewsServiceCachingTest.java new file mode 100644 index 0000000..bde33b6 --- /dev/null +++ b/src/test/java/dev/wms/pwrapi/service/news/NewsServiceCachingTest.java @@ -0,0 +1,43 @@ +package dev.wms.pwrapi.service.news; + +import dev.wms.pwrapi.dao.news.NewsDAO; +import dev.wms.pwrapi.dto.news.FacultyType; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +@SpringBootTest +public class NewsServiceCachingTest { + + @Autowired + private NewsService newsService; + @MockBean + private NewsDAO newsDAO; + + @Test + public void newsDaoShouldBeCalledOnceForMultipleCallsInTTL(){ + for(int i = 0; i < 50; i++) newsService.fetchGeneralNews(); + verify(newsDAO, times(1)) + .parsePwrRSS(any()); + } + + @Test + public void newsDaoShouldBeCalledTwiceForTwoSeparateFacultiesInTTL(){ + for(int i = 0; i < 50; i++){ + newsService.fetchNewsForFaculty(FacultyType.ARCHITEKTURY); + newsService.fetchNewsForFaculty(FacultyType.INFORMATYKI_I_TELEKOMUNIKACJI); + } + + verify(newsDAO, times(1)) + .getFacultyNews(FacultyType.ARCHITEKTURY); + + verify(newsDAO, times(1)) + .getFacultyNews(FacultyType.INFORMATYKI_I_TELEKOMUNIKACJI); + } + +} diff --git a/src/test/java/dev/wms/pwrapi/service/news/NewsServiceTest.java b/src/test/java/dev/wms/pwrapi/service/news/NewsServiceTest.java new file mode 100644 index 0000000..6fb3e6f --- /dev/null +++ b/src/test/java/dev/wms/pwrapi/service/news/NewsServiceTest.java @@ -0,0 +1,29 @@ +package dev.wms.pwrapi.service.news; + +import dev.wms.pwrapi.dto.news.FacultyType; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@SpringBootTest +public class NewsServiceTest { + + @Autowired + private NewsService newsService; + + @Test + public void newsServiceFetchShouldBeNotEmptyForEveryFaculty(){ + for (FacultyType faculty : FacultyType.values()) { + assertFalse(newsService.fetchNewsForFaculty(faculty).getItem().isEmpty()); + } + } + + @Test + public void newsServiceShouldFetchGeneralNews(){ + assertFalse(newsService.fetchGeneralNews().getItem().isEmpty()); + } + +} From 329b97804bb772b703d462321dab6919ce060e94 Mon Sep 17 00:00:00 2001 From: komp15 <77535280+komp15@users.noreply.github.com> Date: Tue, 29 Nov 2022 22:33:24 +0100 Subject: [PATCH 5/5] :memo: Added docs for NewsAPI --- src/main/java/dev/wms/pwrapi/api/NewsAPI.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/main/java/dev/wms/pwrapi/api/NewsAPI.java b/src/main/java/dev/wms/pwrapi/api/NewsAPI.java index 463864c..7876843 100644 --- a/src/main/java/dev/wms/pwrapi/api/NewsAPI.java +++ b/src/main/java/dev/wms/pwrapi/api/NewsAPI.java @@ -3,6 +3,7 @@ import dev.wms.pwrapi.dto.news.Channel; import dev.wms.pwrapi.dto.news.FacultyType; import dev.wms.pwrapi.service.news.NewsService; +import io.swagger.v3.oas.annotations.Operation; import lombok.AllArgsConstructor; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.GetMapping; @@ -19,11 +20,22 @@ public class NewsAPI { private NewsService newsService; @GetMapping("/general") + @Operation(summary = "Returns recent news from main PWr website", + description = "This endpoint implementation is based on RSS scraping, transforming and caching. " + + "RSS is taken from https://pwr.edu.pl/rss/pl/24.xml and which, as we observed, is a copy of news available on " + + "https://pwr.edu.pl/uczelnia/aktualnosci The data is cached for 15 minutes from last call, so recent news can be displayed with maximum 15 " + + "minutes delay.") public ResponseEntity fetchGeneralNews(){ return ResponseEntity.ok(newsService.fetchGeneralNews()); } @GetMapping("/faculty") + @Operation(summary = "Returns recent news from faculty website", + description = "Implementation of this endpoint is based on RSS scraping or HTTP scraping. If the website supports RSS, " + + "rss is used, if not, the traditional HTTP scraping and HTML parsing is used. Detailed info about website and " + + "used method for certain faculties can be viewed here https://github.com/komp15/PWr-API/blob/feature_newsAPI/src/main/java/dev/wms/pwrapi/dto/news/FacultyType.java " + + "The data is cached for 15 minutes from last call for all faculties, so recent news can be displayed with maximum 15 " + + "minutes delay.") public ResponseEntity fetchFacultyNews( @RequestParam FacultyType faculty){ return ResponseEntity.ok(newsService.fetchNewsForFaculty(faculty));