diff --git a/pravtor.ru-crawler/pom.xml b/pravtor.ru-crawler/pom.xml index 006658d..3fff729 100644 --- a/pravtor.ru-crawler/pom.xml +++ b/pravtor.ru-crawler/pom.xml @@ -108,7 +108,7 @@ - by.andd3dfx.pravtor.SearchApp + by.andd3dfx.pravtor.MainApp diff --git a/pravtor.ru-crawler/src/main/java/by/andd3dfx/pravtor/SearchApp.java b/pravtor.ru-crawler/src/main/java/by/andd3dfx/pravtor/MainApp.java similarity index 82% rename from pravtor.ru-crawler/src/main/java/by/andd3dfx/pravtor/SearchApp.java rename to pravtor.ru-crawler/src/main/java/by/andd3dfx/pravtor/MainApp.java index c43976e..8a996a7 100644 --- a/pravtor.ru-crawler/src/main/java/by/andd3dfx/pravtor/SearchApp.java +++ b/pravtor.ru-crawler/src/main/java/by/andd3dfx/pravtor/MainApp.java @@ -1,7 +1,7 @@ package by.andd3dfx.pravtor; -import by.andd3dfx.pravtor.model.BatchSearchResult; -import by.andd3dfx.pravtor.model.TorrentData; +import by.andd3dfx.pravtor.dto.BatchSearchResult; +import by.andd3dfx.pravtor.dto.TorrentData; import by.andd3dfx.pravtor.util.FileUtil; import by.andd3dfx.pravtor.util.SearchUtil; @@ -9,9 +9,9 @@ import java.util.ArrayList; import java.util.Comparator; -public class SearchApp { +public class MainApp { - public static void main(String[] args) throws IOException, InterruptedException { + public static void main(String[] args) throws IOException { if (args.length != 2) { throw new IllegalArgumentException("Should be 2 parameters!"); } diff --git a/pravtor.ru-crawler/src/main/java/by/andd3dfx/pravtor/model/BatchSearchResult.java b/pravtor.ru-crawler/src/main/java/by/andd3dfx/pravtor/dto/BatchSearchResult.java similarity index 82% rename from pravtor.ru-crawler/src/main/java/by/andd3dfx/pravtor/model/BatchSearchResult.java rename to pravtor.ru-crawler/src/main/java/by/andd3dfx/pravtor/dto/BatchSearchResult.java index 8ecda9b..18e47b7 100644 --- a/pravtor.ru-crawler/src/main/java/by/andd3dfx/pravtor/model/BatchSearchResult.java +++ b/pravtor.ru-crawler/src/main/java/by/andd3dfx/pravtor/dto/BatchSearchResult.java @@ -1,4 +1,4 @@ -package by.andd3dfx.pravtor.model; +package by.andd3dfx.pravtor.dto; import java.util.List; import lombok.Getter; diff --git a/pravtor.ru-crawler/src/main/java/by/andd3dfx/pravtor/model/SearchCriteria.java b/pravtor.ru-crawler/src/main/java/by/andd3dfx/pravtor/dto/SearchCriteria.java similarity index 80% rename from pravtor.ru-crawler/src/main/java/by/andd3dfx/pravtor/model/SearchCriteria.java rename to pravtor.ru-crawler/src/main/java/by/andd3dfx/pravtor/dto/SearchCriteria.java index 35bab01..eed6e7f 100644 --- a/pravtor.ru-crawler/src/main/java/by/andd3dfx/pravtor/model/SearchCriteria.java +++ b/pravtor.ru-crawler/src/main/java/by/andd3dfx/pravtor/dto/SearchCriteria.java @@ -1,4 +1,4 @@ -package by.andd3dfx.pravtor.model; +package by.andd3dfx.pravtor.dto; import lombok.Getter; import lombok.RequiredArgsConstructor; diff --git a/pravtor.ru-crawler/src/main/java/by/andd3dfx/pravtor/model/TorrentData.java b/pravtor.ru-crawler/src/main/java/by/andd3dfx/pravtor/dto/TorrentData.java similarity index 86% rename from pravtor.ru-crawler/src/main/java/by/andd3dfx/pravtor/model/TorrentData.java rename to pravtor.ru-crawler/src/main/java/by/andd3dfx/pravtor/dto/TorrentData.java index f3acfcb..411a5b1 100644 --- a/pravtor.ru-crawler/src/main/java/by/andd3dfx/pravtor/model/TorrentData.java +++ b/pravtor.ru-crawler/src/main/java/by/andd3dfx/pravtor/dto/TorrentData.java @@ -1,4 +1,4 @@ -package by.andd3dfx.pravtor.model; +package by.andd3dfx.pravtor.dto; import by.andd3dfx.crawler.dto.CrawlerData; import lombok.Builder; diff --git a/pravtor.ru-crawler/src/main/java/by/andd3dfx/pravtor/util/FileUtil.java b/pravtor.ru-crawler/src/main/java/by/andd3dfx/pravtor/util/FileUtil.java index 8b096db..3979044 100644 --- a/pravtor.ru-crawler/src/main/java/by/andd3dfx/pravtor/util/FileUtil.java +++ b/pravtor.ru-crawler/src/main/java/by/andd3dfx/pravtor/util/FileUtil.java @@ -1,21 +1,19 @@ package by.andd3dfx.pravtor.util; -import by.andd3dfx.pravtor.model.BatchSearchResult; -import by.andd3dfx.pravtor.model.SearchCriteria; -import by.andd3dfx.pravtor.model.TorrentData; +import by.andd3dfx.pravtor.dto.BatchSearchResult; +import by.andd3dfx.pravtor.dto.SearchCriteria; +import by.andd3dfx.pravtor.dto.TorrentData; import org.apache.commons.lang3.StringUtils; import org.apache.poi.hssf.usermodel.HSSFWorkbook; import org.apache.poi.ss.usermodel.Cell; import org.apache.poi.ss.usermodel.Row; import org.apache.poi.ss.usermodel.Sheet; -import org.apache.poi.ss.usermodel.Workbook; import java.io.FileOutputStream; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Paths; import java.util.List; -import java.util.stream.Collectors; public class FileUtil { diff --git a/pravtor.ru-crawler/src/main/java/by/andd3dfx/pravtor/util/SearchUtil.java b/pravtor.ru-crawler/src/main/java/by/andd3dfx/pravtor/util/SearchUtil.java index 23d178f..f1ddd7a 100644 --- a/pravtor.ru-crawler/src/main/java/by/andd3dfx/pravtor/util/SearchUtil.java +++ b/pravtor.ru-crawler/src/main/java/by/andd3dfx/pravtor/util/SearchUtil.java @@ -1,7 +1,7 @@ package by.andd3dfx.pravtor.util; import by.andd3dfx.crawler.engine.WebCrawler; -import by.andd3dfx.pravtor.model.TorrentData; +import by.andd3dfx.pravtor.dto.TorrentData; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.jsoup.nodes.Document; diff --git a/pravtor.ru-crawler/src/test/java/by/andd3dfx/pravtor/SearchAppTest.java b/pravtor.ru-crawler/src/test/java/by/andd3dfx/pravtor/MainAppTest.java similarity index 88% rename from pravtor.ru-crawler/src/test/java/by/andd3dfx/pravtor/SearchAppTest.java rename to pravtor.ru-crawler/src/test/java/by/andd3dfx/pravtor/MainAppTest.java index 84bf3c2..5751491 100644 --- a/pravtor.ru-crawler/src/test/java/by/andd3dfx/pravtor/SearchAppTest.java +++ b/pravtor.ru-crawler/src/test/java/by/andd3dfx/pravtor/MainAppTest.java @@ -18,21 +18,21 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -public class SearchAppTest { +public class MainAppTest { private final String PARAMS_FILE = "src/test/resources/test-params.txt"; private final String RESULT_XLS_FILE = "target/tmp-result.xls"; - private SearchApp searchApp; + private MainApp mainApp; @Before public void setup() { - searchApp = new SearchApp(); + mainApp = new MainApp(); } @Test - public void testMain() throws IOException, InterruptedException { - searchApp.main(new String[]{PARAMS_FILE, RESULT_XLS_FILE}); + public void testMain() throws IOException { + mainApp.main(new String[]{PARAMS_FILE, RESULT_XLS_FILE}); // Check output XLS existence assertTrue(Files.exists(Path.of(RESULT_XLS_FILE))); @@ -71,7 +71,7 @@ public void testMainWithThreeParamsProvided() throws Exception { private void runMainNCheckExceptionThrow(String[] args) throws Exception { try { - searchApp.main(args); + mainApp.main(args); fail("Exception should be thrown"); } catch (IllegalArgumentException iae) { assertThat(iae.getMessage(), is("Should be 2 parameters!")); diff --git a/pravtor.ru-crawler/src/test/java/by/andd3dfx/pravtor/util/FileUtilTest.java b/pravtor.ru-crawler/src/test/java/by/andd3dfx/pravtor/util/FileUtilTest.java index 00484e0..bfef473 100644 --- a/pravtor.ru-crawler/src/test/java/by/andd3dfx/pravtor/util/FileUtilTest.java +++ b/pravtor.ru-crawler/src/test/java/by/andd3dfx/pravtor/util/FileUtilTest.java @@ -1,8 +1,8 @@ package by.andd3dfx.pravtor.util; -import by.andd3dfx.pravtor.model.BatchSearchResult; -import by.andd3dfx.pravtor.model.SearchCriteria; -import by.andd3dfx.pravtor.model.TorrentData; +import by.andd3dfx.pravtor.dto.BatchSearchResult; +import by.andd3dfx.pravtor.dto.SearchCriteria; +import by.andd3dfx.pravtor.dto.TorrentData; import org.dbunit.dataset.excel.XlsDataSet; import org.junit.Before; import org.junit.Test; diff --git a/pravtor.ru-crawler/src/test/java/by/andd3dfx/pravtor/util/SearchUtilTest.java b/pravtor.ru-crawler/src/test/java/by/andd3dfx/pravtor/util/SearchUtilTest.java index 6b93cbc..591ba00 100644 --- a/pravtor.ru-crawler/src/test/java/by/andd3dfx/pravtor/util/SearchUtilTest.java +++ b/pravtor.ru-crawler/src/test/java/by/andd3dfx/pravtor/util/SearchUtilTest.java @@ -1,10 +1,9 @@ package by.andd3dfx.pravtor.util; -import by.andd3dfx.pravtor.model.TorrentData; +import by.andd3dfx.pravtor.dto.TorrentData; import org.junit.Before; import org.junit.Test; -import java.io.IOException; import java.util.List; import static org.hamcrest.CoreMatchers.is; diff --git a/rabota.by-crawler/src/main/java/by/andd3dfx/sitesparsing/rabotaby/MainApp.java b/rabota.by-crawler/src/main/java/by/andd3dfx/rabotaby/MainApp.java similarity index 73% rename from rabota.by-crawler/src/main/java/by/andd3dfx/sitesparsing/rabotaby/MainApp.java rename to rabota.by-crawler/src/main/java/by/andd3dfx/rabotaby/MainApp.java index a89e23e..790f77c 100644 --- a/rabota.by-crawler/src/main/java/by/andd3dfx/sitesparsing/rabotaby/MainApp.java +++ b/rabota.by-crawler/src/main/java/by/andd3dfx/rabotaby/MainApp.java @@ -1,4 +1,7 @@ -package by.andd3dfx.sitesparsing.rabotaby; +package by.andd3dfx.rabotaby; + +import by.andd3dfx.rabotaby.util.SearchUtil; +import by.andd3dfx.rabotaby.util.StatisticsUtil; import java.io.IOException; import java.nio.file.Files; @@ -16,7 +19,7 @@ public static void main(String[] args) throws IOException { var pageUrl = searchUtil.buildSearchUrl("java"); var searchResult = searchUtil.batchSearch(pageUrl); - var statisticsSortedMap = new Statistics().collectStatistics(searchResult); + var statisticsSortedMap = new StatisticsUtil().collectStatistics(searchResult); Path path = Paths.get(args[0]); byte[] strToBytes = statisticsSortedMap.toString().getBytes(); diff --git a/rabota.by-crawler/src/main/java/by/andd3dfx/sitesparsing/rabotaby/dto/VacancyData.java b/rabota.by-crawler/src/main/java/by/andd3dfx/rabotaby/dto/VacancyData.java similarity index 87% rename from rabota.by-crawler/src/main/java/by/andd3dfx/sitesparsing/rabotaby/dto/VacancyData.java rename to rabota.by-crawler/src/main/java/by/andd3dfx/rabotaby/dto/VacancyData.java index f69c681..49a207a 100644 --- a/rabota.by-crawler/src/main/java/by/andd3dfx/sitesparsing/rabotaby/dto/VacancyData.java +++ b/rabota.by-crawler/src/main/java/by/andd3dfx/rabotaby/dto/VacancyData.java @@ -1,4 +1,4 @@ -package by.andd3dfx.sitesparsing.rabotaby.dto; +package by.andd3dfx.rabotaby.dto; import java.util.Set; diff --git a/rabota.by-crawler/src/main/java/by/andd3dfx/sitesparsing/rabotaby/SearchUtil.java b/rabota.by-crawler/src/main/java/by/andd3dfx/rabotaby/util/SearchUtil.java similarity index 93% rename from rabota.by-crawler/src/main/java/by/andd3dfx/sitesparsing/rabotaby/SearchUtil.java rename to rabota.by-crawler/src/main/java/by/andd3dfx/rabotaby/util/SearchUtil.java index 698b6d5..72972b8 100644 --- a/rabota.by-crawler/src/main/java/by/andd3dfx/sitesparsing/rabotaby/SearchUtil.java +++ b/rabota.by-crawler/src/main/java/by/andd3dfx/rabotaby/util/SearchUtil.java @@ -1,7 +1,7 @@ -package by.andd3dfx.sitesparsing.rabotaby; +package by.andd3dfx.rabotaby.util; import by.andd3dfx.crawler.engine.WebCrawler; -import by.andd3dfx.sitesparsing.rabotaby.dto.VacancyData; +import by.andd3dfx.rabotaby.dto.VacancyData; import lombok.extern.slf4j.Slf4j; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; diff --git a/rabota.by-crawler/src/main/java/by/andd3dfx/sitesparsing/rabotaby/Statistics.java b/rabota.by-crawler/src/main/java/by/andd3dfx/rabotaby/util/StatisticsUtil.java similarity index 88% rename from rabota.by-crawler/src/main/java/by/andd3dfx/sitesparsing/rabotaby/Statistics.java rename to rabota.by-crawler/src/main/java/by/andd3dfx/rabotaby/util/StatisticsUtil.java index 3a3facf..3ae06eb 100644 --- a/rabota.by-crawler/src/main/java/by/andd3dfx/sitesparsing/rabotaby/Statistics.java +++ b/rabota.by-crawler/src/main/java/by/andd3dfx/rabotaby/util/StatisticsUtil.java @@ -1,6 +1,6 @@ -package by.andd3dfx.sitesparsing.rabotaby; +package by.andd3dfx.rabotaby.util; -import by.andd3dfx.sitesparsing.rabotaby.dto.VacancyData; +import by.andd3dfx.rabotaby.dto.VacancyData; import java.util.Collections; import java.util.HashMap; @@ -10,7 +10,7 @@ import static java.util.stream.Collectors.toMap; -public class Statistics { +public class StatisticsUtil { private Map keywordToFreqMap = new HashMap<>(); diff --git a/rabota.by-crawler/src/main/java/by/andd3dfx/sitesparsing/rabotaby/dto/SingleSearchResult.java b/rabota.by-crawler/src/main/java/by/andd3dfx/sitesparsing/rabotaby/dto/SingleSearchResult.java deleted file mode 100644 index 8362da7..0000000 --- a/rabota.by-crawler/src/main/java/by/andd3dfx/sitesparsing/rabotaby/dto/SingleSearchResult.java +++ /dev/null @@ -1,14 +0,0 @@ -package by.andd3dfx.sitesparsing.rabotaby.dto; - -import lombok.AllArgsConstructor; -import lombok.Getter; - -import java.util.List; - -@Getter -@AllArgsConstructor -public class SingleSearchResult { - - private List dataItems; - private String nextPageUrl; -} diff --git a/rabota.by-crawler/src/test/java/by/andd3dfx/sitesparsing/rabotaby/SearchUtilTest.java b/rabota.by-crawler/src/test/java/by/andd3dfx/rabotaby/util/SearchUtilTest.java similarity index 87% rename from rabota.by-crawler/src/test/java/by/andd3dfx/sitesparsing/rabotaby/SearchUtilTest.java rename to rabota.by-crawler/src/test/java/by/andd3dfx/rabotaby/util/SearchUtilTest.java index 591e68f..a576ea4 100644 --- a/rabota.by-crawler/src/test/java/by/andd3dfx/sitesparsing/rabotaby/SearchUtilTest.java +++ b/rabota.by-crawler/src/test/java/by/andd3dfx/rabotaby/util/SearchUtilTest.java @@ -1,4 +1,4 @@ -package by.andd3dfx.sitesparsing.rabotaby; +package by.andd3dfx.rabotaby.util; import org.junit.Before; import org.junit.Test; @@ -11,12 +11,12 @@ public class SearchUtilTest { private static final int RECORDS_PER_PAGE = 20; private SearchUtil searchUtil; - private Statistics statistics; + private StatisticsUtil statisticsUtil; @Before public void setup() { searchUtil = new SearchUtil(); - statistics = new Statistics(); + statisticsUtil = new StatisticsUtil(); } @Test diff --git a/rabota.by-crawler/src/test/java/by/andd3dfx/sitesparsing/rabotaby/StatisticsTest.java b/rabota.by-crawler/src/test/java/by/andd3dfx/rabotaby/util/StatisticsUtilTest.java similarity index 56% rename from rabota.by-crawler/src/test/java/by/andd3dfx/sitesparsing/rabotaby/StatisticsTest.java rename to rabota.by-crawler/src/test/java/by/andd3dfx/rabotaby/util/StatisticsUtilTest.java index 3998d81..fe29715 100644 --- a/rabota.by-crawler/src/test/java/by/andd3dfx/sitesparsing/rabotaby/StatisticsTest.java +++ b/rabota.by-crawler/src/test/java/by/andd3dfx/rabotaby/util/StatisticsUtilTest.java @@ -1,4 +1,4 @@ -package by.andd3dfx.sitesparsing.rabotaby; +package by.andd3dfx.rabotaby.util; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.nullValue; @@ -8,33 +8,34 @@ import java.util.LinkedHashMap; import java.util.Map.Entry; import java.util.Set; + import org.junit.Before; import org.junit.Test; -public class StatisticsTest { +public class StatisticsUtilTest { - private Statistics statistics; + private StatisticsUtil statisticsUtil; @Before public void setUp() { - statistics = new Statistics(); + statisticsUtil = new StatisticsUtil(); } @Test public void testStatisticsCollector() { - statistics.putKeyword("Java"); - statistics.putKeyword("Spring"); - statistics.putKeyword("Java"); - statistics.putKeyword("Spring"); - statistics.putKeyword("SQL"); - statistics.putKeyword("Spring"); - - assertThat("Wrong amount of Java items", statistics.get("Java"), is(2)); - assertThat("Wrong amount of SQL items", statistics.get("SQL"), is(1)); - assertThat("Wrong amount of Spring items", statistics.get("Spring"), is(3)); - assertThat("Wrong amount of absent item", statistics.get("EJB"), is(nullValue())); - - final LinkedHashMap map = statistics.buildSortedMap(); + statisticsUtil.putKeyword("Java"); + statisticsUtil.putKeyword("Spring"); + statisticsUtil.putKeyword("Java"); + statisticsUtil.putKeyword("Spring"); + statisticsUtil.putKeyword("SQL"); + statisticsUtil.putKeyword("Spring"); + + assertThat("Wrong amount of Java items", statisticsUtil.get("Java"), is(2)); + assertThat("Wrong amount of SQL items", statisticsUtil.get("SQL"), is(1)); + assertThat("Wrong amount of Spring items", statisticsUtil.get("Spring"), is(3)); + assertThat("Wrong amount of absent item", statisticsUtil.get("EJB"), is(nullValue())); + + final LinkedHashMap map = statisticsUtil.buildSortedMap(); final Set> entries = map.entrySet(); final Iterator> iterator = entries.iterator();