From 34483e6813d363e12962828f5d81493b5fad5751 Mon Sep 17 00:00:00 2001 From: cooki Date: Fri, 22 Nov 2024 11:11:53 -0500 Subject: [PATCH] All readers switched (#187) --- .../ayasiterecode/AyaSiteRecodeUtils.java | 29 ++++----- .../com/imsweb/algorithms/AlgorithmsTest.java | 21 +++---- .../causespecific/CauseSpecificUtilsTest.java | 46 +++++++------- .../algorithms/napiia/NapiiaUtilsTest.java | 63 +++++++++---------- .../imsweb/algorithms/nhia/NhiaUtilsTest.java | 46 +++++++------- .../TumorSizeOverTimeUtilsTest.java | 42 +++++-------- .../java/lab/SeerSiteRecodeParserLab.java | 41 ++++++------ src/test/java/lab/TractDataLab.java | 62 +++++++++--------- 8 files changed, 161 insertions(+), 189 deletions(-) diff --git a/src/main/java/com/imsweb/algorithms/ayasiterecode/AyaSiteRecodeUtils.java b/src/main/java/com/imsweb/algorithms/ayasiterecode/AyaSiteRecodeUtils.java index 9c61383..f668f5b 100644 --- a/src/main/java/com/imsweb/algorithms/ayasiterecode/AyaSiteRecodeUtils.java +++ b/src/main/java/com/imsweb/algorithms/ayasiterecode/AyaSiteRecodeUtils.java @@ -3,20 +3,16 @@ */ package com.imsweb.algorithms.ayasiterecode; +import java.io.File; import java.io.IOException; import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.Reader; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; import org.apache.commons.lang3.StringUtils; -import com.opencsv.CSVParserBuilder; -import com.opencsv.CSVReader; -import com.opencsv.CSVReaderBuilder; -import com.opencsv.exceptions.CsvException; +import de.siegmar.fastcsv.reader.CsvReader; +import de.siegmar.fastcsv.reader.CsvRecord; public final class AyaSiteRecodeUtils { @@ -94,19 +90,20 @@ private static List readData(String filename) { boolean txt = filename.endsWith(".txt"); - try (Reader reader = new InputStreamReader(is, StandardCharsets.UTF_8); - CSVReader csvReader = new CSVReaderBuilder(reader).withCSVParser(new CSVParserBuilder().withSeparator(txt ? ';' : ',').build()).withSkipLines(txt ? 2 : 1).build()) { - for (String[] row : csvReader.readAll()) { - String beh = StringUtils.trimToNull(row[txt ? 1 : 3]); - String site = StringUtils.trimToNull(row[2]); - String hist = StringUtils.trimToNull(row[txt ? 3 : 1]); - String recode = StringUtils.trimToNull(row[4]); + File csvFile = new File("src/main/resources/ayasiterecode/" + filename); + try (CsvReader reader = CsvReader.builder().fieldSeparator(txt ? ';' : ',').ofCsvRecord(csvFile.toPath())) { + reader.skipLines(txt ? 2 : 1); + reader.stream().forEach(line -> { + String beh = StringUtils.trimToNull(line.getField(txt ? 1 : 3)); + String site = StringUtils.trimToNull(line.getField(2)); + String hist = StringUtils.trimToNull(line.getField(txt ? 3 : 1)); + String recode = StringUtils.trimToNull(line.getField(4)); if (beh != null && site != null && hist != null && recode != null) result.add(new AyaSiteRecodeData(site, hist, beh, StringUtils.leftPad(recode, txt ? 2 : 3, "0"))); - } + }); } } - catch (CsvException | IOException e) { + catch (IOException e) { throw new IllegalStateException("Unable to read " + filename, e); } return result; diff --git a/src/test/java/com/imsweb/algorithms/AlgorithmsTest.java b/src/test/java/com/imsweb/algorithms/AlgorithmsTest.java index bfaddd8..93b39d0 100644 --- a/src/test/java/com/imsweb/algorithms/AlgorithmsTest.java +++ b/src/test/java/com/imsweb/algorithms/AlgorithmsTest.java @@ -6,8 +6,6 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; -import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.util.Collections; import java.util.HashMap; @@ -21,9 +19,8 @@ import org.junit.BeforeClass; import org.junit.Test; -import com.opencsv.CSVReader; -import com.opencsv.CSVReaderBuilder; -import com.opencsv.exceptions.CsvException; +import de.siegmar.fastcsv.reader.CsvReader; +import de.siegmar.fastcsv.reader.NamedCsvRecord; import com.imsweb.algorithms.countyatdiagnosisanalysis.CountyAtDxAnalysisUtils; import com.imsweb.algorithms.internal.Utils; @@ -57,7 +54,7 @@ private static String getBuildProperty(String property) { } @Test - public void testFields() throws IOException, CsvException { + public void testFields() throws IOException { NaaccrDictionary dictionary = NaaccrXmlDictionaryUtils.getMergedDictionaries(NaaccrFormat.NAACCR_VERSION_230); Set ids = new HashSet<>(); @@ -93,12 +90,12 @@ public void testFields() throws IOException, CsvException { if (!field.isNaaccrStandard()) algNonStandardItems.put(field.getId(), field.getNumber()); - try (CSVReader reader = new CSVReaderBuilder(new InputStreamReader(Files.newInputStream(nonStandardItemsFile.toPath()), StandardCharsets.US_ASCII)).withSkipLines(1).build()) { - for (String[] row : reader.readAll()) { - Integer algNum = algNonStandardItems.get(row[0]); - if (algNum != null && algNum.equals(Integer.valueOf(row[1]))) - Assert.fail("Item '" + row[0] + "' has number " + algNum + " in this library, but " + row[1] + " in the submission dictionaries"); - } + try (CsvReader reader = CsvReader.builder().ofNamedCsvRecord(nonStandardItemsFile.toPath())) { + reader.stream().forEach(line -> { + Integer algNum = algNonStandardItems.get(line.getField(0)); + if (algNum != null && algNum.equals(Integer.valueOf(line.getField(1)))) + Assert.fail("Item '" + line.getField(0) + "' has number " + algNum + " in this library, but " + line.getField(1) + " in the submission dictionaries"); + }); } } diff --git a/src/test/java/com/imsweb/algorithms/causespecific/CauseSpecificUtilsTest.java b/src/test/java/com/imsweb/algorithms/causespecific/CauseSpecificUtilsTest.java index 0dabebd..ba202c1 100644 --- a/src/test/java/com/imsweb/algorithms/causespecific/CauseSpecificUtilsTest.java +++ b/src/test/java/com/imsweb/algorithms/causespecific/CauseSpecificUtilsTest.java @@ -3,18 +3,16 @@ */ package com.imsweb.algorithms.causespecific; +import java.io.File; import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; import java.util.Arrays; +import java.util.concurrent.atomic.AtomicInteger; import org.junit.Assert; import org.junit.Test; -import com.opencsv.CSVReader; -import com.opencsv.CSVReaderBuilder; -import com.opencsv.exceptions.CsvException; +import de.siegmar.fastcsv.reader.CsvReader; +import de.siegmar.fastcsv.reader.NamedCsvRecord; public class CauseSpecificUtilsTest { @@ -35,31 +33,31 @@ public void testComputeCauseSpecific() { @Test @SuppressWarnings("ConstantConditions") - public void testCsvFile() throws IOException, CsvException { - int count = 0; - try (InputStream is = Thread.currentThread().getContextClassLoader().getResourceAsStream("causespecific/testCauseSpecific.csv"); - CSVReader reader = new CSVReaderBuilder(new InputStreamReader(is, StandardCharsets.US_ASCII)).withSkipLines(1).build()) { - for (String[] row : reader.readAll()) { + public void testCsvFile() throws IOException { + AtomicInteger count = new AtomicInteger(); + + File csvFile = new File("src/test/resources/causespecific/testCauseSpecific.csv"); + try (CsvReader reader = CsvReader.builder().ofNamedCsvRecord(csvFile.toPath())) { + reader.stream().forEach(line -> { CauseSpecificInputDto input = new CauseSpecificInputDto(); - input.setSequenceNumberCentral(row[0]); - input.setIcdRevisionNumber(row[1]); - input.setCauseOfDeath(row[2]); - input.setPrimarySite(row[3]); - input.setHistologyIcdO3(row[4]); - input.setDateOfLastContactYear(row[5]); + input.setSequenceNumberCentral(line.getField(0)); + input.setIcdRevisionNumber(line.getField(1)); + input.setCauseOfDeath(line.getField(2)); + input.setPrimarySite(line.getField(3)); + input.setHistologyIcdO3(line.getField(4)); + input.setDateOfLastContactYear(line.getField(5)); - String causeSpecificExpected = row[7]; - String causeOtherExpected = row[8]; + String causeSpecificExpected = line.getField(7); + String causeOtherExpected = line.getField(8); String causeSpecificCalculated = CauseSpecificUtils.computeCauseSpecific(input).getCauseSpecificDeathClassification(); String causeOtherCalculated = CauseSpecificUtils.computeCauseSpecific(input).getCauseOtherDeathClassification(); - count++; + count.getAndIncrement(); if (!causeSpecificExpected.equals(causeSpecificCalculated) || !causeOtherExpected.equals(causeOtherCalculated)) { - //System.out.println(SeerSiteRecodeUtils.calculateSiteRecode(row[3], row[4])); - Assert.fail("Unexpected result for row number " + (count + 1) + " " + Arrays.asList(row) + "\nExpected results: " + causeSpecificExpected + ", " + - "" + causeOtherExpected + " But found: " + causeSpecificCalculated + ", " + causeOtherCalculated); + Assert.fail("Unexpected result for row number " + (count.get() + 1) + " " + Arrays.asList(line.getFields().toArray(new String[0])) + "\nExpected results: " + causeSpecificExpected + ", " + + causeOtherExpected + " But found: " + causeSpecificCalculated + ", " + causeOtherCalculated); } - } + }); } } diff --git a/src/test/java/com/imsweb/algorithms/napiia/NapiiaUtilsTest.java b/src/test/java/com/imsweb/algorithms/napiia/NapiiaUtilsTest.java index 17a2875..ed73a93 100644 --- a/src/test/java/com/imsweb/algorithms/napiia/NapiiaUtilsTest.java +++ b/src/test/java/com/imsweb/algorithms/napiia/NapiiaUtilsTest.java @@ -3,21 +3,20 @@ */ package com.imsweb.algorithms.napiia; +import java.io.File; import java.io.IOException; -import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; import org.junit.Assert; import org.junit.Test; -import com.opencsv.CSVReader; -import com.opencsv.CSVReaderBuilder; -import com.opencsv.exceptions.CsvException; +import de.siegmar.fastcsv.reader.CsvReader; +import de.siegmar.fastcsv.reader.NamedCsvRecord; public class NapiiaUtilsTest { @@ -852,42 +851,40 @@ public void testComputeNapiia() { @SuppressWarnings("ConstantConditions") @Test - public void testCsvFile() throws IOException, CsvException { - int line = 0; - try (CSVReader reader = new CSVReaderBuilder( - new InputStreamReader(Thread.currentThread().getContextClassLoader().getResourceAsStream("napiia/testNAPIIA.csv"), StandardCharsets.US_ASCII)).withSkipLines(1).build()) { - for (String[] row : reader.readAll()) { - line++; + public void testCsvFile() throws IOException { + AtomicInteger count = new AtomicInteger(); - //if (line != 10) - // continue; + + File csvFile = new File("src/test/resources/napiia/testNAPIIA.csv"); + try (CsvReader reader = CsvReader.builder().ofNamedCsvRecord(csvFile.toPath())) { + reader.stream().forEach(line -> { + count.getAndIncrement(); Map rec = new HashMap<>(); - rec.put(_PROP_RACE1, row[0].trim().isEmpty() ? null : row[0]); - rec.put(_PROP_RACE2, row[1].trim().isEmpty() ? null : row[1]); - rec.put(_PROP_RACE3, row[2].trim().isEmpty() ? null : row[2]); - rec.put(_PROP_RACE4, row[3].trim().isEmpty() ? null : row[3]); - rec.put(_PROP_RACE5, row[4].trim().isEmpty() ? null : row[4]); - rec.put(_PROP_SPANISH_HISPANIC_ORIGIN, row[5].trim().isEmpty() ? null : row[5]); - rec.put(_PROP_BIRTH_PLACE_COUNTRY, row[6].trim().isEmpty() ? null : row[6]); - rec.put(_PROP_SEX, row[7].trim().isEmpty() ? null : row[7]); - rec.put(_PROP_NAME_LAST, row[8].trim().isEmpty() ? null : row[8]); - rec.put(_PROP_NAME_BIRTH_SURNAME, row[9].trim().isEmpty() ? null : row[9]); - rec.put(_PROP_NAME_FIRST, row[10].trim().isEmpty() ? null : row[10]); - - String napiia = row[11]; - Boolean review = Boolean.valueOf(row[12]); - String reason = row[13].trim().isEmpty() ? null : row[13]; + rec.put(_PROP_RACE1, line.getField(0).trim().isEmpty() ? null : line.getField(0)); + rec.put(_PROP_RACE2, line.getField(1).trim().isEmpty() ? null : line.getField(1)); + rec.put(_PROP_RACE3, line.getField(2).trim().isEmpty() ? null : line.getField(2)); + rec.put(_PROP_RACE4, line.getField(3).trim().isEmpty() ? null : line.getField(3)); + rec.put(_PROP_RACE5, line.getField(4).trim().isEmpty() ? null : line.getField(4)); + rec.put(_PROP_SPANISH_HISPANIC_ORIGIN, line.getField(5).trim().isEmpty() ? null : line.getField(5)); + rec.put(_PROP_BIRTH_PLACE_COUNTRY, line.getField(6).trim().isEmpty() ? null : line.getField(6)); + rec.put(_PROP_SEX, line.getField(7).trim().isEmpty() ? null : line.getField(7)); + rec.put(_PROP_NAME_LAST, line.getField(8).trim().isEmpty() ? null : line.getField(8)); + rec.put(_PROP_NAME_BIRTH_SURNAME, line.getField(9).trim().isEmpty() ? null : line.getField(9)); + rec.put(_PROP_NAME_FIRST, line.getField(10).trim().isEmpty() ? null : line.getField(10)); + + String napiia = line.getField(11); + Boolean review = Boolean.valueOf(line.getField(12)); + String reason = line.getField(13).trim().isEmpty() ? null : line.getField(13); NapiiaResultsDto results = computeNapiia(rec); if (!napiia.equals(results.getNapiiaValue())) - Assert.fail("Unexpected napiia result in CSV data file for row #" + line + " " + Arrays.asList(row) + " " + results.getNapiiaValue()); + Assert.fail("Unexpected napiia result in CSV data file for row #" + count + " " + Arrays.asList(line.getFields().toArray(new String[0])) + " " + results.getNapiiaValue()); if (!review.equals(results.getNeedsHumanReview())) - Assert.fail("Unexpected needs manual review result in CSV data file for row #" + line + " " + Arrays.asList(row) + " " + results.getNeedsHumanReview()); + Assert.fail("Unexpected needs manual review result in CSV data file for row #" + count + " " + Arrays.asList(line.getFields().toArray(new String[0])) + " " + results.getNeedsHumanReview()); if (results.getReasonForReview() == null ? reason != null : !results.getReasonForReview().equals(reason)) - Assert.fail("Unexpected reason for review result in CSV data file for row #" + line + " " + Arrays.asList(row) + " " + results.getReasonForReview()); - } - //System.out.println(line + " cases tested!"); + Assert.fail("Unexpected reason for review result in CSV data file for row #" + count + " " + Arrays.asList(line.getFields().toArray(new String[0])) + " " + results.getReasonForReview()); + }); } } diff --git a/src/test/java/com/imsweb/algorithms/nhia/NhiaUtilsTest.java b/src/test/java/com/imsweb/algorithms/nhia/NhiaUtilsTest.java index 4912fa8..120fe1d 100644 --- a/src/test/java/com/imsweb/algorithms/nhia/NhiaUtilsTest.java +++ b/src/test/java/com/imsweb/algorithms/nhia/NhiaUtilsTest.java @@ -3,9 +3,8 @@ */ package com.imsweb.algorithms.nhia; +import java.io.File; import java.io.IOException; -import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -15,9 +14,8 @@ import org.junit.Assert; import org.junit.Test; -import com.opencsv.CSVReader; -import com.opencsv.CSVReaderBuilder; -import com.opencsv.exceptions.CsvException; +import de.siegmar.fastcsv.reader.CsvReader; +import de.siegmar.fastcsv.reader.NamedCsvRecord; public class NhiaUtilsTest { @@ -345,34 +343,32 @@ public void testPatientSet() { @SuppressWarnings("ConstantConditions") @Test - public void testCsvFile() throws IOException, CsvException { - try (CSVReader reader = new CSVReaderBuilder( - new InputStreamReader(Thread.currentThread().getContextClassLoader().getResourceAsStream("nhia/testNHIA.csv"), StandardCharsets.US_ASCII)).withSkipLines(1).build()) { - for (String[] row : reader.readAll()) { + public void testCsvFile() throws IOException { + File csvFile = new File("src/test/resources/nhia/testNHIA.csv"); + try (CsvReader reader = CsvReader.builder().ofNamedCsvRecord(csvFile.toPath())) { + reader.stream().forEach(line -> { Map rec = new HashMap<>(); - rec.put(_PROP_SPANISH_HISPANIC_ORIGIN, row[0]); - rec.put(_PROP_BIRTH_PLACE_COUNTRY, row[1]); - rec.put(_PROP_RACE1, row[2]); - rec.put(_PROP_IHS, row[3]); - rec.put(_PROP_STATE_DX, row[4]); - rec.put(_PROP_COUNTY_DX_ANALYSIS, row[5]); - rec.put(_PROP_SEX, row[6]); - rec.put(_PROP_NAME_LAST, row[7]); - rec.put(_PROP_NAME_BIRTH_SURNAME, row[8]); - - String option = row[9]; - String nhia = row[10]; + rec.put(_PROP_SPANISH_HISPANIC_ORIGIN, line.getField(0)); + rec.put(_PROP_BIRTH_PLACE_COUNTRY, line.getField(1)); + rec.put(_PROP_RACE1, line.getField(2)); + rec.put(_PROP_IHS, line.getField(3)); + rec.put(_PROP_STATE_DX, line.getField(4)); + rec.put(_PROP_COUNTY_DX_ANALYSIS, line.getField(5)); + rec.put(_PROP_SEX, line.getField(6)); + rec.put(_PROP_NAME_LAST, line.getField(7)); + rec.put(_PROP_NAME_BIRTH_SURNAME, line.getField(8)); + + String option = line.getField(9); + String nhia = line.getField(10); if (!nhia.equals(computeNhia(rec, option).getNhia())) - Assert.fail("Unexpected result in CSV data file for row " + Arrays.asList(row)); - } + Assert.fail("Unexpected result in CSV data file for row " + Arrays.asList(line.getFields().toArray(new String[0]))); + }); } } @Test public void testGetLowHispanicCountiesPerState() { - //for (Map.Entry> entry : NhiaUtils.getLowHispanicCountiesPerState().entrySet()) - // System.out.println(entry.getKey() + ": " + entry.getValue()); Assert.assertFalse(NhiaUtils.getLowHispanicCountiesPerState().isEmpty()); } diff --git a/src/test/java/com/imsweb/algorithms/tumorsizeovertime/TumorSizeOverTimeUtilsTest.java b/src/test/java/com/imsweb/algorithms/tumorsizeovertime/TumorSizeOverTimeUtilsTest.java index 416fca0..2b1df66 100644 --- a/src/test/java/com/imsweb/algorithms/tumorsizeovertime/TumorSizeOverTimeUtilsTest.java +++ b/src/test/java/com/imsweb/algorithms/tumorsizeovertime/TumorSizeOverTimeUtilsTest.java @@ -3,19 +3,16 @@ */ package com.imsweb.algorithms.tumorsizeovertime; +import java.io.File; import java.io.IOException; -import java.io.InputStreamReader; -import java.io.LineNumberReader; -import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Objects; import org.junit.Assert; import org.junit.Test; -import com.opencsv.CSVReader; -import com.opencsv.CSVReaderBuilder; -import com.opencsv.exceptions.CsvException; +import de.siegmar.fastcsv.reader.CsvReader; +import de.siegmar.fastcsv.reader.NamedCsvRecord; public class TumorSizeOverTimeUtilsTest { @@ -177,32 +174,27 @@ public void testIsValidTumorSize() { } @Test - @SuppressWarnings("DataFlowIssue") - public void testCsvFile() throws IOException, CsvException { - try (LineNumberReader lineNumberReader = new LineNumberReader(new InputStreamReader( - Thread.currentThread().getContextClassLoader().getResourceAsStream("tumorsizeovertime/tumorsize.test.data.csv"), StandardCharsets.US_ASCII)); - CSVReader reader = new CSVReaderBuilder(lineNumberReader).withSkipLines(1).build()) { - String[] row = reader.readNext(); - while (row != null) { + public void testCsvFile() throws IOException { + File csvFile = new File("src/test/resources/tumorsizeovertime/tumorsize.test.data.csv"); + try (CsvReader reader = CsvReader.builder().ofNamedCsvRecord(csvFile.toPath())) { + reader.stream().forEach(line -> { TumorSizeOverTimeInputDto input = new TumorSizeOverTimeInputDto(); - input.setDxYear(row[0]); - input.setSite(row[1]); - input.setHist(row[2]); - input.setBehavior(row[3]); - input.setEodTumorSize(row[4]); - input.setCsTumorSize(row[5]); - input.setTumorSizeSummary(row[6]); + input.setDxYear(line.getField(0)); + input.setSite(line.getField(1)); + input.setHist(line.getField(2)); + input.setBehavior(line.getField(3)); + input.setEodTumorSize(line.getField(4)); + input.setCsTumorSize(line.getField(5)); + input.setTumorSizeSummary(line.getField(6)); - String expectedResult = row[7]; + String expectedResult = line.getField(7); if ("XX2".equals(expectedResult)) expectedResult = null; String result = TumorSizeOverTimeUtils.computeTumorSizeOverTime(input); if (!Objects.equals(expectedResult, result)) - Assert.fail("Line " + lineNumberReader.getLineNumber() + " - Unexpected result in CSV data file for row " + Arrays.asList(row) + " vs " + result); - - row = reader.readNext(); - } + Assert.fail("Line " + line.getStartingLineNumber() + " - Unexpected result in CSV data file for row " + Arrays.asList(line.getFields().toArray(new String[0])) + " vs " + result); + }); } } } diff --git a/src/test/java/lab/SeerSiteRecodeParserLab.java b/src/test/java/lab/SeerSiteRecodeParserLab.java index 2d226a9..2888a0e 100644 --- a/src/test/java/lab/SeerSiteRecodeParserLab.java +++ b/src/test/java/lab/SeerSiteRecodeParserLab.java @@ -5,17 +5,15 @@ import java.io.File; import java.io.IOException; -import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.commons.lang3.StringUtils; -import com.opencsv.CSVReader; -import com.opencsv.exceptions.CsvException; +import de.siegmar.fastcsv.reader.CsvReader; +import de.siegmar.fastcsv.reader.NamedCsvRecord; /** * Lab class was used to parse data from these locations: @@ -25,33 +23,31 @@ */ public class SeerSiteRecodeParserLab { - public static void main(String[] args) throws IOException, CsvException { + public static void main(String[] args) throws IOException { File input = new File(""); List> newLines = new ArrayList<>(); newLines.add(Arrays.asList("ID", "Name", "Indentation", "Sites IN", "Hist IN", "Hist OUT", "Beh IN", "DX Year Min", "DX Year Max", "Recode")); - int count = 0; - try (CSVReader reader = new CSVReader(new InputStreamReader(Files.newInputStream(input.toPath()), StandardCharsets.US_ASCII))) { - String[] line = reader.readNext(); - while (line != null) { - if (line.length != 6) - throw new IllegalStateException("!!! " + Arrays.toString(line)); + AtomicInteger count = new AtomicInteger(); + try (CsvReader reader = CsvReader.builder().ofNamedCsvRecord(input.toPath())) { + reader.stream().forEach(line -> { + if (line.getFieldCount() != 6) + throw new IllegalStateException("!!! " + Arrays.toString(line.getFields().toArray(new String[0]))); - if ("Site Group".equals(line[0])) { - line = reader.readNext(); - continue; + if ("Site Group".equals(line.getField(0))) { + return; } - String name = line[0].trim(); - String site = line[1].trim(); - String hist = line[2].trim(); - String beh = line[3].trim(); - String recode = line[4].trim(); + String name = line.getField(0).trim(); + String site = line.getField(1).trim(); + String hist = line.getField(2).trim(); + String beh = line.getField(3).trim(); + String recode = line.getField(4).trim(); List newLine = new ArrayList<>(); - newLine.add(String.valueOf(++count)); + newLine.add(String.valueOf(count.incrementAndGet())); newLine.add(name); newLine.add(StringUtils.isEmpty(site) ? "0" : "1"); newLine.add(site.replace(" ", "")); @@ -118,8 +114,7 @@ else if (StringUtils.isNotBlank(newLine.get(histInIdx)) && newLine.get(histInIdx if (newExtraLine != null) newLines.add(newExtraLine); - line = reader.readNext(); - } + }); } List lines = new ArrayList<>(); diff --git a/src/test/java/lab/TractDataLab.java b/src/test/java/lab/TractDataLab.java index 47cb69d..f8c7fea 100644 --- a/src/test/java/lab/TractDataLab.java +++ b/src/test/java/lab/TractDataLab.java @@ -4,12 +4,12 @@ package lab; import java.io.BufferedWriter; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.LineNumberReader; import java.io.OutputStreamWriter; -import java.io.Reader; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; @@ -29,9 +29,8 @@ import org.apache.commons.lang3.StringUtils; -import com.opencsv.CSVReader; -import com.opencsv.CSVReaderBuilder; -import com.opencsv.exceptions.CsvException; +import de.siegmar.fastcsv.reader.CsvReader; +import de.siegmar.fastcsv.reader.NamedCsvRecord; import com.imsweb.algorithms.internal.CountryData; import com.imsweb.layout.LayoutUtils; @@ -144,23 +143,24 @@ public static void main(String[] args) throws IOException { // NAACCR Poverty Indicator 1995-2004 Map naaccrPovertyIndicator9504 = new HashMap<>(); - try (Reader reader = new InputStreamReader(Thread.currentThread().getContextClassLoader().getResourceAsStream("povertyindicator/poverty-indicator-1995-2004.csv"), StandardCharsets.US_ASCII); - CSVReader csvReader = new CSVReaderBuilder(reader).withSkipLines(1).build()) { - for (String[] row : csvReader.readAll()) - naaccrPovertyIndicator9504.put(new DataKey(row[0], row[1], row[2]), row[3]); + + File csvFile = new File("src/test/resources/povertyindicator/poverty-indicator-1995-2004.csv"); + try (CsvReader reader = CsvReader.builder().ofNamedCsvRecord(csvFile.toPath())) { + reader.stream().forEach(line -> naaccrPovertyIndicator9504.put(new DataKey(line.getField(0), line.getField(1), line.getField(2)), line.getField(3))); } - catch (CsvException | IOException e) { + catch (IOException e) { throw new IllegalStateException(e); } // NAACCR Poverty Indicator 2005-2007 Map naaccrPovertyIndicator0507 = new HashMap<>(); - try (Reader reader = new InputStreamReader(Thread.currentThread().getContextClassLoader().getResourceAsStream("povertyindicator/poverty-indicator-2005-2007.csv"), StandardCharsets.US_ASCII); - CSVReader csvReader = new CSVReaderBuilder(reader).withSkipLines(1).build()) { - for (String[] row : csvReader.readAll()) - naaccrPovertyIndicator0507.put(new DataKey(row[0], row[1], row[2]), row[3]); + + + csvFile = new File("src/test/resources/povertyindicator/poverty-indicator-2005-2007.csv"); + try (CsvReader reader = CsvReader.builder().ofNamedCsvRecord(csvFile.toPath())) { + reader.stream().forEach(line -> naaccrPovertyIndicator0507.put(new DataKey(line.getField(0), line.getField(1), line.getField(2)), line.getField(3))); } - catch (CsvException | IOException e) { + catch (IOException e) { throw new IllegalStateException(e); } @@ -169,14 +169,13 @@ public static void main(String[] args) throws IOException { try (InputStream is = Thread.currentThread().getContextClassLoader().getResourceAsStream("ruralurban/rural-urban-commuting-area-2000.csv")) { if (is == null) throw new IllegalStateException("Missing data file!"); - try (Reader reader = new InputStreamReader(is, StandardCharsets.US_ASCII); - CSVReader csvReader = new CSVReaderBuilder(reader).withSkipLines(1).build()) { + csvFile = new File("src/test/resources/ruralurban/rural-urban-commuting-area-2000.csv"); + try (CsvReader reader = CsvReader.builder().ofNamedCsvRecord(csvFile.toPath())) { List urbanCommutingAreas = Arrays.asList("1.0", "1.1", "2.0", "2.1", "3.0", "4.1", "5.1", "7.1", "8.1", "10.1"); - - for (String[] row : csvReader.readAll()) { - String primary = row[3]; - String secondary = row[4]; + reader.stream().forEach(line -> { + String primary = line.getField(3); + String secondary = line.getField(4); String val; if (primary.equals("99")) @@ -186,11 +185,11 @@ else if (urbanCommutingAreas.contains(secondary)) else val = "2"; - ruca2000Values.put(new DataKey(row[0], row[1], row[2]), val); - } + ruca2000Values.put(new DataKey(line.getField(0), line.getField(1), line.getField(2)), val); + }); } } - catch (CsvException | IOException e) { + catch (IOException e) { throw new IllegalStateException(e); } @@ -199,16 +198,17 @@ else if (urbanCommutingAreas.contains(secondary)) try (InputStream is = Thread.currentThread().getContextClassLoader().getResourceAsStream("ruralurban/urban-rural-indicator-code-2000.csv")) { if (is == null) throw new IllegalStateException("Missing data file!"); - try (Reader reader = new InputStreamReader(is, StandardCharsets.US_ASCII); - CSVReader csvReader = new CSVReaderBuilder(reader).withSkipLines(1).build()) { - for (String[] row : csvReader.readAll()) { - if (row[4].length() != 1) - throw new IllegalStateException("Found unexpected format for URIC value: " + row[4]); - uric2000Values.put(new DataKey(row[0], row[1], row[2]), row[4]); - } + + csvFile = new File("src/test/resources/ruralurban/urban-rural-indicator-code-2000.csv"); + try (CsvReader reader = CsvReader.builder().ofNamedCsvRecord(csvFile.toPath())) { + reader.stream().forEach(line -> { + if (line.getField(4).length() != 1) + throw new IllegalStateException("Found unexpected format for URIC value: " + line.getField(4)); + uric2000Values.put(new DataKey(line.getField(0), line.getField(1), line.getField(2)), line.getField(4)); + }); } } - catch (CsvException | IOException e) { + catch (IOException e) { throw new IllegalStateException(e); }