Skip to content

Commit

Permalink
All readers switched (#187)
Browse files Browse the repository at this point in the history
  • Loading branch information
@CookiIMS
CookiIMS committed Nov 22, 2024
1 parent ea93502 commit 34483e6
Show file tree
Hide file tree
Showing 8 changed files with 161 additions and 189 deletions.
29 changes: 13 additions & 16 deletions src/main/java/com/imsweb/algorithms/ayasiterecode/AyaSiteRecodeUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,20 +3,16 @@
*/
package com.imsweb.algorithms.ayasiterecode;

import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;

import org.apache.commons.lang3.StringUtils;

import com.opencsv.CSVParserBuilder;
import com.opencsv.CSVReader;
import com.opencsv.CSVReaderBuilder;
import com.opencsv.exceptions.CsvException;
import de.siegmar.fastcsv.reader.CsvReader;
import de.siegmar.fastcsv.reader.CsvRecord;

public final class AyaSiteRecodeUtils {

Expand Down Expand Up @@ -94,19 +90,20 @@ private static List<AyaSiteRecodeData> readData(String filename) {

boolean txt = filename.endsWith(".txt");

try (Reader reader = new InputStreamReader(is, StandardCharsets.UTF_8);
CSVReader csvReader = new CSVReaderBuilder(reader).withCSVParser(new CSVParserBuilder().withSeparator(txt ? ';' : ',').build()).withSkipLines(txt ? 2 : 1).build()) {
for (String[] row : csvReader.readAll()) {
String beh = StringUtils.trimToNull(row[txt ? 1 : 3]);
String site = StringUtils.trimToNull(row[2]);
String hist = StringUtils.trimToNull(row[txt ? 3 : 1]);
String recode = StringUtils.trimToNull(row[4]);
File csvFile = new File("src/main/resources/ayasiterecode/" + filename);
try (CsvReader<CsvRecord> reader = CsvReader.builder().fieldSeparator(txt ? ';' : ',').ofCsvRecord(csvFile.toPath())) {
reader.skipLines(txt ? 2 : 1);
reader.stream().forEach(line -> {
String beh = StringUtils.trimToNull(line.getField(txt ? 1 : 3));
String site = StringUtils.trimToNull(line.getField(2));
String hist = StringUtils.trimToNull(line.getField(txt ? 3 : 1));
String recode = StringUtils.trimToNull(line.getField(4));
if (beh != null && site != null && hist != null && recode != null)
result.add(new AyaSiteRecodeData(site, hist, beh, StringUtils.leftPad(recode, txt ? 2 : 3, "0")));
}
});
}
}
catch (CsvException | IOException e) {
catch (IOException e) {
throw new IllegalStateException("Unable to read " + filename, e);
}
return result;
Expand Down
21 changes: 9 additions & 12 deletions src/test/java/com/imsweb/algorithms/AlgorithmsTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,6 @@
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.Collections;
import java.util.HashMap;
Expand All @@ -21,9 +19,8 @@
import org.junit.BeforeClass;
import org.junit.Test;

import com.opencsv.CSVReader;
import com.opencsv.CSVReaderBuilder;
import com.opencsv.exceptions.CsvException;
import de.siegmar.fastcsv.reader.CsvReader;
import de.siegmar.fastcsv.reader.NamedCsvRecord;

import com.imsweb.algorithms.countyatdiagnosisanalysis.CountyAtDxAnalysisUtils;
import com.imsweb.algorithms.internal.Utils;
Expand Down Expand Up @@ -57,7 +54,7 @@ private static String getBuildProperty(String property) {
}

@Test
public void testFields() throws IOException, CsvException {
public void testFields() throws IOException {
NaaccrDictionary dictionary = NaaccrXmlDictionaryUtils.getMergedDictionaries(NaaccrFormat.NAACCR_VERSION_230);

Set<String> ids = new HashSet<>();
Expand Down Expand Up @@ -93,12 +90,12 @@ public void testFields() throws IOException, CsvException {
if (!field.isNaaccrStandard())
algNonStandardItems.put(field.getId(), field.getNumber());

try (CSVReader reader = new CSVReaderBuilder(new InputStreamReader(Files.newInputStream(nonStandardItemsFile.toPath()), StandardCharsets.US_ASCII)).withSkipLines(1).build()) {
for (String[] row : reader.readAll()) {
Integer algNum = algNonStandardItems.get(row[0]);
if (algNum != null && algNum.equals(Integer.valueOf(row[1])))
Assert.fail("Item '" + row[0] + "' has number " + algNum + " in this library, but " + row[1] + " in the submission dictionaries");
}
try (CsvReader<NamedCsvRecord> reader = CsvReader.builder().ofNamedCsvRecord(nonStandardItemsFile.toPath())) {
reader.stream().forEach(line -> {
Integer algNum = algNonStandardItems.get(line.getField(0));
if (algNum != null && algNum.equals(Integer.valueOf(line.getField(1))))
Assert.fail("Item '" + line.getField(0) + "' has number " + algNum + " in this library, but " + line.getField(1) + " in the submission dictionaries");
});
}
}

Expand Down
46 changes: 22 additions & 24 deletions src/test/java/com/imsweb/algorithms/causespecific/CauseSpecificUtilsTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,18 +3,16 @@
*/
package com.imsweb.algorithms.causespecific;

import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.concurrent.atomic.AtomicInteger;

import org.junit.Assert;
import org.junit.Test;

import com.opencsv.CSVReader;
import com.opencsv.CSVReaderBuilder;
import com.opencsv.exceptions.CsvException;
import de.siegmar.fastcsv.reader.CsvReader;
import de.siegmar.fastcsv.reader.NamedCsvRecord;

public class CauseSpecificUtilsTest {

Expand All @@ -35,31 +33,31 @@ public void testComputeCauseSpecific() {

@Test
@SuppressWarnings("ConstantConditions")
public void testCsvFile() throws IOException, CsvException {
int count = 0;
try (InputStream is = Thread.currentThread().getContextClassLoader().getResourceAsStream("causespecific/testCauseSpecific.csv");
CSVReader reader = new CSVReaderBuilder(new InputStreamReader(is, StandardCharsets.US_ASCII)).withSkipLines(1).build()) {
for (String[] row : reader.readAll()) {
public void testCsvFile() throws IOException {
AtomicInteger count = new AtomicInteger();

File csvFile = new File("src/test/resources/causespecific/testCauseSpecific.csv");
try (CsvReader<NamedCsvRecord> reader = CsvReader.builder().ofNamedCsvRecord(csvFile.toPath())) {
reader.stream().forEach(line -> {
CauseSpecificInputDto input = new CauseSpecificInputDto();
input.setSequenceNumberCentral(row[0]);
input.setIcdRevisionNumber(row[1]);
input.setCauseOfDeath(row[2]);
input.setPrimarySite(row[3]);
input.setHistologyIcdO3(row[4]);
input.setDateOfLastContactYear(row[5]);
input.setSequenceNumberCentral(line.getField(0));
input.setIcdRevisionNumber(line.getField(1));
input.setCauseOfDeath(line.getField(2));
input.setPrimarySite(line.getField(3));
input.setHistologyIcdO3(line.getField(4));
input.setDateOfLastContactYear(line.getField(5));

String causeSpecificExpected = row[7];
String causeOtherExpected = row[8];
String causeSpecificExpected = line.getField(7);
String causeOtherExpected = line.getField(8);

String causeSpecificCalculated = CauseSpecificUtils.computeCauseSpecific(input).getCauseSpecificDeathClassification();
String causeOtherCalculated = CauseSpecificUtils.computeCauseSpecific(input).getCauseOtherDeathClassification();
count++;
count.getAndIncrement();
if (!causeSpecificExpected.equals(causeSpecificCalculated) || !causeOtherExpected.equals(causeOtherCalculated)) {
//System.out.println(SeerSiteRecodeUtils.calculateSiteRecode(row[3], row[4]));
Assert.fail("Unexpected result for row number " + (count + 1) + " " + Arrays.asList(row) + "\nExpected results: " + causeSpecificExpected + ", " +
"" + causeOtherExpected + " But found: " + causeSpecificCalculated + ", " + causeOtherCalculated);
Assert.fail("Unexpected result for row number " + (count.get() + 1) + " " + Arrays.asList(line.getFields().toArray(new String[0])) + "\nExpected results: " + causeSpecificExpected + ", "
+ causeOtherExpected + " But found: " + causeSpecificCalculated + ", " + causeOtherCalculated);
}
}
});
}
}

Expand Down
63 changes: 30 additions & 33 deletions src/test/java/com/imsweb/algorithms/napiia/NapiiaUtilsTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,21 +3,20 @@
*/
package com.imsweb.algorithms.napiia;

import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;

import org.junit.Assert;
import org.junit.Test;

import com.opencsv.CSVReader;
import com.opencsv.CSVReaderBuilder;
import com.opencsv.exceptions.CsvException;
import de.siegmar.fastcsv.reader.CsvReader;
import de.siegmar.fastcsv.reader.NamedCsvRecord;

public class NapiiaUtilsTest {

Expand Down Expand Up @@ -852,42 +851,40 @@ public void testComputeNapiia() {

@SuppressWarnings("ConstantConditions")
@Test
public void testCsvFile() throws IOException, CsvException {
int line = 0;
try (CSVReader reader = new CSVReaderBuilder(
new InputStreamReader(Thread.currentThread().getContextClassLoader().getResourceAsStream("napiia/testNAPIIA.csv"), StandardCharsets.US_ASCII)).withSkipLines(1).build()) {
for (String[] row : reader.readAll()) {
line++;
public void testCsvFile() throws IOException {
AtomicInteger count = new AtomicInteger();

//if (line != 10)
// continue;

File csvFile = new File("src/test/resources/napiia/testNAPIIA.csv");
try (CsvReader<NamedCsvRecord> reader = CsvReader.builder().ofNamedCsvRecord(csvFile.toPath())) {
reader.stream().forEach(line -> {
count.getAndIncrement();

Map<String, String> rec = new HashMap<>();
rec.put(_PROP_RACE1, row[0].trim().isEmpty() ? null : row[0]);
rec.put(_PROP_RACE2, row[1].trim().isEmpty() ? null : row[1]);
rec.put(_PROP_RACE3, row[2].trim().isEmpty() ? null : row[2]);
rec.put(_PROP_RACE4, row[3].trim().isEmpty() ? null : row[3]);
rec.put(_PROP_RACE5, row[4].trim().isEmpty() ? null : row[4]);
rec.put(_PROP_SPANISH_HISPANIC_ORIGIN, row[5].trim().isEmpty() ? null : row[5]);
rec.put(_PROP_BIRTH_PLACE_COUNTRY, row[6].trim().isEmpty() ? null : row[6]);
rec.put(_PROP_SEX, row[7].trim().isEmpty() ? null : row[7]);
rec.put(_PROP_NAME_LAST, row[8].trim().isEmpty() ? null : row[8]);
rec.put(_PROP_NAME_BIRTH_SURNAME, row[9].trim().isEmpty() ? null : row[9]);
rec.put(_PROP_NAME_FIRST, row[10].trim().isEmpty() ? null : row[10]);

String napiia = row[11];
Boolean review = Boolean.valueOf(row[12]);
String reason = row[13].trim().isEmpty() ? null : row[13];
rec.put(_PROP_RACE1, line.getField(0).trim().isEmpty() ? null : line.getField(0));
rec.put(_PROP_RACE2, line.getField(1).trim().isEmpty() ? null : line.getField(1));
rec.put(_PROP_RACE3, line.getField(2).trim().isEmpty() ? null : line.getField(2));
rec.put(_PROP_RACE4, line.getField(3).trim().isEmpty() ? null : line.getField(3));
rec.put(_PROP_RACE5, line.getField(4).trim().isEmpty() ? null : line.getField(4));
rec.put(_PROP_SPANISH_HISPANIC_ORIGIN, line.getField(5).trim().isEmpty() ? null : line.getField(5));
rec.put(_PROP_BIRTH_PLACE_COUNTRY, line.getField(6).trim().isEmpty() ? null : line.getField(6));
rec.put(_PROP_SEX, line.getField(7).trim().isEmpty() ? null : line.getField(7));
rec.put(_PROP_NAME_LAST, line.getField(8).trim().isEmpty() ? null : line.getField(8));
rec.put(_PROP_NAME_BIRTH_SURNAME, line.getField(9).trim().isEmpty() ? null : line.getField(9));
rec.put(_PROP_NAME_FIRST, line.getField(10).trim().isEmpty() ? null : line.getField(10));

String napiia = line.getField(11);
Boolean review = Boolean.valueOf(line.getField(12));
String reason = line.getField(13).trim().isEmpty() ? null : line.getField(13);
NapiiaResultsDto results = computeNapiia(rec);

if (!napiia.equals(results.getNapiiaValue()))
Assert.fail("Unexpected napiia result in CSV data file for row #" + line + " " + Arrays.asList(row) + " " + results.getNapiiaValue());
Assert.fail("Unexpected napiia result in CSV data file for row #" + count + " " + Arrays.asList(line.getFields().toArray(new String[0])) + " " + results.getNapiiaValue());
if (!review.equals(results.getNeedsHumanReview()))
Assert.fail("Unexpected needs manual review result in CSV data file for row #" + line + " " + Arrays.asList(row) + " " + results.getNeedsHumanReview());
Assert.fail("Unexpected needs manual review result in CSV data file for row #" + count + " " + Arrays.asList(line.getFields().toArray(new String[0])) + " " + results.getNeedsHumanReview());
if (results.getReasonForReview() == null ? reason != null : !results.getReasonForReview().equals(reason))
Assert.fail("Unexpected reason for review result in CSV data file for row #" + line + " " + Arrays.asList(row) + " " + results.getReasonForReview());
}
//System.out.println(line + " cases tested!");
Assert.fail("Unexpected reason for review result in CSV data file for row #" + count + " " + Arrays.asList(line.getFields().toArray(new String[0])) + " " + results.getReasonForReview());
});
}
}

Expand Down
46 changes: 21 additions & 25 deletions src/test/java/com/imsweb/algorithms/nhia/NhiaUtilsTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,8 @@
*/
package com.imsweb.algorithms.nhia;

import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
Expand All @@ -15,9 +14,8 @@
import org.junit.Assert;
import org.junit.Test;

import com.opencsv.CSVReader;
import com.opencsv.CSVReaderBuilder;
import com.opencsv.exceptions.CsvException;
import de.siegmar.fastcsv.reader.CsvReader;
import de.siegmar.fastcsv.reader.NamedCsvRecord;

public class NhiaUtilsTest {

Expand Down Expand Up @@ -345,34 +343,32 @@ public void testPatientSet() {

@SuppressWarnings("ConstantConditions")
@Test
public void testCsvFile() throws IOException, CsvException {
try (CSVReader reader = new CSVReaderBuilder(
new InputStreamReader(Thread.currentThread().getContextClassLoader().getResourceAsStream("nhia/testNHIA.csv"), StandardCharsets.US_ASCII)).withSkipLines(1).build()) {
for (String[] row : reader.readAll()) {
public void testCsvFile() throws IOException {
File csvFile = new File("src/test/resources/nhia/testNHIA.csv");
try (CsvReader<NamedCsvRecord> reader = CsvReader.builder().ofNamedCsvRecord(csvFile.toPath())) {
reader.stream().forEach(line -> {
Map<String, String> rec = new HashMap<>();
rec.put(_PROP_SPANISH_HISPANIC_ORIGIN, row[0]);
rec.put(_PROP_BIRTH_PLACE_COUNTRY, row[1]);
rec.put(_PROP_RACE1, row[2]);
rec.put(_PROP_IHS, row[3]);
rec.put(_PROP_STATE_DX, row[4]);
rec.put(_PROP_COUNTY_DX_ANALYSIS, row[5]);
rec.put(_PROP_SEX, row[6]);
rec.put(_PROP_NAME_LAST, row[7]);
rec.put(_PROP_NAME_BIRTH_SURNAME, row[8]);

String option = row[9];
String nhia = row[10];
rec.put(_PROP_SPANISH_HISPANIC_ORIGIN, line.getField(0));
rec.put(_PROP_BIRTH_PLACE_COUNTRY, line.getField(1));
rec.put(_PROP_RACE1, line.getField(2));
rec.put(_PROP_IHS, line.getField(3));
rec.put(_PROP_STATE_DX, line.getField(4));
rec.put(_PROP_COUNTY_DX_ANALYSIS, line.getField(5));
rec.put(_PROP_SEX, line.getField(6));
rec.put(_PROP_NAME_LAST, line.getField(7));
rec.put(_PROP_NAME_BIRTH_SURNAME, line.getField(8));

String option = line.getField(9);
String nhia = line.getField(10);

if (!nhia.equals(computeNhia(rec, option).getNhia()))
Assert.fail("Unexpected result in CSV data file for row " + Arrays.asList(row));
}
Assert.fail("Unexpected result in CSV data file for row " + Arrays.asList(line.getFields().toArray(new String[0])));
});
}
}

@Test
public void testGetLowHispanicCountiesPerState() {
//for (Map.Entry<String, List<String>> entry : NhiaUtils.getLowHispanicCountiesPerState().entrySet())
// System.out.println(entry.getKey() + ": " + entry.getValue());
Assert.assertFalse(NhiaUtils.getLowHispanicCountiesPerState().isEmpty());
}

Expand Down
Loading

0 comments on commit 34483e6

Please sign in to comment.