Skip to content

Commit

Permalink
Merge pull request #322 from NOAA-OWP/issue319
Browse files Browse the repository at this point in the history
Issue319
  • Loading branch information
james-d-brown authored Sep 20, 2024
2 parents 5535de7 + 4a9dfed commit c6c8ed8
Show file tree
Hide file tree
Showing 16 changed files with 213 additions and 174 deletions.
5 changes: 4 additions & 1 deletion wres-config/src/wres/config/yaml/DeclarationFactory.java
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@
import wres.config.yaml.components.DatasetOrientation;
import wres.config.yaml.components.EvaluationDeclaration;
import wres.config.yaml.components.Threshold;
import wres.config.yaml.components.ThresholdBuilder;
import wres.config.yaml.components.ThresholdOperator;
import wres.config.yaml.components.ThresholdOrientation;
import wres.config.yaml.components.ThresholdType;
Expand Down Expand Up @@ -131,7 +132,9 @@ public class DeclarationFactory
.setOperator( DEFAULT_THRESHOLD_OPERATOR.canonical() )
.build();
/** Default wrapped threshold. */
public static final Threshold DEFAULT_THRESHOLD = new Threshold( DEFAULT_CANONICAL_THRESHOLD, null, null, null );
public static final Threshold DEFAULT_THRESHOLD =
ThresholdBuilder.builder().threshold( DEFAULT_CANONICAL_THRESHOLD )
.build();

/** To stringify protobufs into presentable JSON. */
public static final Function<MessageOrBuilder, String> PROTBUF_STRINGIFIER = message -> {
Expand Down
12 changes: 11 additions & 1 deletion wres-config/src/wres/config/yaml/DeclarationInterpolator.java
Original file line number Diff line number Diff line change
Expand Up @@ -1676,6 +1676,11 @@ private static Metric addThresholdsToMetric( Map<wres.config.yaml.components.Thr
// would render the request meaningless
if ( parametersBuilder.thresholds()
.size() == 1
// Declared by a user, i.e., not generated by the software?
&& !parametersBuilder.thresholds()
.iterator()
.next()
.generated()
&& parametersBuilder.thresholds()
.iterator()
.next()
Expand All @@ -1699,7 +1704,12 @@ private static Metric addThresholdsToMetric( Map<wres.config.yaml.components.Thr
if ( name.isContinuous()
&& addAllData )
{
valueThresholds.add( DeclarationUtilities.ALL_DATA_THRESHOLD );
// Flag this as an internally generated threshold
Threshold allData = ThresholdBuilder.builder( DeclarationUtilities.ALL_DATA_THRESHOLD )
.generated( true )
.build();

valueThresholds.add( allData );
}

// Render the value thresholds featureful, if possible
Expand Down
10 changes: 6 additions & 4 deletions wres-config/src/wres/config/yaml/DeclarationMigrator.java
Original file line number Diff line number Diff line change
Expand Up @@ -2080,10 +2080,12 @@ private static Set<Threshold> migrateCsvThresholdsInline( ThresholdsConfig.Sourc
.build();
Set<wres.statistics.generated.Threshold> toMigrate = nextEntry.getValue();
Set<Threshold> innerMigrated = toMigrate.stream()
.map( next -> new Threshold( next,
canonicalType,
feature,
featureNameFrom ) )
.map( next -> ThresholdBuilder.builder()
.threshold( next )
.type( canonicalType )
.feature( feature )
.featureNameFrom( featureNameFrom )
.build() )
.collect( Collectors.toCollection( LinkedHashSet::new ) );
migrated.addAll( innerMigrated );
}
Expand Down
62 changes: 32 additions & 30 deletions wres-config/src/wres/config/yaml/DeclarationUtilities.java
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@
import wres.config.yaml.components.SourceBuilder;
import wres.config.yaml.components.SourceInterface;
import wres.config.yaml.components.Threshold;
import wres.config.yaml.components.ThresholdBuilder;
import wres.config.yaml.components.ThresholdType;
import wres.config.yaml.components.TimeInterval;
import wres.config.yaml.components.TimePools;
Expand All @@ -84,14 +85,19 @@ public class DeclarationUtilities

/** All data threshold. */
public static final Threshold ALL_DATA_THRESHOLD =
new Threshold( wres.statistics.generated.Threshold.newBuilder()
.setLeftThresholdValue( Double.NEGATIVE_INFINITY )
.setOperator( wres.statistics.generated.Threshold.ThresholdOperator.GREATER )
.setDataType( wres.statistics.generated.Threshold.ThresholdDataType.LEFT_AND_RIGHT )
.build(),
wres.config.yaml.components.ThresholdType.VALUE,
null,
null );
ThresholdBuilder.builder()
.threshold( wres.statistics.generated.Threshold.newBuilder()
.setLeftThresholdValue( Double.NEGATIVE_INFINITY )
.setOperator( wres.statistics.generated.Threshold.ThresholdOperator.GREATER )
.setDataType( wres.statistics.generated.Threshold.ThresholdDataType.LEFT_AND_RIGHT )
.build() )
.type( wres.config.yaml.components.ThresholdType.VALUE )
// All data threshold, as declared or read from a source, not generated by the software.
// Distinguishing between an all data threshold as declared vs. generated internally is
// important in some contexts, such as when establishing if the threshold was declared
// explicitly as an override for a specific metric
.generated( false )
.build();

/** Re-used message. */
private static final String CANNOT_DETERMINE_TIME_WINDOWS_FROM_MISSING_DECLARATION = "Cannot determine time "
Expand Down Expand Up @@ -1040,7 +1046,10 @@ public static EvaluationDeclaration addThresholds( EvaluationDeclaration declara

/**
* Removes from the declaration any features for which featureful thresholds are undefined. Features are matched on
* name only.
* name only. Only filters against thresholds available within the declaration (i.e., internal). Thus, any
* thresholds from external sources, such as files or threshold services, should be "read into" the declaration
* before using this utility.
*
* @param declaration the declaration
* @return the adjusted declaration
* @throws NullPointerException if the declaration is null
Expand Down Expand Up @@ -1081,27 +1090,20 @@ public static EvaluationDeclaration removeFeaturesWithoutThresholds( EvaluationD
// Create a filter. Only filter when the above sets contain some features
Predicate<GeometryTuple> retain = geoTuple ->
{
if ( geoTuple.hasLeft()
&& !leftFeatureNamesWithThresholds.isEmpty()
&& !leftFeatureNamesWithThresholds.contains( geoTuple.getLeft()
.getName() ) )
{
return false;
}

if ( geoTuple.hasRight()
&& !rightFeatureNamesWithThresholds.isEmpty()
&& !rightFeatureNamesWithThresholds.contains( geoTuple.getRight()
.getName() ) )
{
return false;
}

return !DeclarationUtilities.hasBaseline( declaration )
|| !geoTuple.hasBaseline()
|| baselineFeatureNamesWithThresholds.isEmpty()
|| baselineFeatureNamesWithThresholds.contains( geoTuple.getBaseline()
.getName() );
boolean hasLeft = geoTuple.hasLeft()
&& !leftFeatureNamesWithThresholds.isEmpty()
&& leftFeatureNamesWithThresholds.contains( geoTuple.getLeft()
.getName() );
boolean hasRight = geoTuple.hasRight()
&& !rightFeatureNamesWithThresholds.isEmpty()
&& rightFeatureNamesWithThresholds.contains( geoTuple.getRight()
.getName() );
boolean hasBaseline = geoTuple.hasBaseline()
&& !baselineFeatureNamesWithThresholds.isEmpty()
&& baselineFeatureNamesWithThresholds.contains( geoTuple.getBaseline()
.getName() );

return hasLeft || hasRight || hasBaseline;
};

// Remove the features in all contexts
Expand Down
18 changes: 9 additions & 9 deletions wres-config/src/wres/config/yaml/DeclarationValidator.java
Original file line number Diff line number Diff line change
Expand Up @@ -3746,15 +3746,15 @@ private static List<EvaluationStatusEvent> validateFeaturefulThresholds( Set<Thr
}

// Explicit features and featureful thresholds. Some featureful thresholds must be correlated with features
Set<String> thresholdFeatureNames = thresholds.stream()
.filter( n -> n.featureNameFrom() == orientation )
// Ignore all data, which was added automagically
.filter( n -> !DeclarationUtilities.ALL_DATA_THRESHOLD.threshold()
.equals(
n.threshold() ) )
.map( Threshold::feature )
.map( wres.statistics.generated.Geometry::getName )
.collect( Collectors.toSet() );
Set<String> thresholdFeatureNames =
thresholds.stream()
.filter( n -> n.featureNameFrom() == orientation )
// Ignore all data, which was added automagically
.filter( n -> !DeclarationUtilities.ALL_DATA_THRESHOLD.threshold()
.equals( n.threshold() ) )
.map( Threshold::feature )
.map( wres.statistics.generated.Geometry::getName )
.collect( Collectors.toSet() );

if ( thresholdFeatureNames.isEmpty() )
{
Expand Down
7 changes: 5 additions & 2 deletions wres-config/src/wres/config/yaml/components/Threshold.java
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,14 @@
* @param type the threshold type to help identify the declaration context
* @param feature a feature
* @param featureNameFrom the orientation of the data from which the named feature is taken
* @param generated true if this threshold was generated by the software, false if declared or read from a source
*/
@RecordBuilder
public record Threshold( wres.statistics.generated.Threshold threshold,
ThresholdType type,
Geometry feature,
DatasetOrientation featureNameFrom )
DatasetOrientation featureNameFrom,
boolean generated )
{
/** Logger. */
private static final Logger LOGGER = LoggerFactory.getLogger( Threshold.class );
Expand All @@ -39,7 +41,8 @@ public record Threshold( wres.statistics.generated.Threshold threshold,
*/
public Threshold
{
if ( Objects.nonNull( feature ) && Objects.isNull( featureNameFrom ) )
if ( Objects.nonNull( feature )
&& Objects.isNull( featureNameFrom ) )
{
LOGGER.debug( "Discovered a threshold for feature {}, but the orientation of the feature name was not "
+ "supplied. Assuming an orientation of {}.",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import wres.config.yaml.DeclarationUtilities;
import wres.config.yaml.components.DatasetOrientation;
import wres.config.yaml.components.Threshold;
import wres.config.yaml.components.ThresholdBuilder;
import wres.config.yaml.components.ThresholdOrientation;
import wres.config.yaml.components.ThresholdType;
import wres.statistics.generated.Geometry;
Expand Down Expand Up @@ -178,7 +179,10 @@ private Set<Threshold> getSingletonThreshold( JsonNode thresholdNode,
}

wres.statistics.generated.Threshold nextThreshold = builder.build();
Threshold wrappedThreshold = new Threshold( nextThreshold, type, null, null );
Threshold wrappedThreshold = ThresholdBuilder.builder()
.threshold( nextThreshold )
.type( type )
.build();

return Set.of( wrappedThreshold );
}
Expand Down Expand Up @@ -345,7 +349,12 @@ private Set<Threshold> getEmbellishedThresholds( ObjectReader reader,
}

Threshold nextThreshold =
new Threshold( builder.build(), type, feature, featureNameFrom );
ThresholdBuilder.builder()
.threshold( builder.build() )
.type( type )
.feature( feature )
.featureNameFrom( featureNameFrom )
.build();
thresholds.add( nextThreshold );
}

Expand Down Expand Up @@ -386,7 +395,10 @@ private Set<Threshold> getThresholdsFromArray( ObjectReader reader,
}

wres.statistics.generated.Threshold nextThreshold = thresholdBuilder.build();
Threshold nextWrappedThreshold = new Threshold( nextThreshold, type, null, null );
Threshold nextWrappedThreshold = ThresholdBuilder.builder()
.threshold( nextThreshold )
.type( type )
.build();
thresholds.add( nextWrappedThreshold );
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import org.slf4j.LoggerFactory;

import wres.config.yaml.components.Threshold;
import wres.config.yaml.components.ThresholdBuilder;
import wres.config.yaml.components.ThresholdType;

/**
Expand All @@ -37,7 +38,7 @@ public void serialize( Set<Threshold> thresholds,

LOGGER.debug( "Discovered threshold sets with {} members.", grouped.size() );

if( !grouped.isEmpty() )
if ( !grouped.isEmpty() )
{
// Start the threshold sets
writer.writeStartArray();
Expand Down Expand Up @@ -102,7 +103,12 @@ private Map<Threshold, Set<Threshold>> groupThresholdsByType( Set<Threshold> thr
.clearLeftThresholdProbability()
.build();

Threshold outer = new Threshold( nextInner, next.type(), next.feature(), next.featureNameFrom() );
Threshold outer = ThresholdBuilder.builder()
.threshold( nextInner )
.type( next.type() )
.feature( next.feature() )
.featureNameFrom( next.featureNameFrom() )
.build();

if ( grouped.containsKey( outer ) )
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import wres.config.yaml.DeclarationFactory;
import wres.config.yaml.DeclarationUtilities;
import wres.config.yaml.components.Threshold;
import wres.config.yaml.components.ThresholdBuilder;
import wres.config.yaml.components.ThresholdOrientation;
import wres.config.yaml.components.ThresholdType;
import wres.statistics.generated.Geometry;
Expand Down Expand Up @@ -329,7 +330,9 @@ private boolean hasDefaultMetadata( Set<Threshold> thresholds )
.clearRightThresholdValue()
.build();

Threshold blankOuter = new Threshold( blank, null, null, null );
Threshold blankOuter = ThresholdBuilder.builder()
.threshold( blank )
.build();

if ( !blankOuter.equals( DeclarationFactory.DEFAULT_THRESHOLD )
|| Objects.nonNull( next.feature() ) )
Expand Down
Loading

0 comments on commit c6c8ed8

Please sign in to comment.