Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: Uptime calculation #265

Merged
merged 10 commits into from
Aug 12, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 9 additions & 7 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,18 +13,20 @@ please see [changelog_updates.md](docs/dev/changelog_updates.md).

#### Minor

- Copyable contact email and subject fields on data offer detail dialogs
- Catalog: Organization filter is no longer split into ID and name
- Catalog: Connector filter is no longer split into ID and endpoint
- Catalog: Removed dataspace filter when only one dataspace is known
- Added a message when the CaaS request feature is not available

#### Patch

- Copyable contact email and subject fields on data offer detail dialogs
- Fixed the close button on the self-hosted/CaaS connector choice page [#258](https://github.com/sovity/authority-portal/issues/258)
- Fixed Dashboard showing uptimes of over 100%
- Organization list: Data offer and connector counts now show the correct numbers according to the active environment
- Fixed provider organization ID not showing up on CaaS connectors [#206](https://github.com/sovity/authority-portal/issues/206)
- Keep in mind that sovity needs to be registered in the portal for the ID to show up.
- Already registered connectors will be updated automatically, this process can take up to 24 hours
- Added a message when the CaaS request feature is not available
- Catalog: Removed dataspace filter when only one dataspace is known
- Catalog: Organization filter is no longer split into ID and name
- Catalog: Connector filter is no longer split into ID and endpoint
- Organization list: Data offer and connector counts now show the correct numbers according to the active environment
- Fixed the close button on the self-hosted/CaaS connector choice page [#258](https://github.com/sovity/authority-portal/issues/258)

### Known issues

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ package de.sovity.authorityportal.web.pages

import de.sovity.authorityportal.api.model.ComponentStatusOverview
import de.sovity.authorityportal.api.model.UptimeStatusDto
import de.sovity.authorityportal.db.jooq.enums.ComponentOnlineStatus
import de.sovity.authorityportal.db.jooq.enums.ComponentOnlineStatus.UP
import de.sovity.authorityportal.db.jooq.enums.ComponentType
import de.sovity.authorityportal.db.jooq.enums.ConnectorOnlineStatus
import de.sovity.authorityportal.db.jooq.tables.records.ComponentDowntimesRecord
Expand All @@ -25,11 +25,9 @@ import de.sovity.authorityportal.web.thirdparty.uptimekuma.model.toDto
import de.sovity.authorityportal.web.utils.TimeUtils
import jakarta.enterprise.context.ApplicationScoped
import org.jooq.DSLContext
import java.text.DecimalFormat
import java.text.DecimalFormatSymbols
import java.time.Duration
import java.time.Duration.between
import java.time.OffsetDateTime
import java.util.Locale

@ApplicationScoped
class ComponentStatusApiService(
Expand All @@ -47,7 +45,8 @@ class ComponentStatusApiService(
}

fun getComponentsStatusForOrganizationId(environmentId: String, organizationId: String): ComponentStatusOverview {
val connectorStatuses = connectorStatusQuery.getConnectorStatusInfoByOrganizationIdAndEnvironment(organizationId, environmentId)
val connectorStatuses =
connectorStatusQuery.getConnectorStatusInfoByOrganizationIdAndEnvironment(organizationId, environmentId)
val statusCount = countConnectorStatuses(connectorStatuses)

return buildComponenStatusOverview(statusCount, environmentId)
Expand All @@ -60,7 +59,8 @@ class ComponentStatusApiService(
val latestDapsStatus = componentStatusService.getLatestComponentStatus(ComponentType.DAPS, environmentId)
val latestLoggingHouseStatus =
componentStatusService.getLatestComponentStatus(ComponentType.LOGGING_HOUSE, environmentId)
val latestBrokerCrawlerStatus = componentStatusService.getLatestComponentStatus(ComponentType.CATALOG_CRAWLER, environmentId)
val latestBrokerCrawlerStatus =
componentStatusService.getLatestComponentStatus(ComponentType.CATALOG_CRAWLER, environmentId)
val now = timeUtils.now()

return ComponentStatusOverview(
Expand All @@ -82,12 +82,12 @@ class ComponentStatusApiService(
return null
}

val upSince = Duration.between(latestStatus.timeStamp.toInstant(), now.toInstant()).abs()
val upSince = between(latestStatus.timeStamp.toInstant(), now.toInstant()).abs()
val timeSpan = Duration.ofDays(30)

return UptimeStatusDto(
componentStatus = latestStatus.status.toDto(),
upSinceSeconds = upSince.toSeconds().takeIf { latestStatus.status == ComponentOnlineStatus.UP } ?: 0,
upSinceSeconds = upSince.toSeconds().takeIf { latestStatus.status == UP } ?: 0,
timeSpanSeconds = timeSpan.toSeconds(),
uptimePercentage = calculateUptimePercentage(latestStatus.component, timeSpan, environmentId, now)
)
Expand All @@ -100,55 +100,31 @@ class ComponentStatusApiService(
now: OffsetDateTime
): Double {
val limit = now.minus(timeSpan)
var statusHistoryAsc = componentStatusService.getStatusHistoryAscSince(component, limit, environmentId)
// If no status was found before the limit, the first record in the history is used as base for the calculation
val initialRecord =
componentStatusService.getFirstRecordBefore(component, limit, environmentId) ?: statusHistoryAsc.first()

// If no "UP" status was found, return 0.00
// Also, drop all entries before first "UP" status, to avoid wrong uptime calculation
var tmpLastUpStatus = if (initialRecord.status == ComponentOnlineStatus.UP) initialRecord else {
statusHistoryAsc = statusHistoryAsc.dropWhile { it.status != ComponentOnlineStatus.UP }
statusHistoryAsc.firstOrNull()
}

if (tmpLastUpStatus == null) {
return 0.00
}
val statusHistoryAsc = componentStatusService.getStatusHistoryAscSince(component, limit, environmentId)

// Sum up the total duration of "UP" statuses
var totalUptimeDuration = Duration.ZERO
for (componentRecord in statusHistoryAsc) {
if (componentRecord.status == ComponentOnlineStatus.UP) {
tmpLastUpStatus = componentRecord
} else {
totalUptimeDuration += Duration.between(
tmpLastUpStatus!!.timeStamp.toInstant(),
componentRecord.timeStamp.toInstant()
).abs()
}
}
val first = statusHistoryAsc.first()

// Add time if last status is "UP"
val lastRecord = statusHistoryAsc.lastOrNull() ?: tmpLastUpStatus
if (lastRecord!!.status == ComponentOnlineStatus.UP) {
totalUptimeDuration += Duration.between(lastRecord.timeStamp.toInstant(), now.toInstant()).abs()
val head = when {
first.timeStamp.isBefore(limit) -> ComponentDowntimesRecord(component, first.status, environmentId, limit)
else -> first
}

// Subtract potential uptime before the limit
if (initialRecord.status == ComponentOnlineStatus.UP && initialRecord.timeStamp.isBefore(limit)) {
totalUptimeDuration -= Duration.between(initialRecord.timeStamp.toInstant(), limit.toInstant()).abs()
val body = statusHistoryAsc.drop(1)
val tail = ComponentDowntimesRecord(component, statusHistoryAsc.last().status, environmentId, now)

val whole = listOf(head) + body + listOf(tail)

val duration = whole.zipWithNext().fold(Duration.ZERO) { acc, (start, end) ->
when (start.status) {
UP -> acc + between(start.timeStamp, end.timeStamp)
else -> acc
}
}

// Calculate uptime percentage
val totalDuration =
Duration.between(initialRecord.timeStamp.toInstant(), now.toInstant()).coerceAtMost(timeSpan).abs()
val uptimePercentage = totalUptimeDuration.toMillis().toDouble() / totalDuration.toMillis().toDouble() * 100
val uptime = 100.0 * duration.toMillis() / between(head.timeStamp, now).toMillis()

// Round value to two decimal places
val symbols = DecimalFormatSymbols(Locale.US)
val formatter = DecimalFormat("#.##", symbols)
return formatter.format(uptimePercentage).toDouble()
return uptime
}

private fun countConnectorStatuses(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,21 +49,7 @@ class ComponentStatusService(
val c = Tables.COMPONENT_DOWNTIMES

return dsl.selectFrom(c)
.where(c.COMPONENT.eq(component)).and(c.ENVIRONMENT.eq(environment))
.orderBy(c.TIME_STAMP.desc())
.limit(1)
.fetchOne()
}

fun getFirstRecordBefore(
component: ComponentType,
limit: OffsetDateTime,
environment: String
): ComponentDowntimesRecord? {
val c = Tables.COMPONENT_DOWNTIMES

return dsl.selectFrom(c)
.where(c.COMPONENT.eq(component)).and(c.ENVIRONMENT.eq(environment)).and(c.TIME_STAMP.lessThan(limit))
.where(c.COMPONENT.eq(component), c.ENVIRONMENT.eq(environment))
.orderBy(c.TIME_STAMP.desc())
.limit(1)
.fetchOne()
Expand All @@ -77,7 +63,21 @@ class ComponentStatusService(
val c = Tables.COMPONENT_DOWNTIMES

return dsl.selectFrom(c)
.where(c.COMPONENT.eq(component)).and(c.ENVIRONMENT.eq(environment)).and(c.TIME_STAMP.greaterOrEqual(limit))
.where(
c.COMPONENT.eq(component),
c.ENVIRONMENT.eq(environment),
c.TIME_STAMP.lessThan(limit)
)
.orderBy(c.TIME_STAMP.desc())
.limit(1)
.union(
dsl.selectFrom(c)
.where(
c.COMPONENT.eq(component),
c.ENVIRONMENT.eq(environment),
c.TIME_STAMP.greaterOrEqual(limit)
)
)
.orderBy(c.TIME_STAMP.asc())
.fetch()
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,8 @@ class ComponentStatusApiServiceTest {
val now = OffsetDateTime.now()
val env1 = "test"
val env2 = "dev"
setupStatusHistory(now, env1, env2)
val env3 = "env3"
setupStatusHistory(now, env1, env2, env3)

useMockNow(now)

Expand Down Expand Up @@ -96,6 +97,7 @@ class ComponentStatusApiServiceTest {
// act
val resultEnv1 = componentStatusApiService.getComponentsStatus(env1)
val resultEnv2 = componentStatusApiService.getComponentsStatus(env2)
val resultEnv3 = componentStatusApiService.getComponentsStatus(env3)

// assert
assertThat(resultEnv1.dapsStatus?.componentStatus).isEqualTo(ComponentOnlineStatus.MAINTENANCE.toDto())
Expand All @@ -115,9 +117,19 @@ class ComponentStatusApiServiceTest {
assertThat(resultEnv2.dapsStatus?.timeSpanSeconds).isEqualTo(Duration.ofDays(30).toSeconds())
assertThat(resultEnv2.dapsStatus?.upSinceSeconds).isEqualTo(Duration.ZERO.toSeconds())
assertThat(resultEnv2.loggingHouseStatus).isNull()

assertThat(resultEnv3.loggingHouseStatus?.componentStatus).isEqualTo(ComponentOnlineStatus.DOWN.toDto())
assertThat(resultEnv3.loggingHouseStatus?.uptimePercentage).isCloseTo(50.0, Offset.offset(0.1))
assertThat(resultEnv3.loggingHouseStatus?.timeSpanSeconds).isEqualTo(Duration.ofDays(30).toSeconds())
assertThat(resultEnv3.loggingHouseStatus?.upSinceSeconds).isEqualTo(Duration.ZERO.toSeconds())

assertThat(resultEnv3.dapsStatus?.componentStatus).isEqualTo(ComponentOnlineStatus.UP.toDto())
assertThat(resultEnv3.dapsStatus?.uptimePercentage).isCloseTo(100.0, Offset.offset(0.1))
assertThat(resultEnv3.dapsStatus?.timeSpanSeconds).isEqualTo(Duration.ofDays(30).toSeconds())
assertThat(resultEnv3.dapsStatus?.upSinceSeconds).isCloseTo(Duration.ofDays(15).toSeconds(), Offset.offset(1L))
}

private fun setupStatusHistory(now: OffsetDateTime, environment1: String, environment2: String) {
private fun setupStatusHistory(now: OffsetDateTime, environment1: String, environment2: String, environment3: String) {
val c = Tables.COMPONENT_DOWNTIMES

dsl.insertInto(c)
Expand All @@ -137,6 +149,13 @@ class ComponentStatusApiServiceTest {
// DAPS: Only "DOWN" status, older than 30 days
.values(ComponentType.DAPS, ComponentOnlineStatus.DOWN, environment2, now.minus(Duration.ofDays(31)))
// LH: Empty
// Environment 3
// LH: Up
.values(ComponentType.LOGGING_HOUSE, ComponentOnlineStatus.UP, environment3, now.minus(Duration.ofDays(10)))
.values(ComponentType.LOGGING_HOUSE, ComponentOnlineStatus.PENDING, environment3, now.minus(Duration.ofDays(5)))
.values(ComponentType.LOGGING_HOUSE, ComponentOnlineStatus.DOWN, environment3, now.minus(Duration.ofSeconds(1)))
// DAPS: Only "UP" status
.values(ComponentType.DAPS, ComponentOnlineStatus.UP, environment3, now.minus(Duration.ofDays(15)))
.execute()
}
}
Loading