reorganizing the data-collection for new bufr2geojson oaproc output #4313
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: test docker deployment ⚙️ | |
| on: [ push, pull_request ] | |
| jobs: | |
| main: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| name: setup Python | |
| with: | |
| python-version: "3.10" | |
| - name: install testing requirements 📦 | |
| working-directory: tests | |
| run: | | |
| pip3 install -r requirements.txt | |
| - name: cache schemas 📦 | |
| run: | | |
| pywis-pubsub schema sync | |
| pywcmp bundle sync | |
| - name: display Docker and Python versions 📦 | |
| run: | | |
| docker version | |
| docker compose version | |
| python3 -V | |
| - name: setup wis2downloader | |
| run: | | |
| # make sure current user in docker group | |
| sudo usermod -aG docker $(whoami) | |
| # Add env parameters to tests/test.env | |
| echo "" >> tests/test.env #no new line at end of test.env file | |
| echo "DOCKER_GID=$(getent group docker | cut -d: -f3)" >> tests/test.env | |
| # make sure data dir is in group docker and has 775 permissions | |
| mkdir -p tests/data/downloads | |
| chown -R $(whoami):docker tests/data/downloads | |
| chmod -R 775 tests/data/downloads | |
| - name: setup wis2box configuration, replace localhost with IP on host 📦 | |
| run: | | |
| export IP=$(hostname -I | awk '{print $1}') | |
| cp tests/test.env wis2box.env | |
| sed -i "s/localhost/$IP/g" wis2box.env | |
| cat wis2box.env | |
| python3 wis2box-ctl.py config | |
| - name: build wis2box | |
| run: | | |
| python3 wis2box-ctl.py build | |
| python3 wis2box-ctl.py update | |
| - name: start containers ⚙️ | |
| run: | | |
| python3 wis2box-ctl.py start | |
| python3 wis2box-ctl.py status -a | |
| - name: show environment and check collections exist ⚙️ | |
| run: | | |
| sleep 30 | |
| python3 wis2box-ctl.py execute wis2box environment show | |
| curl http://localhost/oapi/collections/stations | |
| curl http://localhost/oapi/collections/discovery-metadata | |
| curl http://localhost/oapi/collections/messages | |
| - name: downloader subscribe to topics | |
| run: | | |
| python3 wis2box-ctl.py execute wis2box auth add-token -p wis2downloader github123 -y | |
| echo '{"topic": "origin/a/wis2/+/data/core/#"}' > test.json | |
| curl -X POST http://localhost/wis2downloader/subscriptions -H "Content-Type: application/json" -H "Authorization: Bearer github123" -d @test.json | |
| - name: add CAP data, cap-publish | |
| env: | |
| DISCOVERY_METADATA: /data/wis2box/metadata/discovery/int-wmo-test-cap.yml | |
| DISCOVERY_METADATA_ID: urn:wmo:md:int-wmo-test:cap | |
| run: | | |
| python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA | |
| curl -s http://localhost/data/metadata/$DISCOVERY_METADATA_ID.json --output /tmp/$DISCOVERY_METADATA_ID | |
| pywcmp ets validate /tmp/$DISCOVERY_METADATA_ID | |
| python3 tests/send_CAP_as_message.py | |
| - name: add Malawi synop data (csv2bufr synop_bufr template) 🇲🇼 | |
| env: | |
| TOPIC_HIERARCHY: origin/a/wis2/mw-mw_met_centre-test/data/core/weather/surface-based-observations/synop | |
| TERRITORY: MWI | |
| DISCOVERY_METADATA: /data/wis2box/metadata/discovery/mw-surface-weather-observations.yml | |
| DISCOVERY_METADATA_ID: urn:wmo:md:mw-mw_met_centre-test:surface-weather-observations | |
| STATION_METADATA: /data/wis2box/metadata/station/malawi.csv | |
| TEST_DATA: /data/wis2box/observations/malawi | |
| TEST_DATA_UPDATE: /data/wis2box/observations/malawi_update | |
| run: | | |
| python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA | |
| python3 wis2box-ctl.py execute wis2box metadata station publish-collection --path $STATION_METADATA --topic-hierarchy $TOPIC_HIERARCHY | |
| curl -s http://localhost/data/metadata/$DISCOVERY_METADATA_ID.json --output /tmp/$DISCOVERY_METADATA_ID | |
| pywcmp ets validate /tmp/$DISCOVERY_METADATA_ID | |
| python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA | |
| python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA_UPDATE | |
| - name: add Italy synop data (bufr2bufr) 🇮🇹 | |
| env: | |
| TOPIC_HIERARCHY: origin/a/wis2/it-meteoam/data/core/weather/surface-based-observations/synop | |
| TERRITORY: ITA | |
| DISCOVERY_METADATA: /data/wis2box/metadata/discovery/it-surface-weather-observations.yml | |
| DISCOVERY_METADATA_ID: urn:wmo:md:it-meteoam:surface-weather-observations | |
| STATION_METADATA: /data/wis2box/metadata/station/italy.csv | |
| TEST_DATA: /data/wis2box/observations/italy | |
| run: | | |
| python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA | |
| python3 wis2box-ctl.py execute wis2box metadata station publish-collection --path $STATION_METADATA --topic-hierarchy $TOPIC_HIERARCHY | |
| curl -s http://localhost/data/metadata/$DISCOVERY_METADATA_ID.json --output /tmp/$DISCOVERY_METADATA_ID | |
| pywcmp ets validate /tmp/$DISCOVERY_METADATA_ID | |
| python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA | |
| - name: add Algeria synop data (bufr2bufr) 🇩🇿 | |
| env: | |
| TOPIC_HIERARCHY: origin/a/wis2/dz-meteoalgerie/data/core/weather/surface-based-observations/synop | |
| TERRITORY: DZA | |
| DISCOVERY_METADATA: /data/wis2box/metadata/discovery/dz-surface-weather-observations.yml | |
| DISCOVERY_METADATA_ID: urn:wmo:md:dz-meteoalgerie:surface-weather-observations | |
| STATION_METADATA: /data/wis2box/metadata/station/algeria.csv | |
| TEST_DATA: /data/wis2box/observations/algeria | |
| run: | | |
| python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA | |
| python3 wis2box-ctl.py execute wis2box metadata station publish-collection --path $STATION_METADATA --topic-hierarchy $TOPIC_HIERARCHY | |
| curl -s http://localhost/data/metadata/$DISCOVERY_METADATA_ID.json --output /tmp/$DISCOVERY_METADATA_ID | |
| pywcmp ets validate /tmp/$DISCOVERY_METADATA_ID | |
| python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA | |
| - name: add Romania synop data (synop2bufr and csv2bufr aws-template) 🇷🇴 | |
| env: | |
| TOPIC_HIERARCHY: origin/a/wis2/ro-rnimh-test/data/core/weather/surface-based-observations/synop | |
| TERRITORY: ROU | |
| DISCOVERY_METADATA: /data/wis2box/metadata/discovery/ro-synoptic-weather-observations.yml | |
| DISCOVERY_METADATA_ID: urn:wmo:md:ro-rnimh-test:synoptic-weather-observations | |
| STATION_METADATA: /data/wis2box/metadata/station/romania.csv | |
| TEST_DATA: /data/wis2box/observations/romania | |
| run: | | |
| python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA | |
| python3 wis2box-ctl.py execute wis2box metadata station publish-collection --path $STATION_METADATA --topic-hierarchy $TOPIC_HIERARCHY | |
| curl -s http://localhost/data/metadata/$DISCOVERY_METADATA_ID.json --output /tmp/$DISCOVERY_METADATA_ID | |
| pywcmp ets validate /tmp/$DISCOVERY_METADATA_ID | |
| python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA | |
| - name: add Congo synop data (synop2bufr) 🇨🇩 | |
| env: | |
| TOPIC_HIERARCHY: origin/a/wis2/cg-met/data/recommended/weather/surface-based-observations/synop | |
| TERRITORY: COD | |
| DISCOVERY_METADATA: /data/wis2box/metadata/discovery/cd-surface-weather-observations.yml | |
| DISCOVERY_METADATA_ID: urn:wmo:md:cg-met:surface-weather-observations | |
| STATION_METADATA: /data/wis2box/metadata/station/congo.csv | |
| TEST_DATA: /data/wis2box/observations/congo | |
| run: | | |
| python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA | |
| python3 wis2box-ctl.py execute wis2box metadata station publish-collection --path $STATION_METADATA --topic-hierarchy $TOPIC_HIERARCHY | |
| curl -s http://localhost/data/metadata/$DISCOVERY_METADATA_ID.json --output /tmp/$DISCOVERY_METADATA_ID | |
| #pywcmp ets validate /tmp/$DISCOVERY_METADATA_ID # uncomment once wis2box improves support for recommended data | |
| python3 wis2box-ctl.py execute wis2box auth add-token --metadata-id $DISCOVERY_METADATA_ID -p token123 -y | |
| python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA | |
| - name: add example hourly ship data (bufr2bufr) WMO | |
| env: | |
| TOPIC_HIERARCHY: origin/a/wis2/int-wmo-test/data/core/weather/surface-based-observations/ship | |
| DISCOVERY_METADATA: /data/wis2box/metadata/discovery/int-wmo-test-ship.yml | |
| DISCOVERY_METADATA_ID: urn:wmo:md:int-wmo-test:surface-weather-observations:ship | |
| STATION_METADATA: /data/wis2box/metadata/station/wmo-test-ship.csv | |
| TEST_DATA: /data/wis2box/observations/wmo/ship | |
| run: | | |
| python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA | |
| python3 wis2box-ctl.py execute wis2box metadata station publish-collection --path $STATION_METADATA --topic-hierarchy $TOPIC_HIERARCHY | |
| curl -s http://localhost/data/metadata/$DISCOVERY_METADATA_ID.json --output /tmp/$DISCOVERY_METADATA_ID | |
| pywcmp ets validate /tmp/$DISCOVERY_METADATA_ID | |
| python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA | |
| - name: add example drifting buoy data (bufr2bufr) WMO | |
| env: | |
| TOPIC_HIERARCHY: origin/a/wis2/int-wmo-test/data/core/ocean/surface-based-observations/drifting-buoys | |
| DISCOVERY_METADATA: /data/wis2box/metadata/discovery/int-wmo-test-drifting-buoys.yml | |
| DISCOVERY_METADATA_ID: urn:wmo:md:int-wmo-test:surface-weather-observations:drifting-buoys | |
| STATION_METADATA: /data/wis2box/metadata/station/wmo-test-drifting-buoys.csv | |
| TEST_DATA: /data/wis2box/observations/wmo/drifting-buoys | |
| run: | | |
| python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA | |
| python3 wis2box-ctl.py execute wis2box metadata station publish-collection --path $STATION_METADATA --topic-hierarchy $TOPIC_HIERARCHY | |
| curl -s http://localhost/data/metadata/$DISCOVERY_METADATA_ID.json --output /tmp/$DISCOVERY_METADATA_ID | |
| pywcmp ets validate /tmp/$DISCOVERY_METADATA_ID | |
| python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA | |
| - name: add example wind profile data (bufr2bufr) WMO | |
| env: | |
| TOPIC_HIERARCHY: origin/a/wis2/int-wmo-test/data/core/weather/surface-based-observations/wind-profile | |
| DISCOVERY_METADATA: /data/wis2box/metadata/discovery/int-wmo-test-wind-profile.yml | |
| DISCOVERY_METADATA_ID: urn:wmo:md:int-wmo-test:surface-weather-observations:wind-profile | |
| STATION_METADATA: /data/wis2box/metadata/station/wmo-test-wind-profile.csv | |
| TEST_DATA: /data/wis2box/observations/wmo/wind-profile | |
| run: | | |
| python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA | |
| python3 wis2box-ctl.py execute wis2box metadata station publish-collection --path $STATION_METADATA --topic-hierarchy $TOPIC_HIERARCHY | |
| curl -s http://localhost/data/metadata/$DISCOVERY_METADATA_ID.json --output /tmp/$DISCOVERY_METADATA_ID | |
| pywcmp ets validate /tmp/$DISCOVERY_METADATA_ID | |
| python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA | |
| - name: add org-daycli-test data (csv2bufr) | |
| env: | |
| TOPIC_HIERARCHY: origin/a/wis2/org-daycli-test/data/core/climate/surface-based-observations/daily | |
| DISCOVERY_METADATA: /data/wis2box/metadata/discovery/org-daycli-test.yml | |
| DISCOVERY_METADATA_ID: urn:wmo:md:org-daycli-test:surface-climate-observations:daily | |
| STATION_METADATA: /data/wis2box/metadata/station/daycli-test.csv | |
| TEST_DATA: /data/wis2box/observations/daycli-test | |
| run: | | |
| python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA | |
| python3 wis2box-ctl.py execute wis2box metadata station publish-collection --path $STATION_METADATA --topic-hierarchy $TOPIC_HIERARCHY | |
| curl -s http://localhost/data/metadata/$DISCOVERY_METADATA_ID.json --output /tmp/$DISCOVERY_METADATA_ID | |
| pywcmp ets validate /tmp/$DISCOVERY_METADATA_ID | |
| python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA | |
| - name: add China GRIB2 data (universal pipeline) 🇨🇳 | |
| env: | |
| DISCOVERY_METADATA: /data/wis2box/metadata/discovery/cn-grapes-geps-global.yml | |
| DISCOVERY_METADATA_ID: urn:wmo:md:cn-cma:grapes-geps-global | |
| TEST_DATA: /data/wis2box/observations/china | |
| run: | | |
| python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA | |
| curl -s http://localhost/data/metadata/$DISCOVERY_METADATA_ID.json --output /tmp/$DISCOVERY_METADATA_ID | |
| pywcmp ets validate /tmp/$DISCOVERY_METADATA_ID | |
| python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA | |
| - name: sleep 30 seconds then run integration tests ⚙️ | |
| run: | | |
| sleep 30 | |
| docker logs wis2box-management | |
| pytest -s tests/integration | |
| - name: run flake8 ⚙️ | |
| run: | | |
| find . -type f -name "*.py" | xargs flake8 | |
| - name: failed tests 🚩 | |
| if: ${{ failure() }} | |
| run: | | |
| docker logs -n1000 wis2box-management | |
| docker logs -n1000 wis2box-api |