Upgrade to DataFusion 43/Arrow 53.2 #4549
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: [push, pull_request] | |
| jobs: | |
| lint-console: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check out | |
| uses: actions/checkout@v3 | |
| - name: Setup pnpm | |
| uses: pnpm/action-setup@v4 | |
| with: | |
| version: 9.0.6 | |
| run_install: | | |
| - recursive: true | |
| args: [--frozen-lockfile, --strict-peer-dependencies] | |
| - name: Run prettier | |
| run: | | |
| cd webui | |
| pnpm check | |
| build-rust: | |
| runs-on: ubicloud-standard-30 | |
| env: | |
| DATABASE_URL: "postgres://arroyo:arroyo@localhost:5432/arroyo" | |
| steps: | |
| - name: Check out | |
| uses: actions/checkout@v3 | |
| - uses: actions/setup-java@v3 | |
| with: | |
| distribution: 'temurin' | |
| java-version: '11' | |
| - name: Install Rust | |
| uses: actions-rs/toolchain@v1 | |
| with: | |
| toolchain: stable | |
| override: true | |
| - name: Check Formatting | |
| run: cargo fmt -- --check | |
| - uses: actions/setup-python@v5 | |
| name: Setup Python | |
| with: | |
| python-version: '3.12' | |
| - name: Setup pnpm | |
| uses: pnpm/action-setup@v4 | |
| with: | |
| version: 9.0.6 | |
| run_install: | | |
| - recursive: true | |
| args: [--frozen-lockfile, --strict-peer-dependencies] | |
| - name: Install OpenAPI Generator | |
| run: | | |
| pnpm install @openapitools/openapi-generator-cli -g | |
| cd crates/arroyo-openapi && openapi-generator-cli version | |
| - name: Setup Postgres | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install postgresql | |
| sudo systemctl start postgresql | |
| sudo -u postgres psql -c "CREATE USER arroyo WITH PASSWORD 'arroyo' SUPERUSER;" | |
| sudo -u postgres createdb arroyo | |
| pushd /tmp | |
| wget https://github.com/rust-db/refinery/releases/download/0.8.7/refinery-0.8.7-x86_64-unknown-linux-musl.tar.gz | |
| tar xvfz refinery*.tar.gz | |
| mv /tmp/refinery*-musl/refinery /tmp | |
| popd | |
| /tmp/refinery migrate -e DATABASE_URL -p crates/arroyo-api/migrations | |
| - name: Install dependencies | |
| run: | | |
| curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin | |
| sudo apt-get install -y cmake clang ruby unzip postgresql libsasl2-dev netcat | |
| wget https://github.com/protocolbuffers/protobuf/releases/download/v21.8/protoc-21.8-linux-x86_64.zip | |
| unzip protoc*.zip | |
| sudo mv bin/protoc /usr/local/bin | |
| curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh | |
| - name: Install Kafka | |
| run: | | |
| wget --progress=dot --show-progress https://archive.apache.org/dist/kafka/3.5.0/kafka_2.12-3.5.0.tgz | |
| tar xvfz kafka*.tgz | |
| mkdir /tmp/kraft-combined-logs | |
| kafka_*/bin/kafka-storage.sh format -t 9v5PspiySuWU2l5NjTgRuA -c kafka_*/config/kraft/server.properties | |
| kafka_*/bin/kafka-server-start.sh -daemon kafka_*/config/kraft/server.properties | |
| - name: Install mosquitto | |
| run: | | |
| sudo apt-get install -y mosquitto | |
| sudo service mosquitto start | |
| - name: Check Formatting | |
| run: cargo fmt -- --check | |
| - name: Build console | |
| run: | | |
| cd webui | |
| pnpm build | |
| - name: Build | |
| run: cargo build --all-features | |
| - name: Run Clippy | |
| run: cargo clippy --all-features --all-targets --workspace -- -D warnings | |
| - name: Test | |
| run: cargo nextest run -E 'kind(lib)' --all-features | |
| - name: Integ postgres | |
| run: | | |
| mkdir /tmp/arroyo-integ | |
| ARROYO__DISABLE_TELEMETRY=true ARROYO__CHECKPOINT_URL=file:///tmp/arroyo-integ ARROYO__COMPILER__ARTIFACT_URL=file:///tmp/artifacts target/debug/arroyo cluster & | |
| cargo nextest run --package integ -E 'kind(test)' | |
| - name: Integ sqlite | |
| run: | | |
| killall arroyo | |
| ARROYO__DISABLE_TELEMETRY=true ARROYO__CHECKPOINT_URL=file:///tmp/arroyo-integ ARROYO__COMPILER__ARTIFACT_URL=file:///tmp/artifacts ARROYO__DATABASE__TYPE=sqlite target/debug/arroyo cluster & | |
| timeout=10; while ! nc -z localhost 5115 && [ $timeout -gt 0 ]; do sleep 1; timeout=$((timeout - 1)); done; [ $timeout -gt 0 ] | |
| cargo nextest run --package integ -E 'kind(test)' | |
| build-console: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check out | |
| uses: actions/checkout@v3 | |
| - name: Setup pnpm | |
| uses: pnpm/action-setup@v4 | |
| with: | |
| version: 9.0.6 | |
| run_install: | | |
| - recursive: true | |
| args: [--frozen-lockfile, --strict-peer-dependencies] | |
| - name: Build console | |
| run: | | |
| cd webui | |
| pnpm build |