feat: snowpipe streaming

rudderlabs · Sep 23, 2024 · 444da09 · 444da09
1 parent 336b876
commit 444da09
Show file tree

Hide file tree

Showing 29 changed files with 2,433 additions and 25 deletions.
diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
@@ -142,6 +142,7 @@ jobs:
           - integration_test/tracing
           - integration_test/backendconfigunavailability
           - integration_test/trackedusersreporting
+          - integration_test/snowpipestreaming
           - processor
           - regulation-worker
           - router
@@ -168,6 +169,11 @@ jobs:
           go-version-file: 'go.mod'
       - run: go version
       - run: go mod download
+      - name: Login to DockerHub
+        uses: docker/[email protected]
+        with:
+          username: rudderlabs
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
       - name: Package Unit [ ${{ matrix.package }} ]
         env:
           TEST_KAFKA_CONFLUENT_CLOUD_HOST: ${{ secrets.TEST_KAFKA_CONFLUENT_CLOUD_HOST }}
@@ -178,6 +184,7 @@ jobs:
           TEST_KAFKA_AZURE_EVENT_HUBS_CLOUD_CONNECTION_STRING: ${{ secrets.TEST_KAFKA_AZURE_EVENT_HUBS_CLOUD_CONNECTION_STRING }}
           TEST_S3_DATALAKE_CREDENTIALS: ${{ secrets.TEST_S3_DATALAKE_CREDENTIALS }}
           BIGQUERY_INTEGRATION_TEST_CREDENTIALS: ${{ secrets.BIGQUERY_INTEGRATION_TEST_CREDENTIALS }}
+          SNOWPIPE_STREAMING_KEYPAIR_UNENCRYPTED_INTEGRATION_TEST_CREDENTIALS: ${{ secrets.SNOWPIPE_STREAMING_KEYPAIR_UNENCRYPTED_INTEGRATION_TEST_CREDENTIALS }}
         run: make test exclude="${{ matrix.exclude }}" package=${{ matrix.package }}
       - name: Sanitize name for Artifact
         run: |