Add Gaia v20.0.0 to chains running tests in CI (#4210)

* Update bootstrapping of consumer chain

* Add Gaia v20 to CI and update Gaia used in CI tests

* Correctly create consumer chain and opt-in validator

* WIP: Fix CCQs between Stride and Gaia

* Add metrics for ICS31 cross chain queries

* Remove unnecessary code

* Fix clippy

* Disable CCQ test

* Improve error and metric recording for ICS31 cross chain queries

* Use Gaia v20 in CI jobs

* Add changelog entry

.changelog/unreleased/improvements/ibc-integration-test/4204-update-gaia-to-v20.md

@@ -0,0 +1,2 @@
+- Update the version of Gaia running the integration tests in the CI from `v18.1.0`
+  to `v20.0.0` ([\#4204](https://github.com/informalsystems/hermes/issues/4204))

.github/workflows/integration.yaml

@@ -45,7 +45,7 @@ jobs:
       fail-fast: false
       matrix:
         chain:
-          - package: gaia18
+          - package: gaia20
             command: gaiad
             account_prefix: cosmos
             native_token: stake
@@ -178,10 +178,10 @@ jobs:
       fail-fast: false
       matrix:
         chain:
-          - package: .#gaia18 .#stride
+          - package: .#gaia20 .#stride
             command: gaiad,strided
             account_prefix: cosmos,stride
-          - package: .#gaia18 .#neutron
+          - package: .#gaia20 .#neutron
             command: gaiad,neutrond
             account_prefix: cosmos,neutron
     steps:
@@ -218,12 +218,13 @@ jobs:
             --features interchain-security,ica interchain_security::
 
   interchain-security-icq:
+    if: false # Disable CCQ test
     runs-on: ubuntu-20.04
     strategy:
       fail-fast: false
       matrix:
         chain:
-          - package: .#gaia18 .#stride-no-admin
+          - package: .#gaia20 .#stride-no-admin
             command: gaiad,strided
             account_prefix: cosmos,stride
     steps:
@@ -264,7 +265,7 @@ jobs:
       fail-fast: false
       matrix:
         chain:
-          - package: .#celestia .#gaia18
+          - package: .#celestia .#gaia20
             command: celestia-appd,gaiad
             account_prefix: celestia,cosmos
             native_token: utia,stake
@@ -309,7 +310,7 @@ jobs:
       fail-fast: false
       matrix:
         chain:
-          - package: .#gaia18
+          - package: .#gaia20
             command: gaiad
             account_prefix: cosmos
     steps:

.github/workflows/multi-chains.yaml

@@ -58,7 +58,7 @@ jobs:
       fail-fast: false
       matrix:
         first-package:
-          - package: gaia18
+          - package: gaia20
             command: gaiad
             account_prefix: cosmos
           - package: ibc-go-v7-simapp

crates/relayer/src/worker/cross_chain_query.rs

@@ -8,6 +8,7 @@ use crate::error::Error;
 use crate::event::IbcEventWithHeight;
 use crate::foreign_client::ForeignClient;
 use crate::object::CrossChainQuery;
+use crate::telemetry;
 use crate::util::task::{spawn_background_task, Next, TaskError, TaskHandle};
 use crate::worker::WorkerCmd;
 
@@ -74,6 +75,12 @@ fn handle_cross_chain_query<ChainA: ChainHandle, ChainB: ChainHandle>(
 
         // Handle of queried chain has to query data from it's RPC
         info!("request: {}", cross_chain_query.short_name());
+        telemetry!(
+            cross_chain_queries,
+            &cross_chain_query.src_chain_id,
+            &cross_chain_query.dst_chain_id,
+            queries.len()
+        );
         let response = chain_b_handle.cross_chain_query(queries);
         if let Ok(cross_chain_query_responses) = response {
             // Run only when cross chain query response is not empty
@@ -87,7 +94,7 @@ fn handle_cross_chain_query<ChainA: ChainHandle, ChainB: ChainHandle>(
                         },
                         IncludeProof::No,
                     )
-                    .map_err(|_| TaskError::Fatal(RunError::query()))?
+                    .map_err(|e| TaskError::Fatal(RunError::relayer(e)))?
                     .0;
 
                 // Retrieve client based on client id
@@ -96,19 +103,21 @@ fn handle_cross_chain_query<ChainA: ChainHandle, ChainB: ChainHandle>(
                     chain_a_handle.clone(),
                     connection_end.client_id(),
                 )
-                .map_err(|_| TaskError::Fatal(RunError::query()))?;
+                .map_err(|e| TaskError::Fatal(RunError::foreign_client(e)))?;
 
                 let target_height = Height::new(
                     chain_b_handle.id().version(),
                     cross_chain_query_responses.first().unwrap().height as u64,
                 )
-                .map_err(|_| TaskError::Fatal(RunError::query()))?
+                .map_err(|e| TaskError::Fatal(RunError::ics02(e)))?
                 .increment();
 
                 // Push update client msg
                 let mut chain_a_msgs = client_a
                     .wait_and_build_update_client(target_height)
-                    .map_err(|_| TaskError::Fatal(RunError::query()))?;
+                    .map_err(|e| TaskError::Fatal(RunError::foreign_client(e)))?;
+
+                let num_cross_chain_query_responses = cross_chain_query_responses.len();
 
                 for response in cross_chain_query_responses {
                     info!("response arrived: query_id: {}", response.query_id);
@@ -118,18 +127,40 @@ fn handle_cross_chain_query<ChainA: ChainHandle, ChainB: ChainHandle>(
                             .try_to_any(
                                 chain_a_handle
                                     .get_signer()
-                                    .map_err(|_| TaskError::Fatal(RunError::query()))?,
+                                    .map_err(|e| TaskError::Fatal(RunError::relayer(e)))?,
                             )
-                            .map_err(|_| TaskError::Fatal(RunError::query()))?,
+                            .map_err(|e| TaskError::Fatal(RunError::ics31(e)))?,
                     );
                 }
 
-                chain_a_handle
+                let ccq_responses = chain_a_handle
                     .send_messages_and_wait_check_tx(TrackedMsgs::new_uuid(
                         chain_a_msgs,
                         Uuid::new_v4(),
                     ))
-                    .map_err(|_| TaskError::Ignore(RunError::query()))?;
+                    .map_err(|e| {
+                        // Since all the CCQs failed, generate a failure code for the telemetry
+                        let failed_codes =
+                            vec![tendermint::abci::Code::from(1); num_cross_chain_query_responses];
+                        telemetry!(
+                            cross_chain_query_responses,
+                            &cross_chain_query.dst_chain_id,
+                            &cross_chain_query.src_chain_id,
+                            failed_codes
+                        );
+
+                        TaskError::Ignore(RunError::relayer(e))
+                    })?;
+
+                telemetry!(
+                    cross_chain_query_responses,
+                    &cross_chain_query.dst_chain_id,
+                    &cross_chain_query.src_chain_id,
+                    ccq_responses
+                        .iter()
+                        .map(|ccq_response| ccq_response.code)
+                        .collect()
+                );
             }
         }
     }

crates/relayer/src/worker/error.rs

@@ -1,9 +1,12 @@
 use crossbeam_channel::RecvError;
 use flex_error::{define_error, DisplayOnly};
+use ibc_relayer_types::applications::ics31_icq::error::Error as Ics31Error;
 use ibc_relayer_types::core::ics02_client::error::Error as Ics02Error;
 
 use crate::channel::ChannelError;
 use crate::connection::ConnectionError;
+use crate::error::Error as RelayerError;
+use crate::foreign_client::ForeignClientError;
 use crate::link::error::LinkError;
 
 define_error! {
@@ -12,6 +15,10 @@ define_error! {
             [ Ics02Error ]
             | _ | { "client error" },
 
+        Ics31
+            [ Ics31Error ]
+            | _ | { "cross chain query error" },
+
         Connection
             [ ConnectionError ]
             | _ | { "connection error" },
@@ -20,19 +27,24 @@ define_error! {
             [ ChannelError ]
             | _ | { "channel error" },
 
+        ForeignClient
+            [ ForeignClientError ]
+            | _ | { "foreign client error" },
+
         Link
             [ LinkError ]
             | _ | { "link error" },
 
+        Relayer
+            [ RelayerError ]
+            | _ | { "relayer error" },
+
         Retry
             { retries: retry::Error<u64> }
             | e | { format_args!("worker failed after {} retries", e.retries) },
 
         Recv
             [ DisplayOnly<RecvError> ]
             | _ | { "error receiving from channel: sender end has been closed" },
-
-        Query
-            | _ | { "error occurred during querying" }
     }
 }