Skip to content

Commit

Permalink
[chain] When inserting balances, do it in chunks to avoid hitting que…
Browse files Browse the repository at this point in the history
…ry limits
  • Loading branch information
joel-u410 committed Nov 21, 2024
1 parent a0f4cd4 commit 15dd4fa
Showing 1 changed file with 23 additions and 15 deletions.
38 changes: 23 additions & 15 deletions chain/src/repository/balance.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,21 +18,29 @@ pub fn insert_balance(
transaction_conn: &mut PgConnection,
balances: Balances,
) -> anyhow::Result<()> {
diesel::insert_into(balance_changes::table)
.values::<&Vec<BalanceChangesInsertDb>>(
&balances
.into_iter()
.map(BalanceChangesInsertDb::from_balance)
.collect::<Vec<_>>(),
)
.on_conflict((
balance_changes::columns::owner,
balance_changes::columns::token,
balance_changes::columns::height,
))
.do_nothing()
.execute(transaction_conn)
.context("Failed to update balances in db")?;
tracing::info!("Inserting {} balances into db", balances.len());

// Group balances into chunks to avoid hitting the limit of the number of bind parameters in one query.
balances.chunks(10000).try_for_each(|balances_chunk| {
diesel::insert_into(balance_changes::table)
.values::<&Vec<BalanceChangesInsertDb>>(
&balances_chunk
.iter()
.cloned()
.map(BalanceChangesInsertDb::from_balance)
.collect::<Vec<_>>(),
)
.on_conflict((
balance_changes::columns::owner,
balance_changes::columns::token,
balance_changes::columns::height,
))
.do_nothing()
.execute(transaction_conn)
.context("Failed to update balances in db")?;

anyhow::Ok(())
})?;

anyhow::Ok(())
}
Expand Down

0 comments on commit 15dd4fa

Please sign in to comment.