From f0ec446617d526e995c7fa912c306fe1a6a1330f Mon Sep 17 00:00:00 2001 From: Jiaqiang Huang <96465211+River2000i@users.noreply.github.com> Date: Thu, 14 Nov 2024 10:42:59 +0800 Subject: [PATCH 01/28] add log --- dm/tests/many_tables/run.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dm/tests/many_tables/run.sh b/dm/tests/many_tables/run.sh index 3fc99f2c4bb..0ffb9a6a680 100644 --- a/dm/tests/many_tables/run.sh +++ b/dm/tests/many_tables/run.sh @@ -151,7 +151,9 @@ function run() { pkill -hup tidb-server 2>/dev/null || true wait_process_exit tidb-server # now worker will process some binlog events, save table checkpoint and meet downstream error + echo "start incremental_data_2" incremental_data_2 + echo "finish incremental_data_2" sleep 30 resume_num=$(grep 'unit process error' $WORK_DIR/worker1/log/dm-worker.log | wc -l) From 204b8c1f391da86da8ee39e7ea0b62a002d163f0 Mon Sep 17 00:00:00 2001 From: Jiaqiang Huang <96465211+River2000i@users.noreply.github.com> Date: Fri, 15 Nov 2024 15:40:24 +0800 Subject: [PATCH 02/28] add dependency for validate IP addresses with SANs --- dm/tests/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/dm/tests/requirements.txt b/dm/tests/requirements.txt index f2293605cf1..8ca17f61940 100644 --- a/dm/tests/requirements.txt +++ b/dm/tests/requirements.txt @@ -1 +1,2 @@ requests +ipaddress From 81dfe81302f4a080b7c24d580f1841d28f8fb03b Mon Sep 17 00:00:00 2001 From: Jiaqiang Huang <96465211+River2000i@users.noreply.github.com> Date: Mon, 18 Nov 2024 15:48:47 +0800 Subject: [PATCH 03/28] retry when curl command get not zero --- dm/tests/_utils/run_downstream_cluster | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dm/tests/_utils/run_downstream_cluster b/dm/tests/_utils/run_downstream_cluster index 36038c74e07..d51f08ad903 100755 --- a/dm/tests/_utils/run_downstream_cluster +++ b/dm/tests/_utils/run_downstream_cluster @@ -34,7 +34,7 @@ EOF --data-dir "$WORK_DIR/pd" & # wait until PD is online... i=0 - while ! curl "http://$PD_ADDR/pd/api/v1/version"; do + while curl "http://$PD_ADDR/pd/api/v1/version"; do i=$((i + 1)) if [ "$i" -gt 20 ]; then echo 'Failed to start PD' @@ -75,7 +75,7 @@ start_tidb() { --log-file "$WORK_DIR/tidb.log" & i=0 - while ! curl "http://$TIDB_IP:10080/status"; do + while curl "http://$TIDB_IP:10080/status"; do i=$((i + 1)) if [ "$i" -gt 50 ]; then echo 'Failed to start TiDB' From 6c45472caa63b7cbd40aae59186fcad2abcb86fb Mon Sep 17 00:00:00 2001 From: Jiaqiang Huang <96465211+River2000i@users.noreply.github.com> Date: Mon, 18 Nov 2024 15:51:41 +0800 Subject: [PATCH 04/28] add log --- dm/tests/_utils/run_downstream_cluster | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dm/tests/_utils/run_downstream_cluster b/dm/tests/_utils/run_downstream_cluster index d51f08ad903..1b6d2c15411 100755 --- a/dm/tests/_utils/run_downstream_cluster +++ b/dm/tests/_utils/run_downstream_cluster @@ -42,6 +42,7 @@ EOF fi sleep 3 done + echo 'Start PD success' } start_tikv() { @@ -62,6 +63,7 @@ start_tikv() { fi sleep 5 done + echo 'Start TiKV success' } start_tidb() { @@ -83,6 +85,7 @@ start_tidb() { fi sleep 3 done + echo 'Start TiDB success' } start_pd From 69b422c9a7cb80a3fbbd4264f1ce44b2a9c3b4d5 Mon Sep 17 00:00:00 2001 From: Jiaqiang Huang <96465211+River2000i@users.noreply.github.com> Date: Mon, 18 Nov 2024 16:52:02 +0800 Subject: [PATCH 05/28] add log --- dm/tests/tls/run.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/dm/tests/tls/run.sh b/dm/tests/tls/run.sh index 3fc76387f38..5320c4b2f59 100644 --- a/dm/tests/tls/run.sh +++ b/dm/tests/tls/run.sh @@ -42,6 +42,7 @@ EOF --log-file "$WORK_DIR/tidb.log" 2>&1 & sleep 5 + ls -alh ./conf # if execute failed, print tidb's log for debug mysql -uroot -h127.0.0.1 -P4400 --default-character-set utf8 --ssl-ca $cur/conf/ca.pem --ssl-cert $cur/conf/dm.pem --ssl-key $cur/conf/dm.key -E -e "drop database if exists tls" || (cat $WORK_DIR/tidb.log && exit 1) mysql -uroot -h127.0.0.1 -P4400 --default-character-set utf8 --ssl-ca $cur/conf/ca.pem --ssl-cert $cur/conf/dm.pem --ssl-key $cur/conf/dm.key -E -e "drop database if exists dm_meta" From d40822fadee69c594e579a33ee82ce66137f32e3 Mon Sep 17 00:00:00 2001 From: Jiaqiang Huang <96465211+River2000i@users.noreply.github.com> Date: Mon, 18 Nov 2024 17:25:30 +0800 Subject: [PATCH 06/28] debug --- dm/tests/tls/run.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/dm/tests/tls/run.sh b/dm/tests/tls/run.sh index 5320c4b2f59..d8b4d7034e3 100644 --- a/dm/tests/tls/run.sh +++ b/dm/tests/tls/run.sh @@ -42,9 +42,16 @@ EOF --log-file "$WORK_DIR/tidb.log" 2>&1 & sleep 5 - ls -alh ./conf + ls -alh $cur/conf # if execute failed, print tidb's log for debug + # openssl s_client -connect 127.0.0.1:4400 -CAfile /path/to/ca.pem -cert /path/to/dm.pem -key /path/to/dm.key + echo "test openssl" + openssl s_client -connect 127.0.0.1:4400 -CAfile $cur/conf/ca.pem -cert $cur/conf/dm.pem -key $cur/conf/dm.key + echo "show databases" + mysql -uroot -h127.0.0.1 -P4400 --default-character-set utf8 -E -e "SHOW DATABASES;" + echo "drop database1" mysql -uroot -h127.0.0.1 -P4400 --default-character-set utf8 --ssl-ca $cur/conf/ca.pem --ssl-cert $cur/conf/dm.pem --ssl-key $cur/conf/dm.key -E -e "drop database if exists tls" || (cat $WORK_DIR/tidb.log && exit 1) + echo "show databases2" mysql -uroot -h127.0.0.1 -P4400 --default-character-set utf8 --ssl-ca $cur/conf/ca.pem --ssl-cert $cur/conf/dm.pem --ssl-key $cur/conf/dm.key -E -e "drop database if exists dm_meta" } From 3f27bce4a063ae23ef539333bb0a16a52a311403 Mon Sep 17 00:00:00 2001 From: Jiaqiang Huang <96465211+River2000i@users.noreply.github.com> Date: Tue, 19 Nov 2024 11:03:19 +0800 Subject: [PATCH 07/28] Fix TLS generation script to use RSA keys instead of EC keys --- dm/tests/tls/conf/ca.pem | 21 +++++++++++++------ dm/tests/tls/conf/dm.key | 35 ++++++++++++++++++++++++------- dm/tests/tls/conf/dm.pem | 24 ++++++++++++++------- dm/tests/tls/conf/generate_tls.sh | 4 ++-- dm/tests/tls/conf/other.key | 35 ++++++++++++++++++++++++------- dm/tests/tls/conf/other.pem | 25 +++++++++++++++------- dm/tests/tls/run.sh | 20 +++++++++--------- 7 files changed, 114 insertions(+), 50 deletions(-) diff --git a/dm/tests/tls/conf/ca.pem b/dm/tests/tls/conf/ca.pem index 9fc215fa83b..54ccf0ad634 100644 --- a/dm/tests/tls/conf/ca.pem +++ b/dm/tests/tls/conf/ca.pem @@ -1,8 +1,17 @@ -----BEGIN CERTIFICATE----- -MIIBGDCBwAIJAOjYXLFw5V1HMAoGCCqGSM49BAMCMBQxEjAQBgNVBAMMCWxvY2Fs -aG9zdDAgFw0yMDAzMTcxMjAwMzNaGA8yMjkzMTIzMTEyMDAzM1owFDESMBAGA1UE -AwwJbG9jYWxob3N0MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEglCIJD8uVBfD -kuM+UQP+VA7Srbz17WPLA0Sqc+sQ2p6fT6HYKCW60EXiZ/yEC0925iyVbXEEbX4J -xCc2Heow5TAKBggqhkjOPQQDAgNHADBEAiAILL3Zt/3NFeDW9c9UAcJ9lc92E0ZL -GNDuH6i19Fex3wIgT0ZMAKAFSirGGtcLu0emceuk+zVKjJzmYbsLdpj/JuQ= +MIICsTCCAZkCFBW2dYDlpiY0fQo4/BbVb4kWuLGMMA0GCSqGSIb3DQEBCwUAMBQx +EjAQBgNVBAMMCWxvY2FsaG9zdDAgFw0yNDExMTkwMjUxNThaGA8yMjk4MDkwNDAy +NTE1OFowFDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOC +AQ8AMIIBCgKCAQEAzNlitARpbbz2zDeze6X70vGg5Ti0t7xz7MsAAl9ESyf6i/XD +zBDGN5w18YaPUsXbOFz5dJeWtVK8xheJCN19RwYUGWf6BGuLohCA7ygSy43ICVNJ +xeW47xev4RcACu/C2UelBaVQLleSKsMFtEwuGJTlGL9nfxR3NKw7JUYntcy0+1c9 +GJQunFp8UU2ymENJqlIwhtwy5Z56oWRGS34rVdyhmlQ/OrGEinTyyoZBxWJmzbvw +Cqi1+IZKmwobGVSIuNdKxLwn/KNRxXE+tWwVy/+Dq5U3mt4zdBcRHQsSuqxPzS+6 +/3MivAN+5hGebHv1UyfKEQTACyKc8YHryf5SPwIDAQABMA0GCSqGSIb3DQEBCwUA +A4IBAQCPf/9isj85Ncsw7v01apIuklLzHByYZdkD0yY3Igrw2teaOGNKcTuXHiPM +v4QpV5KkkRtL+okNR6hL1BoY4OYkDRD67b8bHqr20JiYUriA1aZ5paW2MVXya8eE +T12fZYpNuYWKfd2YQiWSUuEEhCbr6cBkw9A4oyhVx5+QAFVstysm4U4dI9fn89uK +Ywtcc0ebk7FwjwGfchKW9mGG7mSOcZdGiKxYFxnI+wIf0nwjJvRraQbNpN2RuqSx +Oc/oG8QdRklQ6GnDKEwWb4P2z4r3n16O0ntcyXt03h1pFxrGTNhQ+pwbormCmN/c +t2udn5DvywhPljodj8vnprDScR4g -----END CERTIFICATE----- diff --git a/dm/tests/tls/conf/dm.key b/dm/tests/tls/conf/dm.key index dfdc077bc4d..38105f87366 100644 --- a/dm/tests/tls/conf/dm.key +++ b/dm/tests/tls/conf/dm.key @@ -1,8 +1,27 @@ ------BEGIN EC PARAMETERS----- -BggqhkjOPQMBBw== ------END EC PARAMETERS----- ------BEGIN EC PRIVATE KEY----- -MHcCAQEEICF/GDtVxhTPTP501nOu4jgwGSDY01xN+61xd9MfChw+oAoGCCqGSM49 -AwEHoUQDQgAEgQOv5bQO7xK16vZWhwJqlz2vl19+AXW2Ql7KQyGiBJVSvLbyDLOr -kIeFlHN04iqQ39SKSOSfeGSfRt6doU6IcA== ------END EC PRIVATE KEY----- +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEAuWvI0YtEvBAkO4dHtGdwRJ2aTCsB0jjWrSBz55pMn+4ht1Sl +9YECNx6+1SSenuzfDvvNkplIeI3RI+8QjS6by0yeBQxxp60Zh/HrS2CnRbjEXjig +PnLxL/xnR1MxtlzmKH3wdq0Z7jRSYwHxZ15uKvECq8wmnJuAnKyMdOc3kQYcdD/1 +tAQ8HqKOZr6wVfjj4ApZuctqGNOmkiCgNULljQqhiVbdx9o+xTAx+vBxHpUbXpTf +BZYA8VlKh9As6wwQbNW5GgOItGanulElZRaq8Ql/sk+mEVuNlkownkQgmtL/xnbH +HUehn/OugogrOcR6MZAkEuxN/Sl03VTbHXxP0QIDAQABAoIBAQClE2qD84NMEurO +/I8/818mF1HJQfirGeM12tCVfu986tqlhv0BZst8Gyk2RRXa03N6uX65ZzH38foi +DbIknzwgcKO2Ru/goeE7AIvEXdZN4kMgxzUSo+5KFGI2Q8DcZDScFBNC9LVDNt2q +O46Zwp4+KLeCD4zInECSO6QIZ9k2iEA2Tg1/oxxMO1hZAZv/RBcKWWFHc9sDe8qT +p2proaQBu4Qr4az4I0bKZoQK5QyZwbvd8EPCsTdqRY6KLBNkDn6rsovyTFJ/fzgk +lHl1Ef5zv89i4lX/JJgfNI+YxzX/SFJDGcWmvwQvFsbaSW+SffNGaYevWK9R4idq +vBBj5iKJAoGBAN+3Bc6ap7Vbu8TckbOifRH1u+3v1Awo5gP7XnUJQ1WzvI5qJeit +bkYJ4+SLDrGspdTpeh2LqyjI4f+2XiV67oyIFRhcra9B562COMC3jAXb6B54aT4W +fWBvzBLqsX6V1VnicgXHeF++DFI+54wlULso3X44EUxrk8/UyiY4KVZzAoGBANQu +Bdyo+6pmhHMMqegwhYqmTh/Tqq2Xy4aHMzO/3Ft0ZE/C6VkHFH73xXFUkYkdMNPx +KNLkES7TFmI9jbwBY806NYq8gmFXY/pDjZPfhNr0cQTUIoM/NPL67er1CFKuL0zg +kduUZEP876B+6WLkDw0oh+c8QA8QAL/fd5tVbOurAoGADOucGT+TWgUPRiJbA93p +jUT88pP8Uit2hVpaLTY4vlVgGXWF9Pc3PDJMCCID2j0XYsGNPPGZFATuEH+u1klQ +R6uAVGTd3xZWVA4GSxMOsjPuJvxbT6uq2Av85EbLs3zh0owie7T2fckTGxjo9I4f +KIAGssh4L0qzQbiPaD+sG4MCgYADwxUjhnN/3/V1PMI+da3IeY5lDhG2pjxblEYa +Isq7bGLi7yfIBlIctjPwAHncHoJhH6ODqbuMQv3gnArJTJUCbIfKf/En3K7PHyeB +ebMXVsIyPbim7xX3QO+6/ME7Bg+co1ayt/zXH8L364gj0rtZlmBifzT7j2MUA9hR +DsCxpQKBgQCvPP4d0zZTRobdJtPaLnAgMq6F0RPhzoDzVlcKqXJzcELflSvusdel +L2CPOFs8KXe2BxlJpUUwmhuTIHuwWahkMxx/Dhd5rJ9qlQtDygTKNsaJgnflPscF +mFZkIeDqpeE05X9tcvWdA+QSgpmY7AmjwvlOfCTi0Ftx1s7HPfnhTQ== +-----END RSA PRIVATE KEY----- diff --git a/dm/tests/tls/conf/dm.pem b/dm/tests/tls/conf/dm.pem index d4f846e3a22..e4678a7f2f9 100644 --- a/dm/tests/tls/conf/dm.pem +++ b/dm/tests/tls/conf/dm.pem @@ -1,10 +1,18 @@ -----BEGIN CERTIFICATE----- -MIIBZDCCAQqgAwIBAgIJAIT/lgXUc1JqMAoGCCqGSM49BAMCMBQxEjAQBgNVBAMM -CWxvY2FsaG9zdDAgFw0yMDAzMTcxMjAwMzNaGA8yMjkzMTIzMTEyMDAzM1owDTEL -MAkGA1UEAwwCZG0wWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAASBA6/ltA7vErXq -9laHAmqXPa+XX34BdbZCXspDIaIElVK8tvIMs6uQh4WUc3TiKpDf1IpI5J94ZJ9G -3p2hTohwo0owSDAaBgNVHREEEzARgglsb2NhbGhvc3SHBH8AAAEwCwYDVR0PBAQD -AgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMCBggrBgEFBQcDATAKBggqhkjOPQQDAgNI -ADBFAiEAx6ljJ+tNa55ypWLGNqmXlB4UdMmKmE4RSKJ8mmEelfECIG2ZmCE59rv5 -wImM6KnK+vM2QnEiISH3PeYyyRzQzycu +MIIC+zCCAeOgAwIBAgIUDg53fesx1gIIYkGiIDbjngxBd40wDQYJKoZIhvcNAQEL +BQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MCAXDTI0MTExOTAyNTIxOFoYDzIyOTgw +OTA0MDI1MjE4WjANMQswCQYDVQQDDAJkbTCCASIwDQYJKoZIhvcNAQEBBQADggEP +ADCCAQoCggEBALlryNGLRLwQJDuHR7RncESdmkwrAdI41q0gc+eaTJ/uIbdUpfWB +AjcevtUknp7s3w77zZKZSHiN0SPvEI0um8tMngUMcaetGYfx60tgp0W4xF44oD5y +8S/8Z0dTMbZc5ih98HatGe40UmMB8WdebirxAqvMJpybgJysjHTnN5EGHHQ/9bQE +PB6ijma+sFX44+AKWbnLahjTppIgoDVC5Y0KoYlW3cfaPsUwMfrwcR6VG16U3wWW +APFZSofQLOsMEGzVuRoDiLRmp7pRJWUWqvEJf7JPphFbjZZKMJ5EIJrS/8Z2xx1H +oZ/zroKIKznEejGQJBLsTf0pdN1U2x18T9ECAwEAAaNKMEgwGgYDVR0RBBMwEYIJ +bG9jYWxob3N0hwR/AAABMAsGA1UdDwQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcD +AgYIKwYBBQUHAwEwDQYJKoZIhvcNAQELBQADggEBADMC8yK209WqhIBB+48kxpgU +aIxpu1CJXCHU3M476mZFYXpzfGQvxHJJSzLb6ZdfvK2TruWd1STwlnDanAH9MAxI +nFDtI7TLlpBvtSu9SB8fQLeEY5QUl5ZtBHpLKXkVQ/GMOWKP4q7L7zE7gaeaiVlC +VcvEF/+yeaj/GBu0RE5ia23V1HbthRLcyZnGvTYsyEVwTf+ZbJso3hbbN83WaKiL +avXD267fnfeJbL0No24QWFud/og28pdS1fM7ssyZZYI9k2VlrDPQF2jHQRfIu/bt +NyTNSt8etReb1rYCNKsFK8IPTkWwJmyOCDF+dhFLgA/z9KY2MZh1M+9gR9I5GNY= -----END CERTIFICATE----- diff --git a/dm/tests/tls/conf/generate_tls.sh b/dm/tests/tls/conf/generate_tls.sh index 8f8410690e0..d6090968da7 100644 --- a/dm/tests/tls/conf/generate_tls.sh +++ b/dm/tests/tls/conf/generate_tls.sh @@ -16,12 +16,12 @@ DNS.1 = localhost IP.1 = 127.0.0.1 EOF -openssl ecparam -out "ca.key" -name prime256v1 -genkey +openssl genrsa -out "ca.key" openssl req -new -batch -sha256 -subj '/CN=localhost' -key "ca.key" -out "ca.csr" openssl x509 -req -sha256 -days 100000 -in "ca.csr" -signkey "ca.key" -out "ca.pem" 2>/dev/null for role in dm other; do - openssl ecparam -out "$role.key" -name prime256v1 -genkey + openssl genrsa -out "$role.key" openssl req -new -batch -sha256 -subj "/CN=${role}" -key "$role.key" -out "$role.csr" openssl x509 -req -sha256 -days 100000 -extensions EXT -extfile "ipsan.cnf" -in "$role.csr" -CA "ca.pem" -CAkey "ca.key" -CAcreateserial -out "$role.pem" 2>/dev/null done diff --git a/dm/tests/tls/conf/other.key b/dm/tests/tls/conf/other.key index ee95ca5faa5..718b7bb6ba7 100644 --- a/dm/tests/tls/conf/other.key +++ b/dm/tests/tls/conf/other.key @@ -1,8 +1,27 @@ ------BEGIN EC PARAMETERS----- -BggqhkjOPQMBBw== ------END EC PARAMETERS----- ------BEGIN EC PRIVATE KEY----- -MHcCAQEEICzbWZZ7dtha0lGXlBiP3QjCurPs5ExsR5thIQCcKCKeoAoGCCqGSM49 -AwEHoUQDQgAEt5z9ACtEnsqv0ZPFx5YJhvBNQZJCEb75ZS/kDBiPoISea1HMt1w8 -4ZkeWW+SBCwt0RtwzVPRq9VUGWaFRUOwdQ== ------END EC PRIVATE KEY----- +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEAw1kLMqpCyhCuviPrJkQFoZIO4ocbZO/WPju2L1B9o3QEaqzF +Zs7jTCVVtLynryw12OSay+t2JuTED4hHfSlR0RjjfHj2rV9CRyLty5Njf2ZrmX+V +pUNV/c1AKo4ACZbjy7UpQZ0dQEA6bHE3THq7oLEhREL0/QkZKOEoy8CEZ8Bk2DaR +eOepFzNX9ZSuFZ/Kfljl9skJ3D/0Sg0RjGeIddhITbW8SuR7NEEcHbjweEYCuoMr +W+90FYBx0GxAACAqzyO+QzsWaq7WbDkUYeEJjwCZ9jJthYsXGnfwpLde9A4poJs5 +8LvHLq1KqDOuVXdyjQFBjZLWBVQfOTdbvzTenwIDAQABAoIBACO6G1Qu03xFRNA9 +nXooIqeEIPjJIPd67cIABbftjqzgQRaJdqjw2Z1Jjr05a6mb+lUqqiz1aWx/lmQI +1ppLfk0xshlHvkXDEEoWV0tOL3pTbwZp0SKiiA2dBklCIzudvxe4kE0RwRHJLWY7 +5pM0NzCSdLUZUdkdWt6hH2b0VMLsmIq+6L2Th8Ns3Z88+UL2UAEK0qex7TKRNRlc +wRDf9pydcqUdPu+3IBvyww/MVXJfEiZHPc9kI8Nrt0Puo/3qnhQG//2jLPJFQUod +vXOcS1bX7UAYP2NQ1YNrHloX079D7OmSN+C03pvS62fVGWwDhVQiXd56VimPEzB8 +Rz0fgqECgYEA/ldf+hqPBvXNrBAF8Z0rh/kgBSv/rGvHCOTijNL9i5dWOx1gnF4d +kx94XPmGLKcnDiSdg8tlzc9FDW17t9gmycF4oHdAmZkWy1R6rH4OcVifuQ3DqLsg +qlDQZpjySsBQN4kY/lbrmKAsXzVWDecBIrMWM6IYCQFrTaC4A9rmI5cCgYEAxJ8t +vnGbB2QQbf+PJxef+xy4TtGb2lT0SrnrNBVcVFcYbe3/AdT7iXviGULhtIJiIykR +hcMoZu9mhWnydKpf5hn9jxE9XoB4khcfrjcVIGk6DBgHk6IhGxDOgGispgNO9uMH +WWWu4KtuyBsocG/k9jJZDJ7Mltpc0h4gQyT23jkCgYBDNIk6x+vFFgwri+xWJIaN +HknUtASR5cSo1/n45gNy8CEr655Mkx3TczxVrOVERUKvnQM2APe8gykm3CgVsvrE +0udx1zDtpSL9xSvJCmH8l0NVHYr2ucNu3X3DrmRPVjOcCRDem88S/DfdTHWZpNGK +/rE+IXWiY4qb73zQH/YxrwKBgQCyxVPvGxXss2Noz/qVfZp809EUAOrmqWVwZ/hd +Y6qOGJ5inQjF3qNRjqy1cSUMXZvjAh0ndJjQdozNctX6k0ocSbdMFE2rFb4KfjHU +0cpIrMQVaOfCqJ/XHwHSqNmJG+r4+NSeyKAnMFfKYz7ydBpGauKiYeP8qM/KT4/f +3MUEqQKBgQC2zESZWJR49qMhsQlDREeD/pZjbX76DoVfUOuvNpI4mFP9Gj1cp92V +8sBVSREHeRoN/DltT8TtJgGXSzfW8pQWRxQh2BguryHYROJsXA0aVNgQPHmBLz49 +V6UOSgXetUnm9TSkSGcMKDt93NwN1e6sJ3OJruAMTG3Y6guKN20sTg== +-----END RSA PRIVATE KEY----- diff --git a/dm/tests/tls/conf/other.pem b/dm/tests/tls/conf/other.pem index 5710106ffe6..c164884e341 100644 --- a/dm/tests/tls/conf/other.pem +++ b/dm/tests/tls/conf/other.pem @@ -1,10 +1,19 @@ -----BEGIN CERTIFICATE----- -MIIBZzCCAQ2gAwIBAgIJAIT/lgXUc1JrMAoGCCqGSM49BAMCMBQxEjAQBgNVBAMM -CWxvY2FsaG9zdDAgFw0yMDAzMTcxMjAwMzNaGA8yMjkzMTIzMTEyMDAzM1owEDEO -MAwGA1UEAwwFb3RoZXIwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAAS3nP0AK0Se -yq/Rk8XHlgmG8E1BkkIRvvllL+QMGI+ghJ5rUcy3XDzhmR5Zb5IELC3RG3DNU9Gr -1VQZZoVFQ7B1o0owSDAaBgNVHREEEzARgglsb2NhbGhvc3SHBH8AAAEwCwYDVR0P -BAQDAgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMCBggrBgEFBQcDATAKBggqhkjOPQQD -AgNIADBFAiEA34/Vz7SaJWqYOgOLyr+y1OwiT9R7yTgBQCSSvGC+HpsCIA20BhNe -RnicYz+9qOQRxAFP1wpIyMMgOK4tKuZhx+/J +MIIC/jCCAeagAwIBAgIUDg53fesx1gIIYkGiIDbjngxBd4wwDQYJKoZIhvcNAQEL +BQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MCAXDTI0MTExOTAyNTE0M1oYDzIyOTgw +OTA0MDI1MTQzWjAQMQ4wDAYDVQQDDAVvdGhlcjCCASIwDQYJKoZIhvcNAQEBBQAD +ggEPADCCAQoCggEBAMNZCzKqQsoQrr4j6yZEBaGSDuKHG2Tv1j47ti9QfaN0BGqs +xWbO40wlVbS8p68sNdjkmsvrdibkxA+IR30pUdEY43x49q1fQkci7cuTY39ma5l/ +laVDVf3NQCqOAAmW48u1KUGdHUBAOmxxN0x6u6CxIURC9P0JGSjhKMvAhGfAZNg2 +kXjnqRczV/WUrhWfyn5Y5fbJCdw/9EoNEYxniHXYSE21vErkezRBHB248HhGArqD +K1vvdBWAcdBsQAAgKs8jvkM7Fmqu1mw5FGHhCY8AmfYybYWLFxp38KS3XvQOKaCb +OfC7xy6tSqgzrlV3co0BQY2S1gVUHzk3W7803p8CAwEAAaNKMEgwGgYDVR0RBBMw +EYIJbG9jYWxob3N0hwR/AAABMAsGA1UdDwQEAwIFoDAdBgNVHSUEFjAUBggrBgEF +BQcDAgYIKwYBBQUHAwEwDQYJKoZIhvcNAQELBQADggEBAKaIFX66bIZRaiaLoQqw +nvremDwYX16RTDRXPmF8Jp0Rs7Xm8WeQF4B4HDhwWtkti9muecN2CzeYTlgzkbaK +yMG2mndmbO/W6pWuWObrRZF9TA+4jIQ/r5NuBcoVLwe0s4/EWxPqZyndNxBZCNr+ +cpUlcG4QScrFfuj87K6yi+z1t7w0F1YMoRt0uD2MspWG2TQv7tR1iWMQRy9rVF9E ++GWEMPQRdMAG1fkTtlRd+VgcZXGWF7XEYWK0nSIBBXJf9mQGsl6XZcBqRMY2vgTa +Jk0Nix/aUsVP1cclVNQ9vL5qWUBnr9/3TseyqHcKFOrM+E6+vXFlVs33WJ2awwqP +HZ4= -----END CERTIFICATE----- diff --git a/dm/tests/tls/run.sh b/dm/tests/tls/run.sh index d8b4d7034e3..5b4a50c57e5 100644 --- a/dm/tests/tls/run.sh +++ b/dm/tests/tls/run.sh @@ -17,7 +17,7 @@ function get_mysql_ssl_data_path() { function setup_tidb_with_tls() { echo "run a new tidb server with tls" - cat - >"$WORK_DIR/tidb-tls-config.toml" <"/dm/tidb-tls-config.toml" <&1 & + --config /dm/tidb-tls-config.toml \ + --log-file "dm/tidb.log" 2>&1 & sleep 5 ls -alh $cur/conf From 768db29e6ddaea805a64b4f6dd5a02dfee573cdb Mon Sep 17 00:00:00 2001 From: Jiaqiang Huang <96465211+River2000i@users.noreply.github.com> Date: Tue, 19 Nov 2024 11:31:35 +0800 Subject: [PATCH 08/28] revet debug --- dm/tests/tls/run.sh | 31 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/dm/tests/tls/run.sh b/dm/tests/tls/run.sh index 5b4a50c57e5..214755943b5 100644 --- a/dm/tests/tls/run.sh +++ b/dm/tests/tls/run.sh @@ -26,32 +26,29 @@ status-port = 10090 [security] # set the path for certificates. Empty string means disabling secure connections. -ssl-ca = "/dm/tiflow/dm/tests/tls/conf/ca.pem" -ssl-cert = "/dm/tiflow/dm/tests/tls/conf/dm.pem" -ssl-key = "/dm/tiflow/dm/tests/tls/conf/dm.key" -cluster-ssl-ca = "/dm/tiflow/dm/tests/tls/conf/ca.pem" -cluster-ssl-cert = "/dm/tiflow/dm/tests/tls/conf/dm.pem" -cluster-ssl-key = "/dm/tiflow/dm/tests/tls/conf/dm.key" +ssl-ca = "$cur/conf/ca.pem" +ssl-cert = "$cur/conf/dm.pem" +ssl-key = "$cur/conf/dm.key" +cluster-ssl-ca = "$cur/conf/ca.pem" +cluster-ssl-cert = "$cur/conf/dm.pem" +cluster-ssl-key = "$cur/conf/dm.key" EOF bin/tidb-server \ -P 4400 \ - --path /dm/tidb \ + --path $WORK_DIR/tidb \ --store unistore \ - --config /dm/tidb-tls-config.toml \ - --log-file "dm/tidb.log" 2>&1 & + --config $WORK_DIR/tidb-tls-config.toml \ + --log-file "$WORK_DIR/tidb.log" 2>&1 & sleep 5 - ls -alh $cur/conf - # if execute failed, print tidb's log for debug - # openssl s_client -connect 127.0.0.1:4400 -CAfile /path/to/ca.pem -cert /path/to/dm.pem -key /path/to/dm.key - echo "test openssl" - openssl s_client -connect 127.0.0.1:4400 -CAfile $cur/conf/ca.pem -cert $cur/conf/dm.pem -key $cur/conf/dm.key - echo "show databases" + + echo "show databases without TLS" mysql -uroot -h127.0.0.1 -P4400 --default-character-set utf8 -E -e "SHOW DATABASES;" - echo "drop database1" + echo "drop database tls with TLS" + # if execute failed, print tidb's log for debug mysql -uroot -h127.0.0.1 -P4400 --default-character-set utf8 --ssl-ca $cur/conf/ca.pem --ssl-cert $cur/conf/dm.pem --ssl-key $cur/conf/dm.key -E -e "drop database if exists tls" || (cat $WORK_DIR/tidb.log && exit 1) - echo "show databases2" + echo "drop database dm_meta with TLS" mysql -uroot -h127.0.0.1 -P4400 --default-character-set utf8 --ssl-ca $cur/conf/ca.pem --ssl-cert $cur/conf/dm.pem --ssl-key $cur/conf/dm.key -E -e "drop database if exists dm_meta" } From 6d181365f1e34c981b0319c34f7a9f49164d7504 Mon Sep 17 00:00:00 2001 From: Jiaqiang Huang <96465211+River2000i@users.noreply.github.com> Date: Tue, 19 Nov 2024 11:33:13 +0800 Subject: [PATCH 09/28] fix --- dm/tests/_utils/run_downstream_cluster | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dm/tests/_utils/run_downstream_cluster b/dm/tests/_utils/run_downstream_cluster index 1b6d2c15411..997fbc6f14a 100755 --- a/dm/tests/_utils/run_downstream_cluster +++ b/dm/tests/_utils/run_downstream_cluster @@ -55,7 +55,7 @@ start_tikv() { --log-file "$WORK_DIR/tikv.log" \ --log-level info \ -s "$WORK_DIR/tikv" & - while ! curl "http://$PD_ADDR/pd/api/v1/cluster/status" | tee /dev/stderr | grep '"is_initialized": true'; do + while curl "http://$PD_ADDR/pd/api/v1/cluster/status" | tee /dev/stderr | grep '"is_initialized": true'; do i=$((i + 1)) if [ "$i" -gt 20 ]; then echo 'Failed to initialize TiKV cluster' From df4ec6176b726d6f8e8e8a734c8a12166697f386 Mon Sep 17 00:00:00 2001 From: Jiaqiang Huang <96465211+River2000i@users.noreply.github.com> Date: Tue, 19 Nov 2024 12:02:48 +0800 Subject: [PATCH 10/28] revert --- dm/tests/tls/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dm/tests/tls/run.sh b/dm/tests/tls/run.sh index 214755943b5..b5046829b78 100644 --- a/dm/tests/tls/run.sh +++ b/dm/tests/tls/run.sh @@ -17,7 +17,7 @@ function get_mysql_ssl_data_path() { function setup_tidb_with_tls() { echo "run a new tidb server with tls" - cat - >"/dm/tidb-tls-config.toml" <"$WORK_DIR/tidb-tls-config.toml" < Date: Wed, 20 Nov 2024 10:40:17 +0800 Subject: [PATCH 11/28] Revert "Fix TLS generation script to use RSA keys instead of EC keys" This reverts commit 3f27bce4a063ae23ef539333bb0a16a52a311403. --- dm/tests/tls/conf/ca.pem | 21 ++++++------------- dm/tests/tls/conf/dm.key | 35 +++++++------------------------ dm/tests/tls/conf/dm.pem | 24 +++++++-------------- dm/tests/tls/conf/generate_tls.sh | 4 ++-- dm/tests/tls/conf/other.key | 35 +++++++------------------------ dm/tests/tls/conf/other.pem | 25 +++++++--------------- 6 files changed, 40 insertions(+), 104 deletions(-) diff --git a/dm/tests/tls/conf/ca.pem b/dm/tests/tls/conf/ca.pem index 54ccf0ad634..9fc215fa83b 100644 --- a/dm/tests/tls/conf/ca.pem +++ b/dm/tests/tls/conf/ca.pem @@ -1,17 +1,8 @@ -----BEGIN CERTIFICATE----- -MIICsTCCAZkCFBW2dYDlpiY0fQo4/BbVb4kWuLGMMA0GCSqGSIb3DQEBCwUAMBQx -EjAQBgNVBAMMCWxvY2FsaG9zdDAgFw0yNDExMTkwMjUxNThaGA8yMjk4MDkwNDAy -NTE1OFowFDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOC -AQ8AMIIBCgKCAQEAzNlitARpbbz2zDeze6X70vGg5Ti0t7xz7MsAAl9ESyf6i/XD -zBDGN5w18YaPUsXbOFz5dJeWtVK8xheJCN19RwYUGWf6BGuLohCA7ygSy43ICVNJ -xeW47xev4RcACu/C2UelBaVQLleSKsMFtEwuGJTlGL9nfxR3NKw7JUYntcy0+1c9 -GJQunFp8UU2ymENJqlIwhtwy5Z56oWRGS34rVdyhmlQ/OrGEinTyyoZBxWJmzbvw -Cqi1+IZKmwobGVSIuNdKxLwn/KNRxXE+tWwVy/+Dq5U3mt4zdBcRHQsSuqxPzS+6 -/3MivAN+5hGebHv1UyfKEQTACyKc8YHryf5SPwIDAQABMA0GCSqGSIb3DQEBCwUA -A4IBAQCPf/9isj85Ncsw7v01apIuklLzHByYZdkD0yY3Igrw2teaOGNKcTuXHiPM -v4QpV5KkkRtL+okNR6hL1BoY4OYkDRD67b8bHqr20JiYUriA1aZ5paW2MVXya8eE -T12fZYpNuYWKfd2YQiWSUuEEhCbr6cBkw9A4oyhVx5+QAFVstysm4U4dI9fn89uK -Ywtcc0ebk7FwjwGfchKW9mGG7mSOcZdGiKxYFxnI+wIf0nwjJvRraQbNpN2RuqSx -Oc/oG8QdRklQ6GnDKEwWb4P2z4r3n16O0ntcyXt03h1pFxrGTNhQ+pwbormCmN/c -t2udn5DvywhPljodj8vnprDScR4g +MIIBGDCBwAIJAOjYXLFw5V1HMAoGCCqGSM49BAMCMBQxEjAQBgNVBAMMCWxvY2Fs +aG9zdDAgFw0yMDAzMTcxMjAwMzNaGA8yMjkzMTIzMTEyMDAzM1owFDESMBAGA1UE +AwwJbG9jYWxob3N0MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEglCIJD8uVBfD +kuM+UQP+VA7Srbz17WPLA0Sqc+sQ2p6fT6HYKCW60EXiZ/yEC0925iyVbXEEbX4J +xCc2Heow5TAKBggqhkjOPQQDAgNHADBEAiAILL3Zt/3NFeDW9c9UAcJ9lc92E0ZL +GNDuH6i19Fex3wIgT0ZMAKAFSirGGtcLu0emceuk+zVKjJzmYbsLdpj/JuQ= -----END CERTIFICATE----- diff --git a/dm/tests/tls/conf/dm.key b/dm/tests/tls/conf/dm.key index 38105f87366..dfdc077bc4d 100644 --- a/dm/tests/tls/conf/dm.key +++ b/dm/tests/tls/conf/dm.key @@ -1,27 +1,8 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEpAIBAAKCAQEAuWvI0YtEvBAkO4dHtGdwRJ2aTCsB0jjWrSBz55pMn+4ht1Sl -9YECNx6+1SSenuzfDvvNkplIeI3RI+8QjS6by0yeBQxxp60Zh/HrS2CnRbjEXjig -PnLxL/xnR1MxtlzmKH3wdq0Z7jRSYwHxZ15uKvECq8wmnJuAnKyMdOc3kQYcdD/1 -tAQ8HqKOZr6wVfjj4ApZuctqGNOmkiCgNULljQqhiVbdx9o+xTAx+vBxHpUbXpTf -BZYA8VlKh9As6wwQbNW5GgOItGanulElZRaq8Ql/sk+mEVuNlkownkQgmtL/xnbH -HUehn/OugogrOcR6MZAkEuxN/Sl03VTbHXxP0QIDAQABAoIBAQClE2qD84NMEurO -/I8/818mF1HJQfirGeM12tCVfu986tqlhv0BZst8Gyk2RRXa03N6uX65ZzH38foi -DbIknzwgcKO2Ru/goeE7AIvEXdZN4kMgxzUSo+5KFGI2Q8DcZDScFBNC9LVDNt2q -O46Zwp4+KLeCD4zInECSO6QIZ9k2iEA2Tg1/oxxMO1hZAZv/RBcKWWFHc9sDe8qT -p2proaQBu4Qr4az4I0bKZoQK5QyZwbvd8EPCsTdqRY6KLBNkDn6rsovyTFJ/fzgk -lHl1Ef5zv89i4lX/JJgfNI+YxzX/SFJDGcWmvwQvFsbaSW+SffNGaYevWK9R4idq -vBBj5iKJAoGBAN+3Bc6ap7Vbu8TckbOifRH1u+3v1Awo5gP7XnUJQ1WzvI5qJeit -bkYJ4+SLDrGspdTpeh2LqyjI4f+2XiV67oyIFRhcra9B562COMC3jAXb6B54aT4W -fWBvzBLqsX6V1VnicgXHeF++DFI+54wlULso3X44EUxrk8/UyiY4KVZzAoGBANQu -Bdyo+6pmhHMMqegwhYqmTh/Tqq2Xy4aHMzO/3Ft0ZE/C6VkHFH73xXFUkYkdMNPx -KNLkES7TFmI9jbwBY806NYq8gmFXY/pDjZPfhNr0cQTUIoM/NPL67er1CFKuL0zg -kduUZEP876B+6WLkDw0oh+c8QA8QAL/fd5tVbOurAoGADOucGT+TWgUPRiJbA93p -jUT88pP8Uit2hVpaLTY4vlVgGXWF9Pc3PDJMCCID2j0XYsGNPPGZFATuEH+u1klQ -R6uAVGTd3xZWVA4GSxMOsjPuJvxbT6uq2Av85EbLs3zh0owie7T2fckTGxjo9I4f -KIAGssh4L0qzQbiPaD+sG4MCgYADwxUjhnN/3/V1PMI+da3IeY5lDhG2pjxblEYa -Isq7bGLi7yfIBlIctjPwAHncHoJhH6ODqbuMQv3gnArJTJUCbIfKf/En3K7PHyeB -ebMXVsIyPbim7xX3QO+6/ME7Bg+co1ayt/zXH8L364gj0rtZlmBifzT7j2MUA9hR -DsCxpQKBgQCvPP4d0zZTRobdJtPaLnAgMq6F0RPhzoDzVlcKqXJzcELflSvusdel -L2CPOFs8KXe2BxlJpUUwmhuTIHuwWahkMxx/Dhd5rJ9qlQtDygTKNsaJgnflPscF -mFZkIeDqpeE05X9tcvWdA+QSgpmY7AmjwvlOfCTi0Ftx1s7HPfnhTQ== ------END RSA PRIVATE KEY----- +-----BEGIN EC PARAMETERS----- +BggqhkjOPQMBBw== +-----END EC PARAMETERS----- +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEICF/GDtVxhTPTP501nOu4jgwGSDY01xN+61xd9MfChw+oAoGCCqGSM49 +AwEHoUQDQgAEgQOv5bQO7xK16vZWhwJqlz2vl19+AXW2Ql7KQyGiBJVSvLbyDLOr +kIeFlHN04iqQ39SKSOSfeGSfRt6doU6IcA== +-----END EC PRIVATE KEY----- diff --git a/dm/tests/tls/conf/dm.pem b/dm/tests/tls/conf/dm.pem index e4678a7f2f9..d4f846e3a22 100644 --- a/dm/tests/tls/conf/dm.pem +++ b/dm/tests/tls/conf/dm.pem @@ -1,18 +1,10 @@ -----BEGIN CERTIFICATE----- -MIIC+zCCAeOgAwIBAgIUDg53fesx1gIIYkGiIDbjngxBd40wDQYJKoZIhvcNAQEL -BQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MCAXDTI0MTExOTAyNTIxOFoYDzIyOTgw -OTA0MDI1MjE4WjANMQswCQYDVQQDDAJkbTCCASIwDQYJKoZIhvcNAQEBBQADggEP -ADCCAQoCggEBALlryNGLRLwQJDuHR7RncESdmkwrAdI41q0gc+eaTJ/uIbdUpfWB -AjcevtUknp7s3w77zZKZSHiN0SPvEI0um8tMngUMcaetGYfx60tgp0W4xF44oD5y -8S/8Z0dTMbZc5ih98HatGe40UmMB8WdebirxAqvMJpybgJysjHTnN5EGHHQ/9bQE -PB6ijma+sFX44+AKWbnLahjTppIgoDVC5Y0KoYlW3cfaPsUwMfrwcR6VG16U3wWW -APFZSofQLOsMEGzVuRoDiLRmp7pRJWUWqvEJf7JPphFbjZZKMJ5EIJrS/8Z2xx1H -oZ/zroKIKznEejGQJBLsTf0pdN1U2x18T9ECAwEAAaNKMEgwGgYDVR0RBBMwEYIJ -bG9jYWxob3N0hwR/AAABMAsGA1UdDwQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcD -AgYIKwYBBQUHAwEwDQYJKoZIhvcNAQELBQADggEBADMC8yK209WqhIBB+48kxpgU -aIxpu1CJXCHU3M476mZFYXpzfGQvxHJJSzLb6ZdfvK2TruWd1STwlnDanAH9MAxI -nFDtI7TLlpBvtSu9SB8fQLeEY5QUl5ZtBHpLKXkVQ/GMOWKP4q7L7zE7gaeaiVlC -VcvEF/+yeaj/GBu0RE5ia23V1HbthRLcyZnGvTYsyEVwTf+ZbJso3hbbN83WaKiL -avXD267fnfeJbL0No24QWFud/og28pdS1fM7ssyZZYI9k2VlrDPQF2jHQRfIu/bt -NyTNSt8etReb1rYCNKsFK8IPTkWwJmyOCDF+dhFLgA/z9KY2MZh1M+9gR9I5GNY= +MIIBZDCCAQqgAwIBAgIJAIT/lgXUc1JqMAoGCCqGSM49BAMCMBQxEjAQBgNVBAMM +CWxvY2FsaG9zdDAgFw0yMDAzMTcxMjAwMzNaGA8yMjkzMTIzMTEyMDAzM1owDTEL +MAkGA1UEAwwCZG0wWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAASBA6/ltA7vErXq +9laHAmqXPa+XX34BdbZCXspDIaIElVK8tvIMs6uQh4WUc3TiKpDf1IpI5J94ZJ9G +3p2hTohwo0owSDAaBgNVHREEEzARgglsb2NhbGhvc3SHBH8AAAEwCwYDVR0PBAQD +AgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMCBggrBgEFBQcDATAKBggqhkjOPQQDAgNI +ADBFAiEAx6ljJ+tNa55ypWLGNqmXlB4UdMmKmE4RSKJ8mmEelfECIG2ZmCE59rv5 +wImM6KnK+vM2QnEiISH3PeYyyRzQzycu -----END CERTIFICATE----- diff --git a/dm/tests/tls/conf/generate_tls.sh b/dm/tests/tls/conf/generate_tls.sh index d6090968da7..8f8410690e0 100644 --- a/dm/tests/tls/conf/generate_tls.sh +++ b/dm/tests/tls/conf/generate_tls.sh @@ -16,12 +16,12 @@ DNS.1 = localhost IP.1 = 127.0.0.1 EOF -openssl genrsa -out "ca.key" +openssl ecparam -out "ca.key" -name prime256v1 -genkey openssl req -new -batch -sha256 -subj '/CN=localhost' -key "ca.key" -out "ca.csr" openssl x509 -req -sha256 -days 100000 -in "ca.csr" -signkey "ca.key" -out "ca.pem" 2>/dev/null for role in dm other; do - openssl genrsa -out "$role.key" + openssl ecparam -out "$role.key" -name prime256v1 -genkey openssl req -new -batch -sha256 -subj "/CN=${role}" -key "$role.key" -out "$role.csr" openssl x509 -req -sha256 -days 100000 -extensions EXT -extfile "ipsan.cnf" -in "$role.csr" -CA "ca.pem" -CAkey "ca.key" -CAcreateserial -out "$role.pem" 2>/dev/null done diff --git a/dm/tests/tls/conf/other.key b/dm/tests/tls/conf/other.key index 718b7bb6ba7..ee95ca5faa5 100644 --- a/dm/tests/tls/conf/other.key +++ b/dm/tests/tls/conf/other.key @@ -1,27 +1,8 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEpAIBAAKCAQEAw1kLMqpCyhCuviPrJkQFoZIO4ocbZO/WPju2L1B9o3QEaqzF -Zs7jTCVVtLynryw12OSay+t2JuTED4hHfSlR0RjjfHj2rV9CRyLty5Njf2ZrmX+V -pUNV/c1AKo4ACZbjy7UpQZ0dQEA6bHE3THq7oLEhREL0/QkZKOEoy8CEZ8Bk2DaR -eOepFzNX9ZSuFZ/Kfljl9skJ3D/0Sg0RjGeIddhITbW8SuR7NEEcHbjweEYCuoMr -W+90FYBx0GxAACAqzyO+QzsWaq7WbDkUYeEJjwCZ9jJthYsXGnfwpLde9A4poJs5 -8LvHLq1KqDOuVXdyjQFBjZLWBVQfOTdbvzTenwIDAQABAoIBACO6G1Qu03xFRNA9 -nXooIqeEIPjJIPd67cIABbftjqzgQRaJdqjw2Z1Jjr05a6mb+lUqqiz1aWx/lmQI -1ppLfk0xshlHvkXDEEoWV0tOL3pTbwZp0SKiiA2dBklCIzudvxe4kE0RwRHJLWY7 -5pM0NzCSdLUZUdkdWt6hH2b0VMLsmIq+6L2Th8Ns3Z88+UL2UAEK0qex7TKRNRlc -wRDf9pydcqUdPu+3IBvyww/MVXJfEiZHPc9kI8Nrt0Puo/3qnhQG//2jLPJFQUod -vXOcS1bX7UAYP2NQ1YNrHloX079D7OmSN+C03pvS62fVGWwDhVQiXd56VimPEzB8 -Rz0fgqECgYEA/ldf+hqPBvXNrBAF8Z0rh/kgBSv/rGvHCOTijNL9i5dWOx1gnF4d -kx94XPmGLKcnDiSdg8tlzc9FDW17t9gmycF4oHdAmZkWy1R6rH4OcVifuQ3DqLsg -qlDQZpjySsBQN4kY/lbrmKAsXzVWDecBIrMWM6IYCQFrTaC4A9rmI5cCgYEAxJ8t -vnGbB2QQbf+PJxef+xy4TtGb2lT0SrnrNBVcVFcYbe3/AdT7iXviGULhtIJiIykR -hcMoZu9mhWnydKpf5hn9jxE9XoB4khcfrjcVIGk6DBgHk6IhGxDOgGispgNO9uMH -WWWu4KtuyBsocG/k9jJZDJ7Mltpc0h4gQyT23jkCgYBDNIk6x+vFFgwri+xWJIaN -HknUtASR5cSo1/n45gNy8CEr655Mkx3TczxVrOVERUKvnQM2APe8gykm3CgVsvrE -0udx1zDtpSL9xSvJCmH8l0NVHYr2ucNu3X3DrmRPVjOcCRDem88S/DfdTHWZpNGK -/rE+IXWiY4qb73zQH/YxrwKBgQCyxVPvGxXss2Noz/qVfZp809EUAOrmqWVwZ/hd -Y6qOGJ5inQjF3qNRjqy1cSUMXZvjAh0ndJjQdozNctX6k0ocSbdMFE2rFb4KfjHU -0cpIrMQVaOfCqJ/XHwHSqNmJG+r4+NSeyKAnMFfKYz7ydBpGauKiYeP8qM/KT4/f -3MUEqQKBgQC2zESZWJR49qMhsQlDREeD/pZjbX76DoVfUOuvNpI4mFP9Gj1cp92V -8sBVSREHeRoN/DltT8TtJgGXSzfW8pQWRxQh2BguryHYROJsXA0aVNgQPHmBLz49 -V6UOSgXetUnm9TSkSGcMKDt93NwN1e6sJ3OJruAMTG3Y6guKN20sTg== ------END RSA PRIVATE KEY----- +-----BEGIN EC PARAMETERS----- +BggqhkjOPQMBBw== +-----END EC PARAMETERS----- +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEICzbWZZ7dtha0lGXlBiP3QjCurPs5ExsR5thIQCcKCKeoAoGCCqGSM49 +AwEHoUQDQgAEt5z9ACtEnsqv0ZPFx5YJhvBNQZJCEb75ZS/kDBiPoISea1HMt1w8 +4ZkeWW+SBCwt0RtwzVPRq9VUGWaFRUOwdQ== +-----END EC PRIVATE KEY----- diff --git a/dm/tests/tls/conf/other.pem b/dm/tests/tls/conf/other.pem index c164884e341..5710106ffe6 100644 --- a/dm/tests/tls/conf/other.pem +++ b/dm/tests/tls/conf/other.pem @@ -1,19 +1,10 @@ -----BEGIN CERTIFICATE----- -MIIC/jCCAeagAwIBAgIUDg53fesx1gIIYkGiIDbjngxBd4wwDQYJKoZIhvcNAQEL -BQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MCAXDTI0MTExOTAyNTE0M1oYDzIyOTgw -OTA0MDI1MTQzWjAQMQ4wDAYDVQQDDAVvdGhlcjCCASIwDQYJKoZIhvcNAQEBBQAD -ggEPADCCAQoCggEBAMNZCzKqQsoQrr4j6yZEBaGSDuKHG2Tv1j47ti9QfaN0BGqs -xWbO40wlVbS8p68sNdjkmsvrdibkxA+IR30pUdEY43x49q1fQkci7cuTY39ma5l/ -laVDVf3NQCqOAAmW48u1KUGdHUBAOmxxN0x6u6CxIURC9P0JGSjhKMvAhGfAZNg2 -kXjnqRczV/WUrhWfyn5Y5fbJCdw/9EoNEYxniHXYSE21vErkezRBHB248HhGArqD -K1vvdBWAcdBsQAAgKs8jvkM7Fmqu1mw5FGHhCY8AmfYybYWLFxp38KS3XvQOKaCb -OfC7xy6tSqgzrlV3co0BQY2S1gVUHzk3W7803p8CAwEAAaNKMEgwGgYDVR0RBBMw -EYIJbG9jYWxob3N0hwR/AAABMAsGA1UdDwQEAwIFoDAdBgNVHSUEFjAUBggrBgEF -BQcDAgYIKwYBBQUHAwEwDQYJKoZIhvcNAQELBQADggEBAKaIFX66bIZRaiaLoQqw -nvremDwYX16RTDRXPmF8Jp0Rs7Xm8WeQF4B4HDhwWtkti9muecN2CzeYTlgzkbaK -yMG2mndmbO/W6pWuWObrRZF9TA+4jIQ/r5NuBcoVLwe0s4/EWxPqZyndNxBZCNr+ -cpUlcG4QScrFfuj87K6yi+z1t7w0F1YMoRt0uD2MspWG2TQv7tR1iWMQRy9rVF9E -+GWEMPQRdMAG1fkTtlRd+VgcZXGWF7XEYWK0nSIBBXJf9mQGsl6XZcBqRMY2vgTa -Jk0Nix/aUsVP1cclVNQ9vL5qWUBnr9/3TseyqHcKFOrM+E6+vXFlVs33WJ2awwqP -HZ4= +MIIBZzCCAQ2gAwIBAgIJAIT/lgXUc1JrMAoGCCqGSM49BAMCMBQxEjAQBgNVBAMM +CWxvY2FsaG9zdDAgFw0yMDAzMTcxMjAwMzNaGA8yMjkzMTIzMTEyMDAzM1owEDEO +MAwGA1UEAwwFb3RoZXIwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAAS3nP0AK0Se +yq/Rk8XHlgmG8E1BkkIRvvllL+QMGI+ghJ5rUcy3XDzhmR5Zb5IELC3RG3DNU9Gr +1VQZZoVFQ7B1o0owSDAaBgNVHREEEzARgglsb2NhbGhvc3SHBH8AAAEwCwYDVR0P +BAQDAgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMCBggrBgEFBQcDATAKBggqhkjOPQQD +AgNIADBFAiEA34/Vz7SaJWqYOgOLyr+y1OwiT9R7yTgBQCSSvGC+HpsCIA20BhNe +RnicYz+9qOQRxAFP1wpIyMMgOK4tKuZhx+/J -----END CERTIFICATE----- From 8f360eda05ffbc8020aa2a054aadb9525fa3f6c4 Mon Sep 17 00:00:00 2001 From: Jiaqiang Huang <96465211+River2000i@users.noreply.github.com> Date: Wed, 20 Nov 2024 18:23:02 +0800 Subject: [PATCH 12/28] fix --- dm/tests/_utils/run_downstream_cluster | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/dm/tests/_utils/run_downstream_cluster b/dm/tests/_utils/run_downstream_cluster index 997fbc6f14a..85a6eb36653 100755 --- a/dm/tests/_utils/run_downstream_cluster +++ b/dm/tests/_utils/run_downstream_cluster @@ -34,12 +34,13 @@ EOF --data-dir "$WORK_DIR/pd" & # wait until PD is online... i=0 - while curl "http://$PD_ADDR/pd/api/v1/version"; do + while ! curl -s "http://$PD_ADDR/pd/api/v1/version" > /dev/null; do i=$((i + 1)) if [ "$i" -gt 20 ]; then echo 'Failed to start PD' return 1 fi + echo 'Waiting for PD ready' sleep 3 done echo 'Start PD success' @@ -61,6 +62,7 @@ start_tikv() { echo 'Failed to initialize TiKV cluster' return 1 fi + echo 'Waiting for TiKV ready' sleep 5 done echo 'Start TiKV success' @@ -77,12 +79,13 @@ start_tidb() { --log-file "$WORK_DIR/tidb.log" & i=0 - while curl "http://$TIDB_IP:10080/status"; do + while ! curl -s "http://$TIDB_IP:10080/status" > /dev/null; do i=$((i + 1)) if [ "$i" -gt 50 ]; then echo 'Failed to start TiDB' return 1 fi + echo 'Waiting for TiDB ready' sleep 3 done echo 'Start TiDB success' From edec99b13f217aa5a1f7c95027336f2a0ac7230c Mon Sep 17 00:00:00 2001 From: Jiaqiang Huang <96465211+River2000i@users.noreply.github.com> Date: Thu, 21 Nov 2024 10:57:39 +0800 Subject: [PATCH 13/28] check curl get 200 --- dm/tests/_utils/run_downstream_cluster | 78 +++++++++++++++----------- 1 file changed, 46 insertions(+), 32 deletions(-) diff --git a/dm/tests/_utils/run_downstream_cluster b/dm/tests/_utils/run_downstream_cluster index 85a6eb36653..b21718135b8 100755 --- a/dm/tests/_utils/run_downstream_cluster +++ b/dm/tests/_utils/run_downstream_cluster @@ -33,17 +33,21 @@ EOF --config "$WORK_DIR/pd.toml" \ --data-dir "$WORK_DIR/pd" & # wait until PD is online... - i=0 - while ! curl -s "http://$PD_ADDR/pd/api/v1/version" > /dev/null; do - i=$((i + 1)) - if [ "$i" -gt 20 ]; then - echo 'Failed to start PD' - return 1 - fi - echo 'Waiting for PD ready' - sleep 3 - done - echo 'Start PD success' + i=0 + while true; do + response=$(curl -s -o /dev/null -w "%{http_code}" "http://$PD_ADDR/pd/api/v1/version") + if [ "$response" -eq 200 ]; then + echo 'Start PD success' + break + fi + i=$((i + 1)) + if [ "$i" -gt 20 ]; then + echo 'Failed to start PD' + return 1 + fi + echo 'Waiting for PD ready...' + sleep 3 + done } start_tikv() { @@ -56,16 +60,22 @@ start_tikv() { --log-file "$WORK_DIR/tikv.log" \ --log-level info \ -s "$WORK_DIR/tikv" & - while curl "http://$PD_ADDR/pd/api/v1/cluster/status" | tee /dev/stderr | grep '"is_initialized": true'; do - i=$((i + 1)) - if [ "$i" -gt 20 ]; then - echo 'Failed to initialize TiKV cluster' - return 1 - fi - echo 'Waiting for TiKV ready' - sleep 5 - done - echo 'Start TiKV success' + # wait until TiKV is online... + i=0 + while true; do + response=$(curl -s -o /dev/null -w "%{http_code}" "http://$PD_ADDR/pd/api/v1/cluster/status") + if curl -s "http://$PD_ADDR/pd/api/v1/cluster/status" | grep '"is_initialized": true'; then + echo 'Start TiKV success' + break + fi + i=$((i + 1)) + if [ "$i" -gt 20 ]; then + echo 'Failed to initialize TiKV cluster' + return 1 + fi + echo 'Waiting for TiKV ready...' + sleep 5 + done } start_tidb() { @@ -77,18 +87,22 @@ start_tidb() { --store tikv \ --path "$PD_ADDR" \ --log-file "$WORK_DIR/tidb.log" & - + # wait until TiDB is online... i=0 - while ! curl -s "http://$TIDB_IP:10080/status" > /dev/null; do - i=$((i + 1)) - if [ "$i" -gt 50 ]; then - echo 'Failed to start TiDB' - return 1 - fi - echo 'Waiting for TiDB ready' - sleep 3 - done - echo 'Start TiDB success' + while true; do + response=$(curl -s -o /dev/null -w "%{http_code}" "http://$TIDB_IP:10080/status") + if [ "$response" -eq 200 ]; then + echo 'Start TiDB success' + break + fi + i=$((i + 1)) + if [ "$i" -gt 50 ]; then + echo 'Failed to start TiDB' + return 1 + fi + echo 'Waiting for TiDB ready...' + sleep 3 + done } start_pd From 24a3b14b328afb59fc1a12a6f30589d172fca884 Mon Sep 17 00:00:00 2001 From: Jiaqiang Huang <96465211+River2000i@users.noreply.github.com> Date: Thu, 21 Nov 2024 11:32:03 +0800 Subject: [PATCH 14/28] fmt --- dm/tests/_utils/run_downstream_cluster | 90 +++++++++++++------------- 1 file changed, 45 insertions(+), 45 deletions(-) diff --git a/dm/tests/_utils/run_downstream_cluster b/dm/tests/_utils/run_downstream_cluster index b21718135b8..62dc1548c50 100755 --- a/dm/tests/_utils/run_downstream_cluster +++ b/dm/tests/_utils/run_downstream_cluster @@ -33,21 +33,21 @@ EOF --config "$WORK_DIR/pd.toml" \ --data-dir "$WORK_DIR/pd" & # wait until PD is online... - i=0 - while true; do - response=$(curl -s -o /dev/null -w "%{http_code}" "http://$PD_ADDR/pd/api/v1/version") - if [ "$response" -eq 200 ]; then - echo 'Start PD success' - break - fi - i=$((i + 1)) - if [ "$i" -gt 20 ]; then - echo 'Failed to start PD' - return 1 - fi - echo 'Waiting for PD ready...' - sleep 3 - done + i=0 + while true; do + response=$(curl -s -o /dev/null -w "%{http_code}" "http://$PD_ADDR/pd/api/v1/version") + if [ "$response" -eq 200 ]; then + echo 'Start PD success' + break + fi + i=$((i + 1)) + if [ "$i" -gt 20 ]; then + echo 'Failed to start PD' + return 1 + fi + echo 'Waiting for PD ready...' + sleep 3 + done } start_tikv() { @@ -60,22 +60,22 @@ start_tikv() { --log-file "$WORK_DIR/tikv.log" \ --log-level info \ -s "$WORK_DIR/tikv" & - # wait until TiKV is online... + # wait until TiKV is online... i=0 - while true; do - response=$(curl -s -o /dev/null -w "%{http_code}" "http://$PD_ADDR/pd/api/v1/cluster/status") - if curl -s "http://$PD_ADDR/pd/api/v1/cluster/status" | grep '"is_initialized": true'; then - echo 'Start TiKV success' - break - fi - i=$((i + 1)) - if [ "$i" -gt 20 ]; then - echo 'Failed to initialize TiKV cluster' - return 1 - fi - echo 'Waiting for TiKV ready...' - sleep 5 - done + while true; do + response=$(curl -s -o /dev/null -w "%{http_code}" "http://$PD_ADDR/pd/api/v1/cluster/status") + if curl -s "http://$PD_ADDR/pd/api/v1/cluster/status" | grep '"is_initialized": true'; then + echo 'Start TiKV success' + break + fi + i=$((i + 1)) + if [ "$i" -gt 20 ]; then + echo 'Failed to initialize TiKV cluster' + return 1 + fi + echo 'Waiting for TiKV ready...' + sleep 5 + done } start_tidb() { @@ -87,22 +87,22 @@ start_tidb() { --store tikv \ --path "$PD_ADDR" \ --log-file "$WORK_DIR/tidb.log" & - # wait until TiDB is online... + # wait until TiDB is online... i=0 - while true; do - response=$(curl -s -o /dev/null -w "%{http_code}" "http://$TIDB_IP:10080/status") - if [ "$response" -eq 200 ]; then - echo 'Start TiDB success' - break - fi - i=$((i + 1)) - if [ "$i" -gt 50 ]; then - echo 'Failed to start TiDB' - return 1 - fi - echo 'Waiting for TiDB ready...' - sleep 3 - done + while true; do + response=$(curl -s -o /dev/null -w "%{http_code}" "http://$TIDB_IP:10080/status") + if [ "$response" -eq 200 ]; then + echo 'Start TiDB success' + break + fi + i=$((i + 1)) + if [ "$i" -gt 50 ]; then + echo 'Failed to start TiDB' + return 1 + fi + echo 'Waiting for TiDB ready...' + sleep 3 + done } start_pd From aae5267e6e7122a2104403591acb6a69b1eb08ba Mon Sep 17 00:00:00 2001 From: Jiaqiang Huang <96465211+River2000i@users.noreply.github.com> Date: Thu, 21 Nov 2024 11:51:23 +0800 Subject: [PATCH 15/28] use env_var set mysql cli password --- dm/tests/_utils/run_sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dm/tests/_utils/run_sql b/dm/tests/_utils/run_sql index 555fc4a2337..66d91a40a28 100755 --- a/dm/tests/_utils/run_sql +++ b/dm/tests/_utils/run_sql @@ -18,5 +18,5 @@ fi if [[ $# -ge 4 ]]; then echo "$1" | iconv -f utf8 -t $4 | mysql -u$user -h127.0.0.1 -P$2 -p$3 --default-character-set $4 -E >>$OUTFILE else - mysql -u$user -h127.0.0.1 -P$2 -p$3 --default-character-set utf8 -E -e "$1" >>$OUTFILE + MYSQL_PWD=$3 mysql -u$user -h127.0.0.1 -P$2 --default-character-set utf8 -E -e "$1" >>$OUTFILE fi From 060bdf4c31a1dd659dcfd54d90664d9275823fd8 Mon Sep 17 00:00:00 2001 From: Jiaqiang Huang <96465211+River2000i@users.noreply.github.com> Date: Thu, 21 Nov 2024 12:22:53 +0800 Subject: [PATCH 16/28] debug --- dm/tests/_utils/run_downstream_cluster | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dm/tests/_utils/run_downstream_cluster b/dm/tests/_utils/run_downstream_cluster index 62dc1548c50..775d0baa7a9 100755 --- a/dm/tests/_utils/run_downstream_cluster +++ b/dm/tests/_utils/run_downstream_cluster @@ -33,9 +33,12 @@ EOF --config "$WORK_DIR/pd.toml" \ --data-dir "$WORK_DIR/pd" & # wait until PD is online... + echo "after start PD...." i=0 while true; do + echo "run curl before...." response=$(curl -s -o /dev/null -w "%{http_code}" "http://$PD_ADDR/pd/api/v1/version") + echo "run curl after...." if [ "$response" -eq 200 ]; then echo 'Start PD success' break From 2b5be5ea5ce53a383780c33d4832999718ced772 Mon Sep 17 00:00:00 2001 From: Jiaqiang Huang <96465211+River2000i@users.noreply.github.com> Date: Thu, 21 Nov 2024 12:47:55 +0800 Subject: [PATCH 17/28] debug --- dm/tests/_utils/run_downstream_cluster | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dm/tests/_utils/run_downstream_cluster b/dm/tests/_utils/run_downstream_cluster index 775d0baa7a9..b731bdd7b90 100755 --- a/dm/tests/_utils/run_downstream_cluster +++ b/dm/tests/_utils/run_downstream_cluster @@ -36,6 +36,10 @@ EOF echo "after start PD...." i=0 while true; do + echo "test curl 1" + curl www.baidu.com + echo "test curl 2" + curl http://$PD_ADDR echo "run curl before...." response=$(curl -s -o /dev/null -w "%{http_code}" "http://$PD_ADDR/pd/api/v1/version") echo "run curl after...." From 907d518d78a0c6d507a470ae41d1473e69d8b1a2 Mon Sep 17 00:00:00 2001 From: Jiaqiang Huang <96465211+River2000i@users.noreply.github.com> Date: Thu, 21 Nov 2024 13:20:21 +0800 Subject: [PATCH 18/28] debug --- dm/tests/_utils/run_downstream_cluster | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/dm/tests/_utils/run_downstream_cluster b/dm/tests/_utils/run_downstream_cluster index b731bdd7b90..307e7a6e345 100755 --- a/dm/tests/_utils/run_downstream_cluster +++ b/dm/tests/_utils/run_downstream_cluster @@ -42,6 +42,7 @@ EOF curl http://$PD_ADDR echo "run curl before...." response=$(curl -s -o /dev/null -w "%{http_code}" "http://$PD_ADDR/pd/api/v1/version") + echo "curl response: $response" echo "run curl after...." if [ "$response" -eq 200 ]; then echo 'Start PD success' @@ -69,12 +70,7 @@ start_tikv() { -s "$WORK_DIR/tikv" & # wait until TiKV is online... i=0 - while true; do - response=$(curl -s -o /dev/null -w "%{http_code}" "http://$PD_ADDR/pd/api/v1/cluster/status") - if curl -s "http://$PD_ADDR/pd/api/v1/cluster/status" | grep '"is_initialized": true'; then - echo 'Start TiKV success' - break - fi + while ! curl "http://$PD_ADDR/pd/api/v1/cluster/status" | tee /dev/stderr | grep '"is_initialized": true'; do i=$((i + 1)) if [ "$i" -gt 20 ]; then echo 'Failed to initialize TiKV cluster' @@ -98,6 +94,7 @@ start_tidb() { i=0 while true; do response=$(curl -s -o /dev/null -w "%{http_code}" "http://$TIDB_IP:10080/status") + echo "curl response: $response" if [ "$response" -eq 200 ]; then echo 'Start TiDB success' break From 8f0cfe079a6e4c04741789299c3b100ffb928798 Mon Sep 17 00:00:00 2001 From: Jiaqiang Huang <96465211+River2000i@users.noreply.github.com> Date: Thu, 21 Nov 2024 13:45:59 +0800 Subject: [PATCH 19/28] debug --- dm/tests/_utils/run_downstream_cluster | 1 + 1 file changed, 1 insertion(+) diff --git a/dm/tests/_utils/run_downstream_cluster b/dm/tests/_utils/run_downstream_cluster index 307e7a6e345..8a48f75c2b9 100755 --- a/dm/tests/_utils/run_downstream_cluster +++ b/dm/tests/_utils/run_downstream_cluster @@ -90,6 +90,7 @@ start_tidb() { --store tikv \ --path "$PD_ADDR" \ --log-file "$WORK_DIR/tidb.log" & + echo "Finish start TiDB" # wait until TiDB is online... i=0 while true; do From 30e3bfec3530b3e2dcbdc0158045958bebfe0173 Mon Sep 17 00:00:00 2001 From: Jiaqiang Huang <96465211+River2000i@users.noreply.github.com> Date: Thu, 21 Nov 2024 13:46:52 +0800 Subject: [PATCH 20/28] debug --- dm/tests/_utils/run_downstream_cluster | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dm/tests/_utils/run_downstream_cluster b/dm/tests/_utils/run_downstream_cluster index 8a48f75c2b9..15aceddfc4e 100755 --- a/dm/tests/_utils/run_downstream_cluster +++ b/dm/tests/_utils/run_downstream_cluster @@ -1,7 +1,7 @@ #!/bin/bash # tools to run a TiDB cluster # parameter 1: work directory -set -eu +set -eux WORK_DIR=$1 export PD_PEER_ADDR="127.0.0.1:2380" From 19b2ade7e244fc153a15d178ddb186f4825a8cb3 Mon Sep 17 00:00:00 2001 From: Jiaqiang Huang <96465211+River2000i@users.noreply.github.com> Date: Thu, 21 Nov 2024 15:32:15 +0800 Subject: [PATCH 21/28] fix --- dm/tests/_utils/run_downstream_cluster | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/dm/tests/_utils/run_downstream_cluster b/dm/tests/_utils/run_downstream_cluster index 15aceddfc4e..5ea7c13b67a 100755 --- a/dm/tests/_utils/run_downstream_cluster +++ b/dm/tests/_utils/run_downstream_cluster @@ -2,7 +2,7 @@ # tools to run a TiDB cluster # parameter 1: work directory set -eux -WORK_DIR=$1 +WORK_DIR=/tmp export PD_PEER_ADDR="127.0.0.1:2380" export PD_ADDR="127.0.0.1:2379" @@ -32,18 +32,13 @@ EOF --log-file "$WORK_DIR/pd.log" \ --config "$WORK_DIR/pd.toml" \ --data-dir "$WORK_DIR/pd" & + sleep 10 # wait until PD is online... echo "after start PD...." i=0 while true; do - echo "test curl 1" - curl www.baidu.com - echo "test curl 2" - curl http://$PD_ADDR - echo "run curl before...." response=$(curl -s -o /dev/null -w "%{http_code}" "http://$PD_ADDR/pd/api/v1/version") echo "curl response: $response" - echo "run curl after...." if [ "$response" -eq 200 ]; then echo 'Start PD success' break @@ -68,6 +63,7 @@ start_tikv() { --log-file "$WORK_DIR/tikv.log" \ --log-level info \ -s "$WORK_DIR/tikv" & + sleep 10 # wait until TiKV is online... i=0 while ! curl "http://$PD_ADDR/pd/api/v1/cluster/status" | tee /dev/stderr | grep '"is_initialized": true'; do @@ -90,7 +86,7 @@ start_tidb() { --store tikv \ --path "$PD_ADDR" \ --log-file "$WORK_DIR/tidb.log" & - echo "Finish start TiDB" + sleep 10 # wait until TiDB is online... i=0 while true; do @@ -116,3 +112,4 @@ start_tidb mysql -uroot -h127.0.0.1 -P4000 --default-character-set utf8 -e "CREATE USER 'test'@'%' IDENTIFIED BY '123456';" || true mysql -uroot -h127.0.0.1 -P4000 --default-character-set utf8 -e "GRANT ALL PRIVILEGES ON *.* TO 'test'@'%' WITH GRANT OPTION;" || true + From e5ac23ed596876e6480b2a8bee2a2f8eb9cb7e00 Mon Sep 17 00:00:00 2001 From: Jiaqiang Huang <96465211+River2000i@users.noreply.github.com> Date: Thu, 21 Nov 2024 15:33:16 +0800 Subject: [PATCH 22/28] fix --- dm/tests/_utils/run_downstream_cluster | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dm/tests/_utils/run_downstream_cluster b/dm/tests/_utils/run_downstream_cluster index 5ea7c13b67a..03f5adc8f68 100755 --- a/dm/tests/_utils/run_downstream_cluster +++ b/dm/tests/_utils/run_downstream_cluster @@ -2,7 +2,7 @@ # tools to run a TiDB cluster # parameter 1: work directory set -eux -WORK_DIR=/tmp +WORK_DIR=$1 export PD_PEER_ADDR="127.0.0.1:2380" export PD_ADDR="127.0.0.1:2379" @@ -112,4 +112,3 @@ start_tidb mysql -uroot -h127.0.0.1 -P4000 --default-character-set utf8 -e "CREATE USER 'test'@'%' IDENTIFIED BY '123456';" || true mysql -uroot -h127.0.0.1 -P4000 --default-character-set utf8 -e "GRANT ALL PRIVILEGES ON *.* TO 'test'@'%' WITH GRANT OPTION;" || true - From efcb1f1fa302b279cf3850004efc751e6e8ae149 Mon Sep 17 00:00:00 2001 From: Jiaqiang Huang <96465211+River2000i@users.noreply.github.com> Date: Thu, 21 Nov 2024 15:49:52 +0800 Subject: [PATCH 23/28] fmt --- dm/tests/_utils/run_downstream_cluster | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dm/tests/_utils/run_downstream_cluster b/dm/tests/_utils/run_downstream_cluster index 03f5adc8f68..e7fa513270c 100755 --- a/dm/tests/_utils/run_downstream_cluster +++ b/dm/tests/_utils/run_downstream_cluster @@ -63,7 +63,7 @@ start_tikv() { --log-file "$WORK_DIR/tikv.log" \ --log-level info \ -s "$WORK_DIR/tikv" & - sleep 10 + sleep 10 # wait until TiKV is online... i=0 while ! curl "http://$PD_ADDR/pd/api/v1/cluster/status" | tee /dev/stderr | grep '"is_initialized": true'; do From 59e6309ac65e848ec6e272a692eaf9d495755b9f Mon Sep 17 00:00:00 2001 From: Jiaqiang Huang <96465211+River2000i@users.noreply.github.com> Date: Thu, 21 Nov 2024 17:23:16 +0800 Subject: [PATCH 24/28] add log --- dm/tests/_utils/run_downstream_cluster | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dm/tests/_utils/run_downstream_cluster b/dm/tests/_utils/run_downstream_cluster index e7fa513270c..e47ba20b64f 100755 --- a/dm/tests/_utils/run_downstream_cluster +++ b/dm/tests/_utils/run_downstream_cluster @@ -56,6 +56,7 @@ EOF start_tikv() { echo "Starting TiKV..." mkdir -p "$WORK_DIR/tikv" + bin/tikv-server --version bin/tikv-server \ --pd "$PD_ADDR" \ -A "$TIKV_ADDR" \ @@ -79,6 +80,7 @@ start_tikv() { start_tidb() { echo "Starting TiDB..." + bin/tidb-serve --version bin/tidb-server \ -P 4000 \ --status 10080 \ From 2ed926bbf9e1b10873b54ab51f78cf265d4284e1 Mon Sep 17 00:00:00 2001 From: Jiaqiang Huang <96465211+River2000i@users.noreply.github.com> Date: Thu, 21 Nov 2024 18:04:00 +0800 Subject: [PATCH 25/28] fix --- dm/tests/_utils/run_downstream_cluster | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dm/tests/_utils/run_downstream_cluster b/dm/tests/_utils/run_downstream_cluster index e47ba20b64f..8da6e5c3240 100755 --- a/dm/tests/_utils/run_downstream_cluster +++ b/dm/tests/_utils/run_downstream_cluster @@ -80,7 +80,7 @@ start_tikv() { start_tidb() { echo "Starting TiDB..." - bin/tidb-serve --version + bin/tidb-server --version bin/tidb-server \ -P 4000 \ --status 10080 \ From ef7b1cb6f0b0b429f498c41f020a12d26d6f793d Mon Sep 17 00:00:00 2001 From: Jiaqiang Huang <96465211+River2000i@users.noreply.github.com> Date: Thu, 21 Nov 2024 21:57:32 +0800 Subject: [PATCH 26/28] debug --- dm/tests/_utils/run_downstream_cluster | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dm/tests/_utils/run_downstream_cluster b/dm/tests/_utils/run_downstream_cluster index 8da6e5c3240..4b10b348bbe 100755 --- a/dm/tests/_utils/run_downstream_cluster +++ b/dm/tests/_utils/run_downstream_cluster @@ -1,7 +1,7 @@ #!/bin/bash # tools to run a TiDB cluster # parameter 1: work directory -set -eux +set -ux WORK_DIR=$1 export PD_PEER_ADDR="127.0.0.1:2380" From 3efe23f88621d6e71bf0188b67287b7a6d13b1c5 Mon Sep 17 00:00:00 2001 From: Jiaqiang Huang <96465211+River2000i@users.noreply.github.com> Date: Fri, 22 Nov 2024 00:01:43 +0800 Subject: [PATCH 27/28] ignore check running dump task in running stage --- dm/tests/openapi/run.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/dm/tests/openapi/run.sh b/dm/tests/openapi/run.sh index aceae7c16b2..a0fd6f21a2e 100644 --- a/dm/tests/openapi/run.sh +++ b/dm/tests/openapi/run.sh @@ -214,9 +214,6 @@ function test_dump_task() { # start dump task success openapi_task_check "start_task_success" $task_name "" - run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ - "query-status $task_name" \ - "\"stage\": \"Running\"" 1 # wait dump task finish run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ From 1c60df94c539ed844cf6cfeeb79b1720eeeea1c6 Mon Sep 17 00:00:00 2001 From: Jiaqiang Huang <96465211+River2000i@users.noreply.github.com> Date: Fri, 22 Nov 2024 00:05:05 +0800 Subject: [PATCH 28/28] fix --- dm/tests/openapi/run.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dm/tests/openapi/run.sh b/dm/tests/openapi/run.sh index a0fd6f21a2e..a012b4c4109 100644 --- a/dm/tests/openapi/run.sh +++ b/dm/tests/openapi/run.sh @@ -27,7 +27,7 @@ function init_noshard_data() { function init_dump_data() { run_sql_source1 "CREATE TABLE openapi.t1(i TINYINT, j INT UNIQUE KEY);" - run_sql_source1 "INSERT INTO openapi.t1(i,j) VALUES (1, 2),(3,4);" + run_sql_source1 "INSERT INTO openapi.t1(i,j) VALUES (1, 2),(3,4),(5,6),(7,8),(9,10),(11,12),(13,14),(15,16),(17,18);" } function init_shard_data() { @@ -214,6 +214,9 @@ function test_dump_task() { # start dump task success openapi_task_check "start_task_success" $task_name "" + run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ + "query-status $task_name" \ + "\"stage\": \"Running\"" 1 # wait dump task finish run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \