diff --git a/.github/workflows/docker-jepsen.yml b/.github/workflows/docker-jepsen.yml index 6172b0b0..5c37b400 100644 --- a/.github/workflows/docker-jepsen.yml +++ b/.github/workflows/docker-jepsen.yml @@ -8,8 +8,8 @@ env: GO_VERSION: 1.25.6 jobs: - test: - name: jepsen + jepsen: + name: jepsen (8.0) runs-on: ubuntu-22.04 steps: - name: Set up Go 1.x @@ -26,10 +26,10 @@ jobs: go get -v -t -d ./... - name: Test - run: make base_img jepsen_base_img jepsen_test + run: make base_img_8.0 jepsen_base_img jepsen_test - uses: actions/upload-artifact@v7 if: failure() with: - name: logs + name: logs-jepsen path: tests/logs diff --git a/Makefile b/Makefile index 3bfc199a..24bdeea0 100644 --- a/Makefile +++ b/Makefile @@ -48,9 +48,9 @@ jepsen_test: timeout 600 docker exec mysync_zoo1_1 retriable_path_create.sh /test/ha_nodes/mysync_mysql1_1 timeout 600 docker exec mysync_zoo1_1 retriable_path_create.sh /test/ha_nodes/mysync_mysql2_1 timeout 600 docker exec mysync_zoo1_1 retriable_path_create.sh /test/ha_nodes/mysync_mysql3_1 - timeout 600 docker exec mysync_mysql1_1 sh -c "/var/lib/dist/base/generate_certs.sh mysync_mysql1_1.mysync_mysql_net && supervisorctl restart mysync && supervisorctl start mysqld" - timeout 600 docker exec mysync_mysql2_1 sh -c "/var/lib/dist/base/generate_certs.sh mysync_mysql2_1.mysync_mysql_net && supervisorctl restart mysync && supervisorctl start mysqld" - timeout 600 docker exec mysync_mysql3_1 sh -c "/var/lib/dist/base/generate_certs.sh mysync_mysql3_1.mysync_mysql_net && supervisorctl restart mysync && supervisorctl start mysqld" + timeout 600 docker exec mysync_mysql1_1 sh -c "/var/lib/dist/base/generate_certs.sh mysync_mysql1_1.mysync_mysql_net && supervisorctl restart mysync && supervisorctl start mysqld || true" + timeout 600 docker exec mysync_mysql2_1 sh -c "/var/lib/dist/base/generate_certs.sh mysync_mysql2_1.mysync_mysql_net && supervisorctl restart mysync && supervisorctl start mysqld || true" + timeout 600 docker exec mysync_mysql3_1 sh -c "/var/lib/dist/base/generate_certs.sh mysync_mysql3_1.mysync_mysql_net && supervisorctl restart mysync && supervisorctl start mysqld || true" timeout 600 docker exec mysync_mysql1_1 setup.sh timeout 600 bash ./tests/images/copy_keys.sh mkdir -p ./tests/logs @@ -60,7 +60,7 @@ jepsen_test: clean: docker ps | grep mysync | awk '{print $$1}' | xargs docker rm -f || true docker network ls | grep mysync | awk '{print $$1}' | xargs docker network rm || true - docker image ls | grep mysync | awk '{print $$3}' | xargs docker image rm --force || true + docker image ls | grep mysync | awk '{print $$2}' | xargs docker image rm --force || true rm -rf ./tests/logs local_clean: diff --git a/tests/features/external_replication_multiple.feature b/tests/features/external_replication_multiple.feature index e423ae2f..77bb2820 100644 --- a/tests/features/external_replication_multiple.feature +++ b/tests/features/external_replication_multiple.feature @@ -4,7 +4,8 @@ Feature: external replication switchover Given cluster environment is """ MYSYNC_REPLICATION_REPAIR_COOLDOWN=30s - """ + MYSYNC_REPLICATION_REPAIR_MAX_ATTEMPTS=1 + """ Given cluster is up and running Then mysql host "mysql1" should be master And mysql host "mysql2" should be replica of "mysql1" diff --git a/tests/images/jepsen-compose.yml b/tests/images/jepsen-compose.yml index 194e4be0..fd8b2f7e 100644 --- a/tests/images/jepsen-compose.yml +++ b/tests/images/jepsen-compose.yml @@ -2,8 +2,7 @@ version: '2.3' services: zoo1: - build: - context: ./zookeeper_jepsen + build: ./zookeeper_jepsen hostname: mysync_zookeeper1_1 domainname: mysync_mysql_net container_name: mysync_zoo1_1 @@ -28,8 +27,7 @@ services: ipv4_address: 192.168.233.10 zoo2: - build: - context: ./zookeeper_jepsen + build: ./zookeeper_jepsen hostname: mysync_zookeeper2_1 domainname: mysync_mysql_net container_name: mysync_zoo2_1 @@ -54,8 +52,7 @@ services: ipv4_address: 192.168.233.11 zoo3: - build: - context: ./zookeeper_jepsen + build: ./zookeeper_jepsen hostname: mysync_zookeeper3_1 domainname: mysync_mysql_net container_name: mysync_zoo3_1 diff --git a/tests/images/jepsen_common/Dockerfile b/tests/images/jepsen_common/Dockerfile index 759111d6..47c7c8d5 100644 --- a/tests/images/jepsen_common/Dockerfile +++ b/tests/images/jepsen_common/Dockerfile @@ -1,4 +1,4 @@ -FROM mysync-test-base:latest +FROM mysync-test-base8.0:latest RUN (yes | ssh-keygen -m PEM -t rsa -b 2048 -N '' -f /root/test_ssh_key -C root@mysync_jepsen_1 || true) && \ eval `ssh-agent -s` && cp /root/test_ssh_key.pub /root/.ssh/authorized_keys && \ cp /root/test_ssh_key.pub /root/.ssh/id_rsa.pub && \ diff --git a/tests/images/jepsen_main/Dockerfile b/tests/images/jepsen_main/Dockerfile index 0ca1b6a5..a0eaaeb5 100644 --- a/tests/images/jepsen_main/Dockerfile +++ b/tests/images/jepsen_main/Dockerfile @@ -1,11 +1,10 @@ FROM mysync-jepsen-test-base:latest ENV LEIN_ROOT=1 COPY . /var/lib/dist/jepsen_main -RUN apt-get -qq update && apt-get install libjna-java \ +RUN apt-get -qq update && apt-get install -y libjna-java \ gnuplot \ wget \ openjdk-21-jre-headless && \ - update-alternatives --set java /usr/lib/jvm/java-21-openjdk-amd64/bin/java && \ chmod 600 /root/.ssh/id_rsa && \ wget https://raw.githubusercontent.com/technomancy/leiningen/2.12.0/bin/lein -O /usr/bin/lein && \ chmod +x /usr/bin/lein && \ diff --git a/tests/images/mysql/mysync.yaml b/tests/images/mysql/mysync.yaml index 5d1557c2..f1a37004 100644 --- a/tests/images/mysql/mysync.yaml +++ b/tests/images/mysql/mysync.yaml @@ -61,6 +61,7 @@ exclude_users: - 'monitor' - 'event_scheduler' replication_repair_cooldown: ${MYSYNC_REPLICATION_REPAIR_COOLDOWN:-10s} +replication_repair_max_attempts: ${MYSYNC_REPLICATION_REPAIR_MAX_ATTEMPTS:-3} replication_repair_aggressive_mode: ${MYSYNC_REPLICATION_REPAIR_AGGRESSIVE_MODE:-false} test_filesystem_readonly_file: /tmp/readonly replication_channel: '' diff --git a/tests/images/mysql_jepsen/Dockerfile b/tests/images/mysql_jepsen/Dockerfile index 55580797..9ba4cf2b 100644 --- a/tests/images/mysql_jepsen/Dockerfile +++ b/tests/images/mysql_jepsen/Dockerfile @@ -1,26 +1,19 @@ FROM mysync-jepsen-test-base:latest +ENV MYSQL_VERSION="8.0" COPY . /var/lib/dist/mysql COPY ./sh-scripts/my-wait-started.sh /usr/bin/my-wait-started COPY ./mysync /usr/bin/mysync COPY ./sh-scripts/my-resetup.sh /usr/bin/my-resetup COPY ./sh-scripts/my-resetup-wd.sh /usr/bin/my-resetup-wd COPY ./setup.sh /usr/local/bin/setup.sh -COPY ./my.cnf /etc/mysql/my.cnf COPY ./.my.cnf /root/.my.cnf COPY ./supervisor_mysql.conf /etc/supervisor/conf.d RUN chown mysql:root /etc/mysql RUN touch /etc/mysync.yaml RUN chown mysql:mysql /etc/mysync.yaml +COPY ./my.cnf.8.0 /etc/mysql/my.cnf RUN chmod +x /usr/bin/my-resetup -RUN echo 'APT::Install-Recommends "0"; \n\ -APT::Get::Assume-Yes "true"; \n\ -APT::Get::force-yes "true"; \n\ -APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/01buildconfig && \ - apt-get update && apt-get install wget gnupg ca-certificates && \ - apt-get update && \ - apt-get install - ENV MYSQL_ADMIN_USER=admin ENV MYSQL_ADMIN_PASSWORD=admin_pwd diff --git a/tests/images/mysql_jepsen/my.cnf.8.0 b/tests/images/mysql_jepsen/my.cnf.8.0 new file mode 100644 index 00000000..e38e88b4 --- /dev/null +++ b/tests/images/mysql_jepsen/my.cnf.8.0 @@ -0,0 +1,83 @@ +[client] +port = 3306 +socket = /tmp/mysqld.sock + +[mysqld_safe] +nice = 0 +socket = /tmp/mysqld.sock +log_error = /var/log/mysql/error.log +timezone = Europe/Moscow +log_timestamps = SYSTEM +open_files_limit = 65535 + +[mysqld] +plugin_load_add = 'rpl_semi_sync_master=semisync_master.so;rpl_semi_sync_slave=semisync_slave.so' +loose-rpl_semi_sync_master_timeout = 31536000000 +loose-rpl_semi_sync_master_wait_for_slave_count = 1 +loose-rpl_semi_sync_master_wait_no_slave = ON +loose-rpl_semi_sync_master_wait_point = AFTER_SYNC +default_authentication_plugin = mysql_native_password +user = mysql +port = 3306 +pid_file = /tmp/mysqld.pid +socket = /tmp/mysqld.sock +log_error = /var/log/mysql/error.log +log_error_verbosity = 3 +general_log = ON +general_log_file = /var/log/mysql/query.log +basedir = /usr +datadir = /var/lib/mysql +tmpdir = /tmp +lc_messages_dir = /usr/share/mysql +max_connect_errors = 1000000 +local_infile = OFF +replica_net_timeout = 30 + +event_scheduler = ON +skip_external_locking = ON +default_storage_engine = InnoDB +disabled_storage_engines = "MyISAM,MRG_MyISAM,BLACKHOLE,FEDERATED,ARCHIVE,CSV" +explicit_defaults_for_timestamp = ON +log_timestamps = SYSTEM +max_allowed_packet = 16M +thread_stack = 192K + +max_connections = 200 +thread_handling = one-thread-per-connection +thread_cache_size = 5 + +# Slow log +max_slowlog_size = 1G + +# InnoDB settings +innodb_buffer_pool_size = 32M +innodb_file_per_table = ON +innodb_log_file_size = 8M +innodb_autoinc_lock_mode = 2 +innodb_use_native_aio = OFF +innodb_flush_log_at_trx_commit = 1 + +# Replication +# server_id = 2 # should set from env +binlog_format = ROW +gtid_mode = ON +enforce_gtid_consistency = ON +log_replica_updates = ON +max_binlog_size = 100M +binlog_expire_logs_seconds = 259200 +relay_log_recovery = ON +log_bin = mysql-bin-log +relay_log = mysql-relay-log +sync_binlog = 1 +sync_relay_log = 1 + +# we should start in read-only mode to avoid split brain after restart +# mysync will bring us in writable mode +read_only = ON +super_read_only = ON +offline_mode = ON + +[mysqldump] +quick +quote-names +max_allowed_packet = 16M diff --git a/tests/images/mysql_jepsen/start_mysql.sh b/tests/images/mysql_jepsen/start_mysql.sh index 852ae702..f4b0e29b 100644 --- a/tests/images/mysql_jepsen/start_mysql.sh +++ b/tests/images/mysql_jepsen/start_mysql.sh @@ -21,16 +21,16 @@ EOF if [ ! -z "$MYSQL_MASTER" ]; then cat < /etc/mysql/slave.sql SET GLOBAL server_id = $MYSQL_SERVER_ID; - RESET SLAVE FOR CHANNEL ''; - CHANGE MASTER TO MASTER_HOST = '$MYSQL_MASTER', MASTER_USER = 'repl', MASTER_PASSWORD = 'repl_pwd', MASTER_AUTO_POSITION = 1, MASTER_CONNECT_RETRY = 1, MASTER_RETRY_COUNT = 100500 FOR CHANNEL ''; - START SLAVE; + RESET REPLICA FOR CHANNEL ''; + CHANGE REPLICATION SOURCE TO SOURCE_HOST = '$MYSQL_MASTER', SOURCE_USER = 'repl', SOURCE_PASSWORD = 'repl_pwd', SOURCE_AUTO_POSITION = 1, SOURCE_CONNECT_RETRY = 1, SOURCE_RETRY_COUNT = 100500 FOR CHANNEL ''; + START REPLICA; EOF else touch /etc/mysql/slave.sql fi if [ ! -f /var/lib/mysql/auto.cnf ]; then - /usr/sbin/mysqld --initialize --datadir=/var/lib/mysql --init-file=/etc/mysql/init.sql --server-id=$MYSQL_SERVER_ID + /usr/sbin/mysqld --defaults-file=/etc/mysql/my.cnf --initialize --datadir=/var/lib/mysql --init-file=/etc/mysql/init.sql --server-id=$MYSQL_SERVER_ID echo "==INITIALIZED==" else # clean slave script for restarts diff --git a/tests/images/zookeeper_jepsen/generate_certs_with_restart.sh b/tests/images/zookeeper_jepsen/generate_certs_with_restart.sh index b7b92dc6..a74e5ea9 100644 --- a/tests/images/zookeeper_jepsen/generate_certs_with_restart.sh +++ b/tests/images/zookeeper_jepsen/generate_certs_with_restart.sh @@ -1,6 +1,12 @@ #!/bin/bash set -ex +# Wait for supervisord to be ready +for i in $(seq 1 30); do + supervisorctl status && break || true + sleep 1 +done + supervisorctl stop zookeeper ps -aux | grep [z]oo.cfg | awk '{print $2}' | xargs kill || true /var/lib/dist/base/generate_certs.sh $1