From b0eeb1332399b8b17ee3d41db34119ed12e7609d Mon Sep 17 00:00:00 2001 From: Nicolas Werner <nicolas.werner@hotmail.de> Date: Mon, 21 Dec 2020 03:07:18 +0100 Subject: [PATCH 1/8] Add gitlab ci Use attached image Fix changes key Missing stage Fix tags for ci builds Make lint Use branch pipelines Fix changes rule Fix needs Exit success after docker build Try to fix postgres lockfile Whitelist var run in container build Make tests homeserver name generic --- .ci/macos/Brewfile | 7 ++ .ci/synapse/Dockerfile | 12 +++ .ci/synapse/entrypoint.sh | 12 +++ .ci/synapse/service/postgresql/run | 3 + .ci/synapse/service/synapse/run | 2 + .ci/synapse/setup-synapse.sh | 99 +++++++++++++++++++++++ .gitlab-ci.yml | 121 +++++++++++++++++++++++++++++ include/mtx/requests.hpp | 9 ++- include/mtxclient/http/client.hpp | 3 +- lib/structs/requests.cpp | 13 ++-- tests/client_api.cpp | 97 +++++++++++------------ tests/connection.cpp | 9 ++- tests/e2ee.cpp | 6 +- tests/pushrules.cpp | 18 ++--- tests/test_helpers.hpp | 10 ++- 15 files changed, 343 insertions(+), 78 deletions(-) create mode 100644 .ci/macos/Brewfile create mode 100644 .ci/synapse/Dockerfile create mode 100755 .ci/synapse/entrypoint.sh create mode 100755 .ci/synapse/service/postgresql/run create mode 100755 .ci/synapse/service/synapse/run create mode 100755 .ci/synapse/setup-synapse.sh diff --git a/.ci/macos/Brewfile b/.ci/macos/Brewfile new file mode 100644 index 000000000..338713c04 --- /dev/null +++ b/.ci/macos/Brewfile @@ -0,0 +1,7 @@ +tap "nlohmann/json" + +brew "pkg-config" +brew "cmake" +brew "ninja" +brew "openssl" +brew "nlohmann_json" diff --git a/.ci/synapse/Dockerfile b/.ci/synapse/Dockerfile new file mode 100644 index 000000000..a7692ddd8 --- /dev/null +++ b/.ci/synapse/Dockerfile @@ -0,0 +1,12 @@ +FROM matrixdotorg/synapse:v1.24.0 + +COPY setup-synapse.sh /setup-synapse.sh +COPY entrypoint.sh /entrypoint.sh +COPY service /service + +RUN /setup-synapse.sh + +ENTRYPOINT ["/entrypoint.sh"] + +EXPOSE 8008 + diff --git a/.ci/synapse/entrypoint.sh b/.ci/synapse/entrypoint.sh new file mode 100755 index 000000000..d21a79ac6 --- /dev/null +++ b/.ci/synapse/entrypoint.sh @@ -0,0 +1,12 @@ +#!/bin/sh + +sv_stop() { + for s in $(ls -d /service/*) + do + sv stop $s + done +} + +trap "sv_stop; exit" SIGTERM +runsvdir /service & +wait diff --git a/.ci/synapse/service/postgresql/run b/.ci/synapse/service/postgresql/run new file mode 100755 index 000000000..6a35afa0a --- /dev/null +++ b/.ci/synapse/service/postgresql/run @@ -0,0 +1,3 @@ +#!/bin/sh + +exec chpst -u postgres:postgres /usr/lib/postgresql/11/bin/postgres -D '/data2/db' 2>&1 diff --git a/.ci/synapse/service/synapse/run b/.ci/synapse/service/synapse/run new file mode 100755 index 000000000..0c81e9dbd --- /dev/null +++ b/.ci/synapse/service/synapse/run @@ -0,0 +1,2 @@ +#!/bin/sh +exec /start.py diff --git a/.ci/synapse/setup-synapse.sh b/.ci/synapse/setup-synapse.sh new file mode 100755 index 000000000..2cc2311b5 --- /dev/null +++ b/.ci/synapse/setup-synapse.sh @@ -0,0 +1,99 @@ +#!/bin/sh + +set -e + +export DEBIAN_FRONTEND=noninteractive + +apt-get update && apt-get -y install --no-install-recommends runit postgresql openssl + + +mkdir /data2 + +mkdir /data2/db +chown postgres /data2/db + +# Initialise & start the database +su -c '/usr/lib/postgresql/11/bin/initdb -D /data2/db -E "UTF-8" --lc-collate="C" --lc-ctype="C" --username=postgres' postgres +su -c '/usr/lib/postgresql/11/bin/pg_ctl -w -D /data2/db start' postgres +su -c '/usr/lib/postgresql/11/bin/createuser synapse_user' postgres +su -c '/usr/lib/postgresql/11/bin/createdb -O synapse_user synapse' postgres + +sed -i 's,/data,/data2,g' /start.py +sed -i 's,/data,/data2,g' /conf/homeserver.yaml + +SYNAPSE_SERVER_NAME=synapse SYNAPSE_REPORT_STATS=no /start.py generate + +perl -pi -w -e \ + 's/#enable_registration: false/enable_registration: true/g;' data2/homeserver.yaml +perl -pi -w -e \ + 's/tls: false/tls: true/g;' data2/homeserver.yaml +perl -pi -w -e \ + 's/#tls_certificate_path:/tls_certificate_path:/g;' data2/homeserver.yaml +perl -pi -w -e \ + 's/#tls_private_key_path:/tls_private_key_path:/g;' data2/homeserver.yaml + +openssl req -x509 -newkey rsa:4096 -keyout data2/synapse.tls.key -out data2/synapse.tls.crt -days 365 -subj '/CN=synapse' -nodes +chmod 0777 data2/synapse.tls.crt +chmod 0777 data2/synapse.tls.key + +# set db config to postgres +sed -i '/^database/,+4d' /data2/homeserver.yaml + +# yes, the empty line is needed +cat <<EOF >> /data2/homeserver.yaml + +database: + name: psycopg2 + args: + user: synapse_user + database: synapse + host: localhost + cp_min: 5 + cp_max: 10 + +rc_message: + per_second: 10000 + burst_count: 100000 + +rc_registration: + per_second: 10000 + burst_count: 30000 + +rc_login: + address: + per_second: 10000 + burst_count: 30000 + account: + per_second: 10000 + burst_count: 30000 + failed_attempts: + per_second: 10000 + burst_count: 30000 + +rc_admin_redaction: + per_second: 1000 + burst_count: 5000 + +rc_joins: + local: + per_second: 10000 + burst_count: 100000 + remote: + per_second: 10000 + burst_count: 100000 +EOF + +# start synapse and create users +/start.py & + +echo Waiting for synapse to start... +until curl -s -f -k https://localhost:8008/_matrix/client/versions; do echo "Checking ..."; sleep 2; done +echo Register alice +register_new_matrix_user --admin -u alice -p secret -c /data2/homeserver.yaml https://localhost:8008 +echo Register bob +register_new_matrix_user --admin -u bob -p secret -c /data2/homeserver.yaml https://localhost:8008 +echo Register carl +register_new_matrix_user --admin -u carl -p secret -c /data2/homeserver.yaml https://localhost:8008 + +exit 0 + diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d06a826b9..705d77645 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,8 +1,126 @@ +variables: + CCACHE_COMPILERCHECK: content + CCACHE_DIR: "${CI_PROJECT_DIR}/.ccache" + # prevent configure tzdata hanging apt install commands + DEBIAN_FRONTEND: noninteractive + +include: + - template: 'Workflows/Branch-Pipelines.gitlab-ci.yml' + +stages: + - prepare + - build + +build: + stage: prepare + tags: [docker] + image: + name: gcr.io/kaniko-project/executor:debug + entrypoint: [""] + rules: + - if: $CI_COMMIT_BRANCH + changes: + - .ci/synapse/Dockerfile + - .ci/synapse/setup-synapse.sh + - .ci/synapse/service/synapse/* + - .ci/synapse/service/postgresql/* + script: + - mkdir -p /kaniko/.docker + - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json + - /kaniko/executor --whitelist-var-run=false --context $CI_PROJECT_DIR/.ci/synapse --dockerfile $CI_PROJECT_DIR/.ci/synapse/Dockerfile --destination $CI_REGISTRY_IMAGE/synapse:latest + +build-gcc7: + stage: build + image: ubuntu:16.04 + tags: [docker] + services: + - name: $CI_REGISTRY_IMAGE/synapse:latest + alias: synapse + variables: + CXX: g++-8 + CC: gcc-8 + TRAVIS_OS_NAME: linux + before_script: + - apt-get update + - apt-get install -y software-properties-common + - add-apt-repository ppa:ubuntu-toolchain-r/test -y + - apt-get update && apt-get -y install --no-install-recommends ${CXX} ${CC} build-essential ninja-build libssl-dev git ccache curl + # need recommended deps for wget + - apt-get -y install wget + - wget https://github.com/Kitware/CMake/releases/download/v3.19.0/cmake-3.19.0-Linux-x86_64.sh && sh cmake-3.19.0-Linux-x86_64.sh --skip-license --prefix=/usr/local + - /usr/sbin/update-ccache-symlinks + - update-alternatives --install /usr/bin/gcc gcc "/usr/bin/${CC}" 10 + - update-alternatives --install /usr/bin/g++ g++ "/usr/bin/${CXX}" 10 + - update-alternatives --set gcc "/usr/bin/${CC}" + - update-alternatives --set g++ "/usr/bin/${CXX}" + script: + - curl -s -f -k https://synapse:8008/_matrix/client/versions + - export PATH="/usr/lib/ccache:${PATH}" + - export CMAKE_BUILD_PARALLEL_LEVEL=$(cat /proc/cpuinfo | awk '/^processor/{print $3}' | wc -l) + - export PATH="/usr/local/bin/:${PATH}" + - mkdir -p .deps/usr .hunter + - mkdir -p build + - cmake -GNinja -H. -Bbuild + -DCMAKE_INSTALL_PREFIX=.deps/usr + -DHUNTER_ROOT=".hunter" + -DHUNTER_ENABLED=ON -DBUILD_SHARED_LIBS=OFF + -DCMAKE_BUILD_TYPE=Release -DHUNTER_CONFIGURATION_TYPES=Debug + -DCI_BUILD=ON + - cmake --build build + - MTXCLIENT_SERVER=synapse GTEST_OUTPUT=xml:junit-output/ make test + cache: + key: "$CI_JOB_NAME" + paths: + - .hunter/ + - .ccache + artifacts: + reports: + junit: build/junit-output/*.xml + paths: + - build/junit-output/*.xml + +build-macos: + stage: build + tags: [macos] + needs: [] + before_script: + - brew update + - brew bundle --file=./.ci/macos/Brewfile + script: + - export PATH=/usr/local/opt/qt/bin/:${PATH} + - cmake -GNinja -H. -Bbuild + -DCMAKE_BUILD_TYPE=RelWithDebInfo + -DCMAKE_INSTALL_PREFIX=.deps/usr + -DHUNTER_ROOT=".hunter" + -DHUNTER_ENABLED=ON -DBUILD_SHARED_LIBS=OFF + -DCMAKE_BUILD_TYPE=RelWithDebInfo -DHUNTER_CONFIGURATION_TYPES=RelWithDebInfo + -DUSE_BUNDLED_OPENSSL=ON + -DUSE_BUNDLED_BOOST=ON + -DCI_BUILD=ON + - cmake --build build + cache: + key: "${CI_JOB_NAME}" + paths: + - .hunter/ + - "${CCACHE_DIR}" + +linting: + stage: build + image: alpine:latest + tags: [docker] + needs: [] + before_script: + - apk update && apk add clang make git + script: + - make lint + test-pages: + stage: build tags: [docker] image: alpine except: - master + needs: [] before_script: - apk update - apk add doxygen git texlive-full py3-jinja2 py3-pygments @@ -15,10 +133,12 @@ test-pages: - public pages: + stage: build tags: [docker] image: alpine only: - master + needs: [] before_script: - apk update - apk add doxygen git texlive-full py3-jinja2 py3-pygments @@ -29,3 +149,4 @@ pages: artifacts: paths: - public + diff --git a/include/mtx/requests.hpp b/include/mtx/requests.hpp index 29e3d6f8c..d192c061f 100644 --- a/include/mtx/requests.hpp +++ b/include/mtx/requests.hpp @@ -255,7 +255,8 @@ struct KeySignaturesUpload void to_json(json &obj, const KeySignaturesUpload &req); -struct PusherData { +struct PusherData +{ //! Required if `kind` is http. The URL to use to send notifications to. //! MUST be an HTTPS URL with a path of /_matrix/push/v1/notify. std::string url; @@ -270,7 +271,8 @@ void to_json(json &obj, const PusherData &data); //! Request payload for the `POST /_matrix/client/r0/pushers/set` endpoint. -struct SetPusher { +struct SetPusher +{ //! Required. Unique identifier for this pusher. std::string pushkey; //! Required. The kind of pusher to configure. "http" makes a pusher that sends HTTP pokes. @@ -280,7 +282,8 @@ struct SetPusher { //! Required. This is a reverse-DNS style identifier for the application. //! If the `kind` is "email", this is "m.email". std::string app_id; - //! Required. A string that will allow the user to identify what application owns this pusher. + //! Required. A string that will allow the user to identify what application owns this + //! pusher. std::string app_display_name; //! Required. A string that will allow the user to identify what device owns this pusher. std::string device_display_name; diff --git a/include/mtxclient/http/client.hpp b/include/mtxclient/http/client.hpp index 67987f807..14824fb06 100644 --- a/include/mtxclient/http/client.hpp +++ b/include/mtxclient/http/client.hpp @@ -561,8 +561,7 @@ public: void get_turn_server(Callback<mtx::responses::TurnServer> cb); //! Sets, updates, or deletes a pusher - void set_pusher(const mtx::requests::SetPusher &req, - Callback<mtx::responses::Empty> cb); + void set_pusher(const mtx::requests::SetPusher &req, Callback<mtx::responses::Empty> cb); private: template<class Request, class Response> diff --git a/lib/structs/requests.cpp b/lib/structs/requests.cpp index f23d67db5..546272fa2 100644 --- a/lib/structs/requests.cpp +++ b/lib/structs/requests.cpp @@ -187,7 +187,6 @@ to_json(json &obj, const KeySignaturesUpload &req) std::visit([](const auto &e) { return json(e); }, keyVar); } - void to_json(json &obj, const PusherData &data) { @@ -202,16 +201,16 @@ to_json(json &obj, const PusherData &data) void to_json(json &obj, const SetPusher &req) { - obj["pushkey"] = req.pushkey; - obj["kind"] = req.kind; - obj["app_id"] = req.app_id; - obj["app_display_name"] = req.app_display_name; + obj["pushkey"] = req.pushkey; + obj["kind"] = req.kind; + obj["app_id"] = req.app_id; + obj["app_display_name"] = req.app_display_name; obj["device_display_name"] = req.device_display_name; if (!req.profile_tag.empty()) { obj["profile_tag"] = req.profile_tag; } - obj["lang"] = req.lang; - obj["data"] = req.data; + obj["lang"] = req.lang; + obj["data"] = req.data; obj["append"] = req.append; } diff --git a/tests/client_api.cpp b/tests/client_api.cpp index 9332f5d82..b6f153efd 100644 --- a/tests/client_api.cpp +++ b/tests/client_api.cpp @@ -50,7 +50,7 @@ TEST(ClientAPI, Register) "secret", {err->matrix_error.unauthorized.session, mtx::user_interactive::auth::Dummy{}}, [username](const mtx::responses::Register &res, RequestErr err) { - const auto user_id = "@" + username + ":localhost"; + const auto user_id = "@" + username + ":" + server_name(); check_error(err); EXPECT_EQ(res.user_id.to_string(), user_id); @@ -66,17 +66,17 @@ TEST(ClientAPI, LoginSuccess) mtx_client->login("alice", "secret", [](const mtx::responses::Login &res, RequestErr err) { check_error(err); - validate_login("@alice:localhost", res); + validate_login("@alice:" + server_name(), res); }); mtx_client->login("bob", "secret", [](const mtx::responses::Login &res, RequestErr err) { check_error(err); - validate_login("@bob:localhost", res); + validate_login("@bob:" + server_name(), res); }); mtx_client->login("carl", "secret", [](const mtx::responses::Login &res, RequestErr err) { check_error(err); - validate_login("@carl:localhost", res); + validate_login("@carl:" + server_name(), res); }); mtx_client->close(); @@ -268,7 +268,7 @@ TEST(ClientAPI, CreateRoom) mtx_client->create_room(req, [](const mtx::responses::CreateRoom &res, RequestErr err) { check_error(err); ASSERT_TRUE(res.room_id.localpart().size() > 10); - EXPECT_EQ(res.room_id.hostname(), "localhost"); + EXPECT_EQ(res.room_id.hostname(), server_name()); }); mtx_client->close(); @@ -441,7 +441,7 @@ TEST(ClientAPI, CreateRoomInvites) mtx::requests::CreateRoom req; req.name = "Name"; req.topic = "Topic"; - req.invite = {"@bob:localhost", "@carl:localhost"}; + req.invite = {"@bob:" + server_name(), "@carl:" + server_name()}; alice->create_room(req, [bob, carl](const mtx::responses::CreateRoom &res, RequestErr err) { check_error(err); auto room_id = res.room_id.to_string(); @@ -483,7 +483,7 @@ TEST(ClientAPI, JoinRoom) mtx::requests::CreateRoom req; req.name = "Name"; req.topic = "Topic"; - req.invite = {"@bob:localhost"}; + req.invite = {"@bob:" + server_name()}; req.room_alias_name = alias; alice->create_room( req, [bob, alias](const mtx::responses::CreateRoom &res, RequestErr err) { @@ -495,7 +495,7 @@ TEST(ClientAPI, JoinRoom) }); using namespace mtx::identifiers; - bob->join_room("!random_room_id:localhost", + bob->join_room("!random_room_id:" + server_name(), [](const mtx::responses::RoomId &, RequestErr err) { ASSERT_TRUE(err); EXPECT_EQ( @@ -505,7 +505,7 @@ TEST(ClientAPI, JoinRoom) // Join the room using an alias. bob->join_room( - "#" + alias + ":localhost", + "#" + alias + ":" + server_name(), [](const mtx::responses::RoomId &, RequestErr err) { check_error(err); }); }); @@ -532,7 +532,7 @@ TEST(ClientAPI, LeaveRoom) mtx::requests::CreateRoom req; req.name = "Name"; req.topic = "Topic"; - req.invite = {"@bob:localhost"}; + req.invite = {"@bob:" + server_name()}; alice->create_room(req, [bob](const mtx::responses::CreateRoom &res, RequestErr err) { check_error(err); auto room_id = res.room_id; @@ -549,11 +549,12 @@ TEST(ClientAPI, LeaveRoom) }); // Trying to leave a non-existent room should fail. - bob->leave_room("!random_room_id:localhost", [](mtx::responses::Empty, RequestErr err) { - ASSERT_TRUE(err); - EXPECT_EQ(mtx::errors::to_string(err->matrix_error.errcode), "M_UNKNOWN"); - EXPECT_EQ(err->matrix_error.error, "Not a known room"); - }); + bob->leave_room( + "!random_room_id:" + server_name(), [](mtx::responses::Empty, RequestErr err) { + ASSERT_TRUE(err); + EXPECT_EQ(mtx::errors::to_string(err->matrix_error.errcode), "M_UNKNOWN"); + EXPECT_EQ(err->matrix_error.error, "Not a known room"); + }); alice->close(); bob->close(); @@ -585,7 +586,7 @@ TEST(ClientAPI, InviteRoom) auto room_id = res.room_id.to_string(); alice->invite_user(room_id, - "@bob:localhost", + "@bob:" + server_name(), [room_id, bob](const mtx::responses::Empty &, RequestErr err) { check_error(err); @@ -628,7 +629,7 @@ TEST(ClientAPI, KickRoom) alice->invite_user( room_id, - "@bob:localhost", + "@bob:" + server_name(), [room_id, alice, bob](const mtx::responses::Empty &, RequestErr err) { check_error(err); @@ -638,7 +639,7 @@ TEST(ClientAPI, KickRoom) check_error(err); alice->kick_user(room_id, - "@bob:localhost", + "@bob:" + server_name(), [](const mtx::responses::Empty &, RequestErr err) { check_error(err); }); }); @@ -676,7 +677,7 @@ TEST(ClientAPI, BanRoom) alice->invite_user( room_id, - "@bob:localhost", + "@bob:" + server_name(), [room_id, alice, bob](const mtx::responses::Empty &, RequestErr err) { check_error(err); @@ -687,13 +688,13 @@ TEST(ClientAPI, BanRoom) alice->ban_user( room_id, - "@bob:localhost", + "@bob:" + server_name(), [alice, room_id](const mtx::responses::Empty &, RequestErr err) { check_error(err); alice->unban_user( room_id, - "@bob:localhost", + "@bob:" + server_name(), [](const mtx::responses::Empty &, RequestErr err) { check_error(err); }, "You not bad anymore!"); @@ -733,7 +734,7 @@ TEST(ClientAPI, InvalidInvite) auto room_id = res.room_id.to_string(); bob->invite_user(room_id, - "@carl:localhost", + "@carl:" + server_name(), [room_id, bob](const mtx::responses::Empty &, RequestErr err) { ASSERT_TRUE(err); EXPECT_EQ( @@ -836,7 +837,7 @@ TEST(ClientAPI, Typing) mtx::events::EphemeralEvent<mtx::events::ephemeral::Typing>>( room.ephemeral.events.front()) .content.user_ids.front(), - "@alice:localhost"); + "@alice:" + server_name()); }); while (!can_continue) @@ -926,7 +927,7 @@ TEST(ClientAPI, PresenceOverSync) sleep(); mtx::requests::CreateRoom req; - req.invite = {"@bob:localhost"}; + req.invite = {"@bob:" + server_name()}; alice->create_room( req, [alice, bob](const mtx::responses::CreateRoom &res, RequestErr err) { check_error(err); @@ -960,7 +961,7 @@ TEST(ClientAPI, PresenceOverSync) bool found = false; for (const auto &p : s.presence) { if (p.sender == - "@alice:localhost") { + "@alice:" + server_name()) { found = true; EXPECT_EQ( p.content.presence, @@ -998,7 +999,7 @@ TEST(ClientAPI, SendMessages) sleep(); mtx::requests::CreateRoom req; - req.invite = {"@bob:localhost"}; + req.invite = {"@bob:" + server_name()}; alice->create_room( req, [alice, bob](const mtx::responses::CreateRoom &res, RequestErr err) { check_error(err); @@ -1114,7 +1115,7 @@ TEST(ClientAPI, SendStateEvents) sleep(); mtx::requests::CreateRoom req; - req.invite = {"@bob:localhost"}; + req.invite = {"@bob:" + server_name()}; alice->create_room( req, [alice, bob](const mtx::responses::CreateRoom &res, RequestErr err) { check_error(err); @@ -1299,7 +1300,7 @@ TEST(ClientAPI, ReadMarkers) receipts.front()) .content.receipts[event_id]; EXPECT_EQ(users.users.size(), 1); - ASSERT_TRUE(users.users["@alice:localhost"].ts > 0); + ASSERT_TRUE(users.users["@alice:" + server_name()].ts > 0); }); }); @@ -1351,7 +1352,7 @@ TEST(ClientAPI, SendToDevice) EXPECT_EQ(event.content.request_id, "test_request_id"); EXPECT_EQ(event.content.requesting_device_id, "test_req_id"); EXPECT_EQ(event.type, mtx::events::EventType::RoomKeyRequest); - EXPECT_EQ(event.sender, "@alice:localhost"); + EXPECT_EQ(event.sender, "@alice:" + server_name()); }); }); @@ -1452,13 +1453,13 @@ TEST(ClientAPI, RetrieveSingleEvent) auto e = std::get<mtx::events::RoomEvent<mtx::events::msg::Text>>(res); EXPECT_EQ(e.content.body, "Hello Alice!"); - EXPECT_EQ(e.sender, "@bob:localhost"); + EXPECT_EQ(e.sender, "@bob:" + server_name()); EXPECT_EQ(e.event_id, event_id); }); bob->get_event( room_id, - "$random_event:localhost", + "$random_event:" + server_name(), [event_id = res.event_id.to_string()]( const mtx::events::collections::TimelineEvents &, RequestErr err) { ASSERT_TRUE(err); @@ -1516,21 +1517,21 @@ TEST(Groups, Rooms) WAIT_UNTIL(rooms_added == 2) - alice->joined_groups( - [random_group_id](const mtx::responses::JoinedGroups &res, RequestErr err) { - check_error(err); + alice->joined_groups([random_group_id](const mtx::responses::JoinedGroups &res, + RequestErr err) { + check_error(err); - ASSERT_GE(res.groups.size(), 1); + ASSERT_GE(res.groups.size(), 1); - for (const auto &g : res.groups) { - if (g == std::string("+" + random_group_id + ":localhost")) - return; - } + for (const auto &g : res.groups) { + if (g == std::string("+" + random_group_id + ":" + server_name())) + return; + } - FAIL(); - }); + FAIL(); + }); - alice->group_rooms("+" + random_group_id + ":localhost", + alice->group_rooms("+" + random_group_id + ":" + server_name(), [](const nlohmann::json &res, RequestErr err) { check_error(err); EXPECT_GE(res.at("chunk").size(), 2); @@ -1558,13 +1559,13 @@ TEST(Groups, Profiles) json profile; profile["name"] = "Name"; alice->set_group_profile( - "+" + random_group_id + ":localhost", + "+" + random_group_id + ":" + server_name(), profile, [alice, random_group_id](const nlohmann::json &, RequestErr err) { check_error(err); alice->group_profile( - "+" + random_group_id + ":localhost", + "+" + random_group_id + ":" + server_name(), [](const mtx::responses::GroupProfile &res, RequestErr err) { check_error(err); EXPECT_EQ(res.name, "Name"); @@ -1597,7 +1598,7 @@ TEST(ClientAPI, PublicRooms) req.name = "Public Room"; req.topic = "Test"; req.visibility = mtx::common::RoomVisibility::Public; - req.invite = {"@bob:localhost"}; + req.invite = {"@bob:" + server_name()}; req.room_alias_name = alice->generate_txn_id(); req.preset = Preset::PublicChat; @@ -1695,13 +1696,13 @@ TEST(ClientAPI, PublicRooms) check_error(err); }); }, - "localhost", + server_name(), 1); }, - "localhost", + server_name(), 1); }, - "localhost"); + server_name()); }); }); }); diff --git a/tests/connection.cpp b/tests/connection.cpp index 6a78e0e83..02edb88b0 100644 --- a/tests/connection.cpp +++ b/tests/connection.cpp @@ -23,12 +23,13 @@ TEST(Basic, Connection) TEST(Basic, ServerWithPort) { - auto alice = std::make_shared<Client>("matrix.org"); + std::string server = server_name(); + auto alice = std::make_shared<Client>("matrix.org"); alice->verify_certificates(false); - alice->set_server("localhost:8448"); + alice->set_server(server + ":8008"); - EXPECT_EQ(alice->server(), "localhost"); - EXPECT_EQ(alice->port(), 8448); + EXPECT_EQ(alice->server(), server); + EXPECT_EQ(alice->port(), 8008); alice->versions( [](const mtx::responses::Versions &, RequestErr err) { ASSERT_FALSE(err); }); diff --git a/tests/e2ee.cpp b/tests/e2ee.cpp index d828ea9c7..20b0e3399 100644 --- a/tests/e2ee.cpp +++ b/tests/e2ee.cpp @@ -551,7 +551,7 @@ TEST(Encryption, EnableEncryption) mtx::identifiers::Room joined_room; mtx::requests::CreateRoom req; - req.invite = {"@carl:localhost"}; + req.invite = {"@carl:" + server_name()}; bob->create_room( req, [bob, carl, &responses, &joined_room](const mtx::responses::CreateRoom &res, @@ -852,7 +852,7 @@ TEST(Encryption, OlmRoomKeyEncryption) auto out_session = alice_olm->create_outbound_session(bob_curve25519, bob_otk); auto device_msg = alice_olm->create_olm_encrypted_content(out_session.get(), payload, - UserId("@bob:localhost"), + UserId("@bob:" + server_name()), bob_olm->identity_keys().ed25519, bob_curve25519); @@ -1039,7 +1039,7 @@ TEST(Encryption, ShareSecret) auto device_msg = alice_olm->create_olm_encrypted_content( out_session.get(), payload, - UserId("@bob:localhost"), + UserId("@bob:" + server_name()), bob_olm->identity_keys().ed25519, bob_curve25519); diff --git a/tests/pushrules.cpp b/tests/pushrules.cpp index 1c8f9c0a7..1017b9e00 100644 --- a/tests/pushrules.cpp +++ b/tests/pushrules.cpp @@ -236,7 +236,7 @@ TEST(Pushrules, GetGlobalRuleset) client->login( "alice", "secret", [client](const mtx::responses::Login &res, RequestErr err) { check_error(err); - validate_login("@alice:localhost", res); + validate_login("@alice:" + server_name(), res); client->get_pushrules([](const mtx::pushrules::GlobalRuleset &, RequestErr err) { EXPECT_TRUE(!err); @@ -252,7 +252,7 @@ TEST(Pushrules, GetRuleset) client->login( "alice", "secret", [client](const mtx::responses::Login &res, RequestErr err) { check_error(err); - validate_login("@alice:localhost", res); + validate_login("@alice:" + server_name(), res); client->get_pushrules("global", "content", @@ -272,7 +272,7 @@ TEST(Pushrules, PutAndDeleteRuleset) client->login( "alice", "secret", [client](const mtx::responses::Login &res, RequestErr err) { check_error(err); - validate_login("@alice:localhost", res); + validate_login("@alice:" + server_name(), res); mtx::pushrules::PushRule rule; rule.pattern = "cake"; @@ -302,7 +302,7 @@ TEST(Pushrules, RulesetEnabled) client->login( "alice", "secret", [client](const mtx::responses::Login &res, RequestErr err) { check_error(err); - validate_login("@alice:localhost", res); + validate_login("@alice:" + server_name(), res); client->get_pushrules_enabled( "global", @@ -349,7 +349,7 @@ TEST(Pushrules, Actions) client->login( "alice", "secret", [client](const mtx::responses::Login &res, RequestErr err) { check_error(err); - validate_login("@alice:localhost", res); + validate_login("@alice:" + server_name(), res); mtx::pushrules::actions::Actions actions = { {mtx::pushrules::actions::notify{}, @@ -383,7 +383,7 @@ TEST(Pushrules, RoomRuleMute) client->login( "alice", "secret", [client](const mtx::responses::Login &res, RequestErr err) { check_error(err); - validate_login("@alice:localhost", res); + validate_login("@alice:" + server_name(), res); mtx::requests::CreateRoom req; req.name = "Name"; @@ -393,7 +393,7 @@ TEST(Pushrules, RoomRuleMute) req, [client](const mtx::responses::CreateRoom &res, RequestErr err) { check_error(err); ASSERT_TRUE(res.room_id.localpart().size() > 10); - EXPECT_EQ(res.room_id.hostname(), "localhost"); + EXPECT_EQ(res.room_id.hostname(), server_name()); mtx::pushrules::PushRule rule; rule.actions = {mtx::pushrules::actions::dont_notify{}}; @@ -423,7 +423,7 @@ TEST(Pushrules, RoomRuleMentions) client->login( "alice", "secret", [client](const mtx::responses::Login &res, RequestErr err) { check_error(err); - validate_login("@alice:localhost", res); + validate_login("@alice:" + server_name(), res); mtx::requests::CreateRoom req; req.name = "Name"; @@ -433,7 +433,7 @@ TEST(Pushrules, RoomRuleMentions) req, [client](const mtx::responses::CreateRoom &res, RequestErr err) { check_error(err); ASSERT_TRUE(res.room_id.localpart().size() > 10); - EXPECT_EQ(res.room_id.hostname(), "localhost"); + EXPECT_EQ(res.room_id.hostname(), server_name()); mtx::pushrules::PushRule rule; rule.actions = {mtx::pushrules::actions::dont_notify{}}; diff --git a/tests/test_helpers.hpp b/tests/test_helpers.hpp index 7ff864aeb..398c52d3a 100644 --- a/tests/test_helpers.hpp +++ b/tests/test_helpers.hpp @@ -50,11 +50,17 @@ check_error(mtx::http::RequestErr err) ASSERT_FALSE(err); } +inline std::string +server_name() +{ + const char *server_ = std::getenv("MTXCLIENT_SERVER"); + return server_ ? server_ : std::string("localhost"); +} + inline auto make_test_client() { - const char *server = std::getenv("MTXCLIENT_SERVER"); - auto client = std::make_shared<mtx::http::Client>(server ? server : "localhost"); + auto client = std::make_shared<mtx::http::Client>(server_name(), 8008); client->verify_certificates(false); return client; } -- GitLab From f94f725af44c7bd2718061556ee2abbd15b2ae38 Mon Sep 17 00:00:00 2001 From: Nicolas Werner <nicolas.werner@hotmail.de> Date: Fri, 12 Mar 2021 01:03:40 +0100 Subject: [PATCH 2/8] Fix macos brew install python conflicts --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 705d77645..66afc1e36 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -85,7 +85,8 @@ build-macos: needs: [] before_script: - brew update - - brew bundle --file=./.ci/macos/Brewfile + - brew reinstall --force python3 + - brew bundle --file=./.ci/macos/Brewfile --force --cleanup script: - export PATH=/usr/local/opt/qt/bin/:${PATH} - cmake -GNinja -H. -Bbuild -- GitLab From 214112520d0b6aea810d027c0ad1b25be471e660 Mon Sep 17 00:00:00 2001 From: Nicolas Werner <nicolas.werner@hotmail.de> Date: Fri, 12 Mar 2021 01:27:33 +0100 Subject: [PATCH 3/8] Add arm64 builds --- .gitlab-ci.yml | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 66afc1e36..f266c8a99 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -11,9 +11,8 @@ stages: - prepare - build -build: +.build-synapse-image: stage: prepare - tags: [docker] image: name: gcr.io/kaniko-project/executor:debug entrypoint: [""] @@ -24,15 +23,23 @@ build: - .ci/synapse/setup-synapse.sh - .ci/synapse/service/synapse/* - .ci/synapse/service/postgresql/* + - .gitlab-ci.yml script: - mkdir -p /kaniko/.docker - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json - /kaniko/executor --whitelist-var-run=false --context $CI_PROJECT_DIR/.ci/synapse --dockerfile $CI_PROJECT_DIR/.ci/synapse/Dockerfile --destination $CI_REGISTRY_IMAGE/synapse:latest -build-gcc7: +build synapse amd64: + extends: .build-synapse-image + tags: [docker] + +build synapse arm64: + extends: .build-synapse-image + tags: [docker-arm64] + +.build-linux: stage: build image: ubuntu:16.04 - tags: [docker] services: - name: $CI_REGISTRY_IMAGE/synapse:latest alias: synapse @@ -79,6 +86,14 @@ build-gcc7: paths: - build/junit-output/*.xml +build linux amd64: + extends: .build-linux + tags: [docker] + +build linux arm64: + extends: .build-linux + tags: [docker-arm64] + build-macos: stage: build tags: [macos] -- GitLab From b88522499e6cf787cc65eb6ae0a1bb495c716927 Mon Sep 17 00:00:00 2001 From: Nicolas Werner <nicolas.werner@hotmail.de> Date: Fri, 12 Mar 2021 01:39:59 +0100 Subject: [PATCH 4/8] Fix cmake downloads on different CPU arches --- .gitlab-ci.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f266c8a99..8a06c494e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -54,7 +54,7 @@ build synapse arm64: - apt-get update && apt-get -y install --no-install-recommends ${CXX} ${CC} build-essential ninja-build libssl-dev git ccache curl # need recommended deps for wget - apt-get -y install wget - - wget https://github.com/Kitware/CMake/releases/download/v3.19.0/cmake-3.19.0-Linux-x86_64.sh && sh cmake-3.19.0-Linux-x86_64.sh --skip-license --prefix=/usr/local + - wget https://github.com/Kitware/CMake/releases/download/v3.19.0/cmake-3.19.0-Linux-${CMAKE_ARCH}.sh && sh cmake-3.19.0-Linux-${CMAKE_ARCH}.sh --skip-license --prefix=/usr/local - /usr/sbin/update-ccache-symlinks - update-alternatives --install /usr/bin/gcc gcc "/usr/bin/${CC}" 10 - update-alternatives --install /usr/bin/g++ g++ "/usr/bin/${CXX}" 10 @@ -89,10 +89,14 @@ build synapse arm64: build linux amd64: extends: .build-linux tags: [docker] + variables: + CMAKE_ARCH: x86_64 build linux arm64: extends: .build-linux tags: [docker-arm64] + variables: + CMAKE_ARCH: aarch64 build-macos: stage: build -- GitLab From 94c82b4a05213f14ee45744e554cce56ea7f817d Mon Sep 17 00:00:00 2001 From: Nicolas Werner <nicolas.werner@hotmail.de> Date: Fri, 12 Mar 2021 02:26:35 +0100 Subject: [PATCH 5/8] Push images as different names on different arches --- .gitlab-ci.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8a06c494e..6fd748c8c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -27,21 +27,25 @@ stages: script: - mkdir -p /kaniko/.docker - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json - - /kaniko/executor --whitelist-var-run=false --context $CI_PROJECT_DIR/.ci/synapse --dockerfile $CI_PROJECT_DIR/.ci/synapse/Dockerfile --destination $CI_REGISTRY_IMAGE/synapse:latest + - /kaniko/executor --whitelist-var-run=false --context $CI_PROJECT_DIR/.ci/synapse --dockerfile $CI_PROJECT_DIR/.ci/synapse/Dockerfile --destination $CI_REGISTRY_IMAGE/synapse-${CMAKE_ARCH}:latest build synapse amd64: extends: .build-synapse-image tags: [docker] + variables: + CMAKE_ARCH: x86_64 build synapse arm64: extends: .build-synapse-image tags: [docker-arm64] + variables: + CMAKE_ARCH: aarch64 .build-linux: stage: build image: ubuntu:16.04 services: - - name: $CI_REGISTRY_IMAGE/synapse:latest + - name: $CI_REGISTRY_IMAGE/synapse-${CMAKE_ARCH}:latest alias: synapse variables: CXX: g++-8 @@ -54,7 +58,7 @@ build synapse arm64: - apt-get update && apt-get -y install --no-install-recommends ${CXX} ${CC} build-essential ninja-build libssl-dev git ccache curl # need recommended deps for wget - apt-get -y install wget - - wget https://github.com/Kitware/CMake/releases/download/v3.19.0/cmake-3.19.0-Linux-${CMAKE_ARCH}.sh && sh cmake-3.19.0-Linux-${CMAKE_ARCH}.sh --skip-license --prefix=/usr/local + - wget https://github.com/Kitware/CMake/releases/download/v3.19.6/cmake-3.19.6-Linux-${CMAKE_ARCH}.sh && sh cmake-3.19.6-Linux-${CMAKE_ARCH}.sh --skip-license --prefix=/usr/local - /usr/sbin/update-ccache-symlinks - update-alternatives --install /usr/bin/gcc gcc "/usr/bin/${CC}" 10 - update-alternatives --install /usr/bin/g++ g++ "/usr/bin/${CXX}" 10 -- GitLab From 4041cdc62b8c13d367fc21f78b5f208b1313e1fe Mon Sep 17 00:00:00 2001 From: Nicolas Werner <nicolas.werner@hotmail.de> Date: Fri, 12 Mar 2021 02:42:54 +0100 Subject: [PATCH 6/8] rename arch variable --- .gitlab-ci.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6fd748c8c..9346e54ff 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -27,25 +27,25 @@ stages: script: - mkdir -p /kaniko/.docker - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json - - /kaniko/executor --whitelist-var-run=false --context $CI_PROJECT_DIR/.ci/synapse --dockerfile $CI_PROJECT_DIR/.ci/synapse/Dockerfile --destination $CI_REGISTRY_IMAGE/synapse-${CMAKE_ARCH}:latest + - /kaniko/executor --whitelist-var-run=false --context $CI_PROJECT_DIR/.ci/synapse --dockerfile $CI_PROJECT_DIR/.ci/synapse/Dockerfile --destination $CI_REGISTRY_IMAGE/synapse-${M_ARCH}:latest build synapse amd64: extends: .build-synapse-image tags: [docker] variables: - CMAKE_ARCH: x86_64 + M_ARCH: x86_64 build synapse arm64: extends: .build-synapse-image tags: [docker-arm64] variables: - CMAKE_ARCH: aarch64 + M_ARCH: aarch64 .build-linux: stage: build image: ubuntu:16.04 services: - - name: $CI_REGISTRY_IMAGE/synapse-${CMAKE_ARCH}:latest + - name: $CI_REGISTRY_IMAGE/synapse-${M_ARCH}:latest alias: synapse variables: CXX: g++-8 @@ -58,7 +58,7 @@ build synapse arm64: - apt-get update && apt-get -y install --no-install-recommends ${CXX} ${CC} build-essential ninja-build libssl-dev git ccache curl # need recommended deps for wget - apt-get -y install wget - - wget https://github.com/Kitware/CMake/releases/download/v3.19.6/cmake-3.19.6-Linux-${CMAKE_ARCH}.sh && sh cmake-3.19.6-Linux-${CMAKE_ARCH}.sh --skip-license --prefix=/usr/local + - wget https://github.com/Kitware/CMake/releases/download/v3.19.6/cmake-3.19.6-Linux-${M_ARCH}.sh && sh cmake-3.19.6-Linux-${M_ARCH}.sh --skip-license --prefix=/usr/local - /usr/sbin/update-ccache-symlinks - update-alternatives --install /usr/bin/gcc gcc "/usr/bin/${CC}" 10 - update-alternatives --install /usr/bin/g++ g++ "/usr/bin/${CXX}" 10 @@ -94,13 +94,13 @@ build linux amd64: extends: .build-linux tags: [docker] variables: - CMAKE_ARCH: x86_64 + M_ARCH: x86_64 build linux arm64: extends: .build-linux tags: [docker-arm64] variables: - CMAKE_ARCH: aarch64 + M_ARCH: aarch64 build-macos: stage: build -- GitLab From 336cb61e1b122c535b0ed26c9abc1a5bef703455 Mon Sep 17 00:00:00 2001 From: Nicolas Werner <nicolas.werner@hotmail.de> Date: Fri, 12 Mar 2021 03:00:08 +0100 Subject: [PATCH 7/8] Use 3.19.3 of cmake --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9346e54ff..6ec905f9b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -58,7 +58,7 @@ build synapse arm64: - apt-get update && apt-get -y install --no-install-recommends ${CXX} ${CC} build-essential ninja-build libssl-dev git ccache curl # need recommended deps for wget - apt-get -y install wget - - wget https://github.com/Kitware/CMake/releases/download/v3.19.6/cmake-3.19.6-Linux-${M_ARCH}.sh && sh cmake-3.19.6-Linux-${M_ARCH}.sh --skip-license --prefix=/usr/local + - wget https://github.com/Kitware/CMake/releases/download/v3.19.3/cmake-3.19.3-Linux-${M_ARCH}.sh && sh cmake-3.19.3-Linux-${M_ARCH}.sh --skip-license --prefix=/usr/local - /usr/sbin/update-ccache-symlinks - update-alternatives --install /usr/bin/gcc gcc "/usr/bin/${CC}" 10 - update-alternatives --install /usr/bin/g++ g++ "/usr/bin/${CXX}" 10 -- GitLab From 7ea32ce622ef0fc59041801afdcaaa0be5fef463 Mon Sep 17 00:00:00 2001 From: Nicolas Werner <nicolas.werner@hotmail.de> Date: Fri, 12 Mar 2021 03:31:22 +0100 Subject: [PATCH 8/8] Try cmake 3.20.0-rc4 --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6ec905f9b..664e93477 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -58,7 +58,7 @@ build synapse arm64: - apt-get update && apt-get -y install --no-install-recommends ${CXX} ${CC} build-essential ninja-build libssl-dev git ccache curl # need recommended deps for wget - apt-get -y install wget - - wget https://github.com/Kitware/CMake/releases/download/v3.19.3/cmake-3.19.3-Linux-${M_ARCH}.sh && sh cmake-3.19.3-Linux-${M_ARCH}.sh --skip-license --prefix=/usr/local + - wget https://github.com/Kitware/CMake/releases/download/v3.20.0-rc4/cmake-3.20.0-rc4-linux-${M_ARCH}.sh && sh cmake-3.20.0-rc4-linux-${M_ARCH}.sh --skip-license --prefix=/usr/local - /usr/sbin/update-ccache-symlinks - update-alternatives --install /usr/bin/gcc gcc "/usr/bin/${CC}" 10 - update-alternatives --install /usr/bin/g++ g++ "/usr/bin/${CXX}" 10 -- GitLab