Release 0.3.0

CMakeLists.txt

@@ -46,6 +46,7 @@ option(ICHOR_BUILD_BENCHMARKS "Build benchmarks" ON)
 option(ICHOR_BUILD_TESTING "Build tests" ON)
 option(ICHOR_ENABLE_INTERNAL_DEBUGGING "Add verbose logging of Ichor internals" OFF)
 option(ICHOR_ENABLE_INTERNAL_COROUTINE_DEBUGGING "Add verbose logging of Ichor coroutine internals" OFF)
+option(ICHOR_ENABLE_INTERNAL_IO_DEBUGGING "Add verbose logging of Ichor I/O internals" OFF)
 option(ICHOR_BUILD_COVERAGE "Build ichor with coverage" OFF)
 option(ICHOR_USE_SPDLOG "Use spdlog as framework logging implementation" OFF)
 option(ICHOR_USE_ETCD "Add etcd services (may not work, not used much)" OFF)
@@ -68,6 +69,7 @@ else()
 endif()
 option(ICHOR_USE_HIREDIS "Add hiredis dependency" OFF)
 option(ICHOR_MUSL "Use when building for musl instead of glibc" OFF)
+option(ICHOR_AARCH64 "Use when building for aarch64. Turns off some x86-specific compiler flags." OFF)
 
 set(BUILD_TESTING OFF) #disable Catch 2 testing
 
@@ -92,30 +94,26 @@ file(GLOB_RECURSE ICHOR_NETWORK_SOURCES ${ICHOR_TOP_DIR}/src/services/network/*.
 file(GLOB_RECURSE ICHOR_METRICS_SOURCES ${ICHOR_TOP_DIR}/src/services/metrics/*.cpp)
 file(GLOB_RECURSE ICHOR_TIMER_SOURCES ${ICHOR_TOP_DIR}/src/services/timer/*.cpp)
 file(GLOB_RECURSE ICHOR_HIREDIS_SOURCES ${ICHOR_TOP_DIR}/src/services/redis/*.cpp)
+file(GLOB_RECURSE ICHOR_IO_SOURCES ${ICHOR_TOP_DIR}/src/services/io/*.cpp)
 
 file(GLOB SPDLOG_SOURCES ${ICHOR_EXTERNAL_DIR}/spdlog/src/*.cpp)
 
-add_library(ichor ${FMT_SOURCES} ${ICHOR_FRAMEWORK_SOURCES} ${ICHOR_LOGGING_SOURCES} ${ICHOR_NETWORK_SOURCES} ${ICHOR_METRICS_SOURCES} ${ICHOR_TIMER_SOURCES} ${ICHOR_HIREDIS_SOURCES})
+add_library(ichor ${FMT_SOURCES} ${ICHOR_FRAMEWORK_SOURCES} ${ICHOR_LOGGING_SOURCES} ${ICHOR_NETWORK_SOURCES} ${ICHOR_METRICS_SOURCES} ${ICHOR_TIMER_SOURCES} ${ICHOR_HIREDIS_SOURCES} ${ICHOR_IO_SOURCES})
 
 if(ICHOR_ENABLE_INTERNAL_DEBUGGING)
     target_compile_definitions(ichor PUBLIC ICHOR_ENABLE_INTERNAL_DEBUGGING)
 endif()
 if(ICHOR_ENABLE_INTERNAL_COROUTINE_DEBUGGING)
     target_compile_definitions(ichor PUBLIC ICHOR_ENABLE_INTERNAL_COROUTINE_DEBUGGING)
 endif()
+if(ICHOR_ENABLE_INTERNAL_IO_DEBUGGING)
+    target_compile_definitions(ichor PUBLIC ICHOR_ENABLE_INTERNAL_IO_DEBUGGING)
+endif()
 
 if(ICHOR_USE_SPDLOG)
     target_compile_definitions(ichor PUBLIC SPDLOG_COMPILED_LIB SPDLOG_NO_EXCEPTIONS SPDLOG_FMT_EXTERNAL SPDLOG_DISABLE_DEFAULT_LOGGER SPDLOG_NO_ATOMIC_LEVELS ICHOR_USE_SPDLOG SPDLOG_ACTIVE_LEVEL=SPDLOG_LEVEL_TRACE)
 endif()
 
-if(ICHOR_USE_BOOST_BEAST) #json
-    target_compile_definitions(ichor PUBLIC ICHOR_USE_BOOST_JSON BOOST_JSON_STANDALONE)
-    find_package(Boost 1.75.0 REQUIRED COMPONENTS json context)
-    target_include_directories(ichor PUBLIC ${Boost_INCLUDE_DIR})
-    target_link_directories(ichor PUBLIC ${Boost_LIBRARY_DIRS})
-    target_link_libraries(ichor PUBLIC ${Boost_LIBRARIES})
-endif()
-
 if(ICHOR_REMOVE_SOURCE_NAMES)
     target_compile_definitions(ichor PUBLIC ICHOR_REMOVE_SOURCE_NAMES_FROM_LOGGING)
 endif()
@@ -145,7 +143,10 @@ if(ICHOR_USE_BOOST_BEAST)
 endif()
 if(ICHOR_MUSL)
     target_compile_definitions(ichor PUBLIC ICHOR_MUSL)
-    target_link_options(ichor PUBLIC -static-libgcc -static-libstdc++ -static)
+    target_link_options(ichor PUBLIC -static-libgcc -static-libstdc++)
+endif()
+if(ICHOR_AARCH64)
+    target_compile_definitions(ichor PUBLIC ICHOR_AARCH64)
 endif()
 
 if(NOT WIN32)
@@ -249,7 +250,7 @@ endif()
 
 if(ICHOR_USE_THREAD_SANITIZER)
     target_compile_options(ichor PUBLIC -fsanitize=thread -fno-omit-frame-pointer)
-    target_link_options(ichor PUBLIC -fsanitize=thread)
+    target_link_options(ichor PUBLIC -fsanitize=thread -static-libtsan)
 
     # clang on OSX doesn't accept -no-pie for some reason
     if(NOT APPLE)
@@ -278,7 +279,11 @@ if(WIN32 AND ICHOR_USE_HARDENING)
     target_compile_options(ichor PUBLIC /GS /guard:cf)
     target_compile_definitions(ichor PUBLIC ICHOR_USE_HARDENING)
 elseif(ICHOR_USE_HARDENING)
-    target_compile_options(ichor PUBLIC -fstack-protector-strong -fcf-protection)
+    target_compile_options(ichor PUBLIC -fstack-protector-strong)
+
+    if(NOT ICHOR_AARCH64)
+        target_compile_options(ichor PUBLIC -fcf-protection)
+    endif()
 
     # stack clash protection not available on OSX
     if(NOT APPLE)
@@ -376,6 +381,9 @@ if(ICHOR_USE_BOOST_BEAST) #beast
     target_include_directories(ichor PUBLIC ${Boost_INCLUDE_DIRS})
     target_link_directories(ichor PUBLIC ${Boost_LIBRARY_DIRS})
     target_link_libraries(ichor PUBLIC ${Boost_LIBRARIES})
+    find_package(OpenSSL REQUIRED)
+    target_include_directories(ichor PUBLIC ${OPENSSL_INCLUDE_DIR})
+    target_link_libraries(ichor PUBLIC ${OPENSSL_LIBRARIES})
     #_SILENCE_ALL_CXX23_DEPRECATION_WARNINGS -> MSVC gives warnings on things like std::aligned_storage, which is still valid in C++20.
     target_compile_definitions(ichor PUBLIC BOOST_ASIO_NO_DEPRECATED _SILENCE_ALL_CXX23_DEPRECATION_WARNINGS)
 
@@ -424,8 +432,8 @@ if(ICHOR_USE_ETCD)
 endif()
 
 # Detection for backward-cpp, works without external libraries on windows and apple
-# On linux, it still works without these libraries (on gcc at least), but provides less functionality.
-if(ICHOR_USE_BACKWARD)
+# On linux, it still works without these libraries (on gcc + glibc at least), but provides less functionality.
+if(ICHOR_USE_BACKWARD AND NOT ICHOR_MUSL)
     target_compile_definitions(ichor PUBLIC ICHOR_USE_BACKWARD=1)
     if(NOT WIN32 AND NOT APPLE)
         find_package(PkgConfig REQUIRED)

Dockerfile

@@ -7,13 +7,16 @@ RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 60
 RUN update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 60
 RUN update-alternatives --install /usr/bin/cpp cpp /usr/bin/cpp-12 60
 
+# Run all downloads first to be able to use Docker's layers as cache and prevent excessive redownloads
+
 WORKDIR /opt
+RUN wget https://boostorg.jfrog.io/artifactory/main/release/1.83.0/source/boost_1_83_0.tar.bz2
+RUN wget https://github.com/redis/hiredis/archive/refs/tags/v1.2.0.tar.gz
 
 #Build a new enough boost, apt only contains 1.74 which is too old.
-RUN wget https://boostorg.jfrog.io/artifactory/main/release/1.81.0/source/boost_1_81_0.tar.bz2
-RUN tar xf boost_1_81_0.tar.bz2
+RUN tar xf boost_1_83_0.tar.bz2
 
-WORKDIR /opt/boost_1_81_0
+WORKDIR /opt/boost_1_83_0
 
 RUN ./bootstrap.sh --prefix=/usr
 RUN ./b2 variant=release link=static threading=multi
@@ -22,7 +25,6 @@ RUN ./b2 variant=release link=static threading=multi install
 WORKDIR /opt
 
 #Build latest hiredis containing sdevent support, not available yet in apt
-RUN wget https://github.com/redis/hiredis/archive/refs/tags/v1.2.0.tar.gz
 RUN tar xf v1.2.0.tar.gz
 RUN mkdir /opt/hiredis-1.2.0/build
 
@@ -36,4 +38,4 @@ WORKDIR /opt/ichor/build
 
 ENTRYPOINT ["/bin/bash", "-c"]
 
-CMD ["cd /opt/ichor/build && cmake -GNinja -DCMAKE_BUILD_TYPE=RelWithDebInfo -DICHOR_USE_SANITIZERS=0 -DICHOR_USE_HIREDIS=1 -DICHOR_USE_BOOST_BEAST=ON -DICHOR_USE_SPDLOG=ON /opt/ichor/src && ninja"]
+CMD ["cd /opt/ichor/build && cmake -GNinja -DCMAKE_BUILD_TYPE=RelWithDebInfo -DICHOR_USE_SANITIZERS=0 -DICHOR_USE_HIREDIS=1 -DICHOR_USE_BOOST_BEAST=1 -DICHOR_USE_SPDLOG=1 /opt/ichor/src && ninja"]

Dockerfile-asan

@@ -7,13 +7,19 @@ RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 60
 RUN update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 60
 RUN update-alternatives --install /usr/bin/cpp cpp /usr/bin/cpp-12 60
 
+# Run all downloads first to be able to use Docker's layers as cache and prevent excessive redownloads
+
 WORKDIR /opt
+RUN wget https://boostorg.jfrog.io/artifactory/main/release/1.83.0/source/boost_1_83_0.tar.bz2
+RUN wget https://github.com/redis/hiredis/archive/refs/tags/v1.2.0.tar.gz
+
+ENV CFLAGS="-Og -fsanitize=address,undefined"
+ENV CXXFLAGS="-Og -fsanitize=address,undefined"
 
 #Build boost with support for asan
-RUN wget https://boostorg.jfrog.io/artifactory/main/release/1.81.0/source/boost_1_81_0.tar.bz2
-RUN tar xf boost_1_81_0.tar.bz2
+RUN tar xf boost_1_83_0.tar.bz2
 
-WORKDIR /opt/boost_1_81_0
+WORKDIR /opt/boost_1_83_0
 
 RUN ./bootstrap.sh --prefix=/usr
 RUN ./b2 cxxflags="-fsanitize=address,undefined -Og -std=c++17 -DBOOST_USE_ASAN -DBOOST_USE_UCONTEXT"  linkflags="-lasan -lubsan" variant=debug link=static threading=multi context-impl=ucontext
@@ -22,7 +28,6 @@ RUN ./b2 cxxflags="-fsanitize=address,undefined -Og -std=c++17 -DBOOST_USE_ASAN
 WORKDIR /opt
 
 #Build latest hiredis containing sdevent support, not available yet in apt
-RUN wget https://github.com/redis/hiredis/archive/refs/tags/v1.2.0.tar.gz
 RUN tar xf v1.2.0.tar.gz
 RUN mkdir /opt/hiredis-1.2.0/build
 
@@ -34,6 +39,9 @@ RUN mkdir -p /opt/ichor/build
 
 WORKDIR /opt/ichor/build
 
+RUN unset CFLAGS
+RUN unset CXXFLAGS
+
 ENTRYPOINT ["/bin/bash", "-c"]
 
-CMD ["cd /opt/ichor/build && cmake -GNinja -DCMAKE_BUILD_TYPE=Debug -DICHOR_USE_SANITIZERS=1 -DICHOR_USE_HIREDIS=1 -DICHOR_USE_BOOST_BEAST=ON -DICHOR_USE_SPDLOG=ON /opt/ichor/src && ninja"]
+CMD ["cd /opt/ichor/build && cmake -GNinja -DCMAKE_BUILD_TYPE=Debug -DICHOR_USE_SANITIZERS=1 -DICHOR_USE_HIREDIS=1 -DICHOR_USE_BOOST_BEAST=1 -DICHOR_USE_SPDLOG=1 /opt/ichor/src && ninja"]

Dockerfile-musl

@@ -1,17 +1,26 @@
 FROM alpine:3.17
 
 RUN apk update
-RUN apk add gcc g++ build-base cmake openssl-dev git wget make nano sed linux-headers
+RUN apk add gcc g++ build-base cmake git wget make nano sed linux-headers perl
 
 # Run all downloads first to be able to use Docker's layers as cache and prevent excessive redownloads
 
 WORKDIR /opt
 RUN wget https://boostorg.jfrog.io/artifactory/main/release/1.81.0/source/boost_1_81_0.tar.bz2
 RUN wget https://github.com/redis/hiredis/archive/refs/tags/v1.2.0.tar.gz
+RUN wget https://www.openssl.org/source/openssl-3.0.11.tar.gz
+
+ENV LDFLAGS="-static-libgcc -static-libstdc++ -static"
+#Build openssl statically, alpine (and probably most distros) only provide shared libraries. Might be a security thing?
+RUN tar xf openssl-3.0.11.tar.gz
+WORKDIR /opt/openssl-3.0.11
+RUN ./Configure --prefix=/usr --openssldir=/etc/ssl --libdir=lib no-shared
+RUN make -j
+RUN make -j install
 
 WORKDIR /opt
 
-#Build a new enough boost, apt only contains 1.74 which is too old.
+#Build boost statically
 RUN tar xf boost_1_81_0.tar.bz2
 
 WORKDIR /opt/boost_1_81_0
@@ -27,7 +36,6 @@ RUN tar xf v1.2.0.tar.gz
 RUN mkdir /opt/hiredis-1.2.0/build
 
 WORKDIR /opt/hiredis-1.2.0/build
-ENV LDFLAGS="-static-libgcc -static-libstdc++ -static"
 RUN cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo -DDISABLE_TESTS=1 -DCMAKE_INSTALL_PREFIX=/usr -DENABLE_SSL=1 -DBUILD_SHARED_LIBS=0 ..
 RUN make -j && make install
 RUN mkdir -p /opt/ichor/build
@@ -36,4 +44,4 @@ WORKDIR /opt/ichor/build
 
 ENTRYPOINT ["/bin/sh", "-c"]
 
-CMD ["cd /opt/ichor/build && cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo -DICHOR_USE_SANITIZERS=0 -DICHOR_USE_HIREDIS=1 -DICHOR_USE_BOOST_BEAST=1 -DICHOR_USE_SPDLOG=ON -DICHOR_MUSL=1 /opt/ichor/src && make -j"]
+CMD ["cd /opt/ichor/build && cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo -DICHOR_USE_SANITIZERS=0 -DICHOR_USE_HIREDIS=1 -DICHOR_USE_BOOST_BEAST=1 -DICHOR_USE_SPDLOG=1 -DICHOR_MUSL=1 /opt/ichor/src && make -j"]

Dockerfile-musl-aarch64

@@ -0,0 +1,47 @@
+FROM arm64v8/alpine:3.17
+
+RUN apk update
+RUN apk add gcc g++ build-base cmake git wget make nano sed linux-headers perl
+
+# Run all downloads first to be able to use Docker's layers as cache and prevent excessive redownloads
+
+WORKDIR /opt
+RUN wget https://boostorg.jfrog.io/artifactory/main/release/1.81.0/source/boost_1_81_0.tar.bz2
+RUN wget https://github.com/redis/hiredis/archive/refs/tags/v1.2.0.tar.gz
+RUN wget https://www.openssl.org/source/openssl-3.0.11.tar.gz
+
+ENV LDFLAGS="-static-libgcc -static-libstdc++ -static"
+#Build openssl statically, alpine (and probably most distros) only provide shared libraries. Might be a security thing?
+RUN tar xf openssl-3.0.11.tar.gz
+WORKDIR /opt/openssl-3.0.11
+RUN ./Configure --prefix=/usr --openssldir=/etc/ssl --libdir=lib no-shared
+RUN make -j
+RUN make -j install
+
+WORKDIR /opt
+
+#Build boost statically
+RUN tar xf boost_1_81_0.tar.bz2
+
+WORKDIR /opt/boost_1_81_0
+
+RUN ./bootstrap.sh --prefix=/usr
+RUN ./b2 variant=release link=static threading=multi cxxflags="-O2 -std=c++17 -static -fstack-protector-strong -fstack-clash-protection" linkflags="-static-libgcc -static-libstdc++ -static"
+RUN ./b2 variant=release link=static threading=multi cxxflags="-O2 -std=c++17 -static -fstack-protector-strong -fstack-clash-protection" linkflags="-static-libgcc -static-libstdc++ -static" install
+
+WORKDIR /opt
+
+#Build latest hiredis containing sdevent support, not available yet in apt
+RUN tar xf v1.2.0.tar.gz
+RUN mkdir /opt/hiredis-1.2.0/build
+
+WORKDIR /opt/hiredis-1.2.0/build
+RUN cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo -DDISABLE_TESTS=1 -DCMAKE_INSTALL_PREFIX=/usr -DENABLE_SSL=1 -DBUILD_SHARED_LIBS=0 ..
+RUN make -j && make install
+RUN mkdir -p /opt/ichor/build
+
+WORKDIR /opt/ichor/build
+
+ENTRYPOINT ["/bin/sh", "-c"]
+
+CMD ["cd /opt/ichor/build && cmake -DCMAKE_BUILD_TYPE=Debug -DICHOR_USE_SANITIZERS=0 -DICHOR_USE_HIREDIS=1 -DICHOR_USE_BOOST_BEAST=1 -DICHOR_USE_SPDLOG=1 -DICHOR_MUSL=1 -DICHOR_AARCH64=1 /opt/ichor/src && make -j"]

Dockerfile-tsan

@@ -0,0 +1,85 @@
+FROM ubuntu:jammy
+
+RUN apt update
+RUN apt install -y g++-12 gcc-12 build-essential cmake pkg-config git wget ninja-build nano libzip-dev
+
+RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 60
+RUN update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 60
+RUN update-alternatives --install /usr/bin/cpp cpp /usr/bin/cpp-12 60
+
+# Run all downloads first to be able to use Docker's layers as cache and prevent excessive redownloads
+
+WORKDIR /opt
+RUN wget https://github.com/redis/hiredis/archive/refs/tags/v1.2.0.tar.gz
+RUN wget https://boostorg.jfrog.io/artifactory/main/release/1.81.0/source/boost_1_81_0.tar.bz2
+RUN wget https://www.openssl.org/source/openssl-3.0.11.tar.gz
+
+ENV CFLAGS="-Og -fsanitize=thread"
+ENV CXXFLAGS="-Og -fsanitize=thread"
+ENV LDFLAGS="-fsanitize=thread -static-libtsan -static-libgcc -static-libstdc++"
+
+RUN tar xf openssl-3.0.11.tar.gz
+WORKDIR /opt/openssl-3.0.11
+RUN ./Configure --prefix=/usr --openssldir=/etc/ssl --libdir=lib no-shared
+RUN make -j
+RUN make -j install
+
+WORKDIR /opt
+
+#Build boost with support for asan
+RUN tar xf boost_1_81_0.tar.bz2
+
+WORKDIR /opt/boost_1_81_0
+
+RUN <<EOF
+cat >> patch << EOR
+diff -r -u boost-1.81.0/boost/asio/detail/std_fenced_block.hpp boost-1.81.0-fencecd/boost/asio/detail/std_fenced_block.hpp
+--- ./boost/asio/detail/std_fenced_block.hpp	2022-12-14 16:15:35.000000000 +0100
++++ ./boost/asio/detail/std_fenced_block.hpp	2023-10-03 20:17:11.701435674 +0200
+@@ -43,14 +43,16 @@
+   // Constructor for a full fenced block.
+   explicit std_fenced_block(full_t)
+   {
+-    std::atomic_thread_fence(std::memory_order_acquire);
++    _fence.load(std::memory_order_acquire);
+   }
+
+   // Destructor.
+   ~std_fenced_block()
+   {
+-    std::atomic_thread_fence(std::memory_order_release);
++    _fence.store(true, std::memory_order_release);
+   }
++
++  std::atomic<bool> _fence;
+ };
+
+ } // namespace detail
+EOR
+EOF
+RUN patch ./boost/asio/detail/std_fenced_block.hpp < patch
+
+RUN ./bootstrap.sh --prefix=/usr
+RUN ./b2 cxxflags="-fsanitize=thread -Og -std=c++17 -DBOOST_USE_TSAN"  linkflags="-static-libtsan -static-libgcc -static-libstdc++" variant=debug link=static threading=multi
+RUN ./b2 cxxflags="-fsanitize=thread -Og -std=c++17 -DBOOST_USE_TSAN"  linkflags="-static-libtsan -static-libgcc -static-libstdc++" variant=debug link=static threading=multi install
+
+WORKDIR /opt
+
+#Build latest hiredis containing sdevent support, not available yet in apt
+RUN tar xf v1.2.0.tar.gz
+RUN mkdir /opt/hiredis-1.2.0/build
+
+WORKDIR /opt/hiredis-1.2.0/build
+RUN cmake -GNinja -DCMAKE_BUILD_TYPE=RelWithDebInfo -DDISABLE_TESTS=1 -DCMAKE_INSTALL_PREFIX=/usr -DENABLE_SSL=1 ..
+RUN ninja && ninja install
+
+RUN mkdir -p /opt/ichor/build
+
+WORKDIR /opt/ichor/build
+
+RUN unset CFLAGS
+RUN unset CXXFLAGS
+
+ENTRYPOINT ["/bin/bash", "-c"]
+
+CMD ["cd /opt/ichor/build && cmake -GNinja -DCMAKE_BUILD_TYPE=Debug -DICHOR_USE_SANITIZERS=0 -DICHOR_USE_THREAD_SANITIZER=1 -DICHOR_USE_HIREDIS=1 -DICHOR_USE_BOOST_BEAST=1 -DICHOR_USE_SPDLOG=1 -DICHOR_USE_BACKWARD=0 /opt/ichor/src && ninja"]