diff --git a/CMakeLists.txt b/CMakeLists.txt
index 7a23683ea..1f3c009e5 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -68,8 +68,9 @@ if(CMAKE_CXX_COMPILER_FRONTEND_VARIANT STREQUAL "MSVC")
         set(CMAKE_MAP_IMPORTED_CONFIG_RELWITHDEBINFO Release "")
     endif()
 else()
-    add_compile_options("$<$<COMPILE_LANGUAGE:C,CXX>:-D_FORTIFY_SOURCE=2;-fstack-protector-strong;--param=ssp-buffer-size=4>")
-
+    add_compile_options("$<$<COMPILE_LANGUAGE:C,CXX>:-fstack-protector-strong;--param=ssp-buffer-size=4>")
+    # on non-debug builds we can enable optimizations
+    add_compile_options("$<$<AND:$<COMPILE_LANGUAGE:C,CXX>,$<NOT:$<CONFIG:Debug>>>:-O2;-D_FORTIFY_SOURCE=2>")
     # ATL's pack list needs more than the default 1 Mib stack on windows
     if(CMAKE_SYSTEM_NAME STREQUAL "Windows")
         add_link_options("$<$<COMPILE_LANGUAGE:C,CXX>:-Wl,--stack,8388608>")
@@ -136,8 +137,9 @@ if(ENABLE_LTO)
     if(ipo_supported)
         set(CMAKE_INTERPROCEDURAL_OPTIMIZATION_RELEASE TRUE)
         set(CMAKE_INTERPROCEDURAL_OPTIMIZATION_MINSIZEREL TRUE)
+        set(CMAKE_INTERPROCEDURAL_OPTIMIZATION_RELWITHDEBINFO TRUE)
         if(CMAKE_BUILD_TYPE)
-            if(CMAKE_BUILD_TYPE STREQUAL "Release" OR CMAKE_BUILD_TYPE STREQUAL "MinSizeRel")
+            if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug")
                 message(STATUS "IPO / LTO enabled")
             else()
                 message(STATUS "Not enabling IPO / LTO on debug builds")