core: Fix clang build

Recent changes to the build system that made more warnings be flagged as
errors caused building via clang to break.

Fixes #4795

src/audio_core/CMakeLists.txt

@@ -51,8 +51,9 @@ if (NOT MSVC)
         -Werror=implicit-fallthrough
         -Werror=reorder
         -Werror=sign-compare
-        -Werror=unused-but-set-parameter
-        -Werror=unused-but-set-variable
+        -Werror=sign-conversion
+        $<$<CXX_COMPILER_ID:GNU>:-Werror=unused-but-set-parameter>
+        $<$<CXX_COMPILER_ID:GNU>:-Werror=unused-but-set-variable>
         -Werror=unused-variable
     )
 endif()