diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 30629db0..35f130a2 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -135,9 +135,6 @@ set(libssh_SRCS
   wrapper.c
   external/bcrypt_pbkdf.c
   external/blowfish.c
-  external/chacha.c
-  external/poly1305.c
-  chachapoly.c
   config_parser.c
   token.c
   pki_ed25519_common.c
@@ -183,6 +180,14 @@ if (WITH_GCRYPT)
         external/ge25519.c
         external/sc25519.c
        )
+    if (NOT HAVE_GCRYPT_CHACHA_POLY)
+        set(libssh_SRCS
+            ${libssh_SRCS}
+            external/chacha.c
+            external/poly1305.c
+            chachapoly.c
+        )
+    endif (NOT HAVE_GCRYPT_CHACHA_POLY)
 elseif (WITH_MBEDTLS)
     set(libssh_SRCS
         ${libssh_SRCS}
@@ -197,6 +202,9 @@ elseif (WITH_MBEDTLS)
         external/fe25519.c
         external/ge25519.c
         external/sc25519.c
+        external/chacha.c
+        external/poly1305.c
+        chachapoly.c
        )
 else (WITH_GCRYPT)
     set(libssh_SRCS
@@ -206,6 +214,9 @@ else (WITH_GCRYPT)
         ecdh_crypto.c
         libcrypto.c
         dh_crypto.c
+        external/chacha.c
+        external/poly1305.c
+        chachapoly.c
        )
     if (NOT HAVE_OPENSSL_ED25519)
         set(libssh_SRCS