session: add libssh2_session_callback_set2()

Add new `libssh2_session_callback_set2()` API that deprecates
`libssh2_session_callback_set()`.

The new implementation offers the same functionality, but accepts and
returns a generic function pointer (of type `libssh2_cb_generic *`), as
opposed to the old function that used data pointers (`void *`). The new
solution thus avoids data to function (and vice versa) pointer
conversions, which has undefined behaviour in standard C.

About the name: It seems the `*2` suffix was used in the past for
replacement functions for deprecated ones. Let's stick with that.
`*_ex` was preferred for new functions that extend existing ones with
new features.

Closes #1285

docs/Makefile.am

@@ -103,6 +103,7 @@ dist_man_MANS = \
   libssh2_session_banner_set.3 \
   libssh2_session_block_directions.3 \
   libssh2_session_callback_set.3 \
+  libssh2_session_callback_set2.3 \
   libssh2_session_disconnect.3 \
   libssh2_session_disconnect_ex.3 \
   libssh2_session_flag.3 \

docs/TODO

@@ -34,8 +34,9 @@ At next SONAME bump
 * stop using #defined macros as part of the official API. The macros should
   either be turned into real functions or discarded from the API.
 
-* fix the parts of the API where object pointers and function pointers are
+* delete or deprecate libssh2_session_callback_set()
-  mixed like libssh2_session_callback_set()
+
+* bump length arguments in callback functions to size_t/ssize_t
 
 * remove the following functions from the API/ABI
 

docs/libssh2_agent_sign.3

@@ -39,7 +39,7 @@ string at the start of identity->blob.
 \fImethod_len\fP - The length of the method parameter.
 
 Sign data using an ssh-agent. This function can be used in a callback
-registered with libssh2_session_callback_set(3) using
+registered with libssh2_session_callback_set2(3) using
 LIBSSH2_CALLBACK_AUTHAGENT_SIGN to sign an authentication challenge from a
 server. However, the client is responsible for implementing the code that calls
 this callback in response to a SSH2_AGENTC_SIGN_REQUEST message.
@@ -51,4 +51,4 @@ Added in libssh2 1.11.0
 .BR libssh2_agent_init(3)
 .BR libssh2_agent_get_identity(3)
 .BR libssh2_agent_userauth(3)
-.BR libssh2_session_callback_set(3)
+.BR libssh2_session_callback_set2(3)

docs/libssh2_channel_x11_req_ex.3

@@ -29,7 +29,7 @@ libssh2_channel_x11_req(LIBSSH2_CHANNEL *channel,
 \fIscreen_number\fP - the XLL screen to forward
 
 Request an X11 forwarding on \fIchannel\fP. To use X11 forwarding,
-.BR libssh2_session_callback_set(3)
+.BR libssh2_session_callback_set2(3)
 must first be called to set \fBLIBSSH2_CALLBACK_X11\fP. This callback will be
 invoked when the remote host accepts the X11 forwarding.
 .SH RETURN VALUE
@@ -44,4 +44,4 @@ LIBSSH2_ERROR_EAGAIN is a negative number, it is not really a failure per se.
 \fILIBSSH2_ERROR_CHANNEL_REQUEST_DENIED\fP -
 .SH SEE ALSO
 .BR libssh2_channel_open_ex(3)
-.BR libssh2_session_callback_set(3)
+.BR libssh2_session_callback_set2(3)

docs/libssh2_session_callback_set.3

@@ -12,128 +12,20 @@ libssh2_session_callback_set(LIBSSH2_SESSION *session,
                              int cbtype, void *callback);
 .fi
 .SH DESCRIPTION
-Sets a custom callback handler for a previously initialized session
+This function is \fBDEPRECATED\fP in 1.11.1. Use the
-object. Callbacks are triggered by the receipt of special packets at the
+\fIlibssh2_session_callback_set2(3)\fP function instead!
-Transport layer. To disable a callback, set it to NULL.
 
-\fIsession\fP - Session instance as returned by
+This implementation is expecting and returning a data pointer for callback
-.BR libssh2_session_init_ex(3)
+functions.
 
-\fIcbtype\fP - Callback type. One of the types listed in Callback Types.
+For the details about the replacement function, see
-
+.BR libssh2_session_callback_set2(3)
-\fIcallback\fP - Pointer to custom callback function. The prototype for
+which is expecting and returning a function pointer.
-this function must match the associated callback declaration macro.
-.SH CALLBACK TYPES
-.IP LIBSSH2_CALLBACK_IGNORE
-Called when a SSH_MSG_IGNORE message is received
-.IP LIBSSH2_CALLBACK_DEBUG
-Called when a SSH_MSG_DEBUG message is received
-.IP LIBSSH2_CALLBACK_DISCONNECT
-Called when a SSH_MSG_DISCONNECT message is received
-.IP LIBSSH2_CALLBACK_MACERROR
-Called when a mismatched MAC has been detected in the transport layer. If the
-function returns 0, the packet will be accepted nonetheless.
-.IP LIBSSH2_CALLBACK_X11
-Called when an X11 connection has been accepted
-.IP LIBSSH2_CALLBACK_SEND
-Called when libssh2 wants to send data on the connection. Can be set to a
-custom function to handle I/O your own way.
-
-The prototype of the callback:
-
-.nf
-ssize_t sendcb(libssh2_socket_t sockfd, const void *buffer,
-               size_t length, int flags, void **abstract);
-.fi
-
-\fBsockfd\fP is the socket to write to, \fBbuffer\fP points to the data to
-send, \fBlength\fP is the size of the data, \fBflags\fP is the flags that
-would have been used to a \fIsend()\fP call and \fBabstract\fP is a pointer
-to the abstract pointer set in the \fIlibssh2_session_init_ex(3)\fP call.
-
-The callback returns the number of bytes sent, or \-1 for error. The special
-return code \fB-EAGAIN\fP can be returned to signal that the send was aborted
-to prevent getting blocked and it needs to be called again.
-.IP LIBSSH2_CALLBACK_RECV
-Called when libssh2 wants to read data from the connection. Can be set to a
-custom function to handle I/O your own way.
-
-The prototype of the callback:
-
-.nf
-ssize_t recvcb(libssh2_socket_t sockfd, void *buffer,
-               size_t length, int flags, void **abstract);
-.fi
-
-\fBsockfd\fP is the socket to read from, \fBbuffer\fP where to store received
-data into, \fBlength\fP is the size of the buffer, \fBflags\fP is the flags
-that would have been used to a \fIrecv()\fP call and \fBabstract\fP is a pointer
-to the abstract pointer set in the \fIlibssh2_session_init_ex(3)\fP call.
-
-The callback returns the number of bytes read, or \-1 for error. The special
-return code \fB-EAGAIN\fP can be returned to signal that the read was aborted
-to prevent getting blocked and it needs to be called again.
-.IP LIBSSH2_CALLBACK_AUTHAGENT
-Called during authentication process to allow the client to connect to the
-ssh-agent and perform any setup, such as configuring the agent or adding keys.
-
-The prototype of the callback:
-
-.nf
-void authagent(LIBSSH2_SESSION* session, LIBSSH2_CHANNEL *channel,
-               void **abstract);
-.fi
-.IP LIBSSH2_CALLBACK_AUTHAGENT_IDENTITIES
-Not called by libssh2. The client is responsible for calling this method when
-a SSH2_AGENTC_REQUEST_IDENTITIES message has been received.
-
-The prototype of the callback:
-
-.nf
-void identities(LIBSSH2_SESSION* session, void *buffer,
-                const char *agent_path,
-                void **abstract)
-.fi
-
-\fBbuffer\fP must be filled in by the callback. Different clients may implement
-this differently. For example, one client may pass in an unsigned char ** for
-this parameter, while another may pass in a pointer to a struct.
-
-Regardless of the type of buffer used, the client will need to send back a list
-of identities in the following format.
-
-uint32 buffer length
-uint32 number of entries
-entries
-
-Where each entry in the entries list is of the format:
-
-string data
-cstring comment
-
-\fBagent_path\fP The path to a running ssh-agent on the client machine, from
-which identities can be listed.
-.IP LIBSSH2_CALLBACK_AUTHAGENT_SIGN
-Not called by libssh2. The client is responsible for calling this method when
-a SSH2_AGENTC_SIGN_REQUEST message has been received.
-
-The prototype of the callback:
-
-.nf
-void sign(LIBSSH2_SESSION* session,
-          unsigned char *blob, unsigned int blen,
-          const unsigned char *data, unsigned int dlen,
-          unsigned char **sig, unsigned int *sig_len,
-          const char *agent_path,
-          void **abstract);
-.fi
-
-When interfacing with an ssh-agent installed on the client system, this method
-can call libssh2_agent_sign(3) to perform signing.
 
 .SH RETURN VALUE
 Pointer to previous callback handler. Returns NULL if no prior callback
 handler was set or the callback type was unknown.
 .SH SEE ALSO
+.BR libssh2_session_callback_set2(3)
 .BR libssh2_session_init_ex(3)
 .BR libssh2_agent_sign(3)

docs/libssh2_session_callback_set2.3

@@ -0,0 +1,139 @@
+.\" Copyright (C) The libssh2 project and its contributors.
+.\" SPDX-License-Identifier: BSD-3-Clause
+.TH libssh2_session_callback_set2 3 "13 Dec 2023" "libssh2 1.11.1" "libssh2"
+.SH NAME
+libssh2_session_callback_set2 - set a callback function
+.SH SYNOPSIS
+.nf
+#include <libssh2.h>
+
+libssh2_cb_generic *
+libssh2_session_callback_set2(LIBSSH2_SESSION *session, int cbtype,
+                              libssh2_cb_generic *callback);
+.fi
+.SH DESCRIPTION
+Sets a custom callback handler for a previously initialized session
+object. Callbacks are triggered by the receipt of special packets at the
+Transport layer. To disable a callback, set it to NULL.
+
+\fIsession\fP - Session instance as returned by
+.BR libssh2_session_init_ex(3)
+
+\fIcbtype\fP - Callback type. One of the types listed in Callback Types.
+
+\fIcallback\fP - Pointer to custom callback function. The prototype for
+this function must match the associated callback declaration macro.
+.SH CALLBACK TYPES
+.IP LIBSSH2_CALLBACK_IGNORE
+Called when a SSH_MSG_IGNORE message is received
+.IP LIBSSH2_CALLBACK_DEBUG
+Called when a SSH_MSG_DEBUG message is received
+.IP LIBSSH2_CALLBACK_DISCONNECT
+Called when a SSH_MSG_DISCONNECT message is received
+.IP LIBSSH2_CALLBACK_MACERROR
+Called when a mismatched MAC has been detected in the transport layer. If the
+function returns 0, the packet will be accepted nonetheless.
+.IP LIBSSH2_CALLBACK_X11
+Called when an X11 connection has been accepted
+.IP LIBSSH2_CALLBACK_SEND
+Called when libssh2 wants to send data on the connection. Can be set to a
+custom function to handle I/O your own way.
+
+The prototype of the callback:
+
+.nf
+ssize_t sendcb(libssh2_socket_t sockfd, const void *buffer,
+               size_t length, int flags, void **abstract);
+.fi
+
+\fBsockfd\fP is the socket to write to, \fBbuffer\fP points to the data to
+send, \fBlength\fP is the size of the data, \fBflags\fP is the flags that
+would have been used to a \fIsend()\fP call and \fBabstract\fP is a pointer
+to the abstract pointer set in the \fIlibssh2_session_init_ex(3)\fP call.
+
+The callback returns the number of bytes sent, or \-1 for error. The special
+return code \fB-EAGAIN\fP can be returned to signal that the send was aborted
+to prevent getting blocked and it needs to be called again.
+.IP LIBSSH2_CALLBACK_RECV
+Called when libssh2 wants to read data from the connection. Can be set to a
+custom function to handle I/O your own way.
+
+The prototype of the callback:
+
+.nf
+ssize_t recvcb(libssh2_socket_t sockfd, void *buffer,
+               size_t length, int flags, void **abstract);
+.fi
+
+\fBsockfd\fP is the socket to read from, \fBbuffer\fP where to store received
+data into, \fBlength\fP is the size of the buffer, \fBflags\fP is the flags
+that would have been used to a \fIrecv()\fP call and \fBabstract\fP is a pointer
+to the abstract pointer set in the \fIlibssh2_session_init_ex(3)\fP call.
+
+The callback returns the number of bytes read, or \-1 for error. The special
+return code \fB-EAGAIN\fP can be returned to signal that the read was aborted
+to prevent getting blocked and it needs to be called again.
+.IP LIBSSH2_CALLBACK_AUTHAGENT
+Called during authentication process to allow the client to connect to the
+ssh-agent and perform any setup, such as configuring the agent or adding keys.
+
+The prototype of the callback:
+
+.nf
+void authagent(LIBSSH2_SESSION* session, LIBSSH2_CHANNEL *channel,
+               void **abstract);
+.fi
+.IP LIBSSH2_CALLBACK_AUTHAGENT_IDENTITIES
+Not called by libssh2. The client is responsible for calling this method when
+a SSH2_AGENTC_REQUEST_IDENTITIES message has been received.
+
+The prototype of the callback:
+
+.nf
+void identities(LIBSSH2_SESSION* session, void *buffer,
+                const char *agent_path,
+                void **abstract)
+.fi
+
+\fBbuffer\fP must be filled in by the callback. Different clients may implement
+this differently. For example, one client may pass in an unsigned char ** for
+this parameter, while another may pass in a pointer to a struct.
+
+Regardless of the type of buffer used, the client will need to send back a list
+of identities in the following format.
+
+uint32 buffer length
+uint32 number of entries
+entries
+
+Where each entry in the entries list is of the format:
+
+string data
+cstring comment
+
+\fBagent_path\fP The path to a running ssh-agent on the client machine, from
+which identities can be listed.
+.IP LIBSSH2_CALLBACK_AUTHAGENT_SIGN
+Not called by libssh2. The client is responsible for calling this method when
+a SSH2_AGENTC_SIGN_REQUEST message has been received.
+
+The prototype of the callback:
+
+.nf
+void sign(LIBSSH2_SESSION* session,
+          unsigned char *blob, unsigned int blen,
+          const unsigned char *data, unsigned int dlen,
+          unsigned char **sig, unsigned int *sig_len,
+          const char *agent_path,
+          void **abstract);
+.fi
+
+When interfacing with an ssh-agent installed on the client system, this method
+can call libssh2_agent_sign(3) to perform signing.
+
+.SH RETURN VALUE
+Pointer to previous callback handler. Returns NULL if no prior callback
+handler was set or the callback type was unknown.
+.SH SEE ALSO
+.BR libssh2_session_init_ex(3)
+.BR libssh2_agent_sign(3)

example/x11.c

@@ -339,13 +339,9 @@ int main(int argc, char *argv[])
     if(set_debug_on == 1)
         libssh2_trace(session, LIBSSH2_TRACE_CONN);
 
-    /* ignore pedantic warnings by gcc on the callback argument */
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wpedantic"
     /* Set X11 Callback */
-    libssh2_session_callback_set(session, LIBSSH2_CALLBACK_X11,
+    libssh2_session_callback_set2(session, LIBSSH2_CALLBACK_X11,
-                                 (void *)x11_callback);
+                                  (libssh2_cb_generic *)x11_callback);
-#pragma GCC diagnostic pop
 
     /* Authenticate via password */
     rc = libssh2_userauth_password(session, username, password);

include/libssh2.h

@@ -330,6 +330,8 @@ typedef struct _LIBSSH2_SK_SIG_INFO {
 #define LIBSSH2_SK_PRESENCE_REQUIRED     0x01
 #define LIBSSH2_SK_VERIFICATION_REQUIRED 0x04
 
+/* FIXME: update lengths to size_t (or ssize_t): */
+
 /* Callbacks for special SSH packets */
 #define LIBSSH2_IGNORE_FUNC(name) \
     void name(LIBSSH2_SESSION *session, const char *message, int message_len, \
@@ -645,6 +647,13 @@ libssh2_session_init_ex(LIBSSH2_ALLOC_FUNC((*my_alloc)),
 
 LIBSSH2_API void **libssh2_session_abstract(LIBSSH2_SESSION *session);
 
+typedef void (libssh2_cb_generic)(void);
+
+LIBSSH2_API libssh2_cb_generic *
+libssh2_session_callback_set2(LIBSSH2_SESSION *session, int cbtype,
+                              libssh2_cb_generic *callback);
+
+LIBSSH2_DEPRECATED(1.11.1, "Use libssh2_session_callback_set2()")
 LIBSSH2_API void *libssh2_session_callback_set(LIBSSH2_SESSION *session,
                                                int cbtype, void *callback);
 LIBSSH2_API int libssh2_session_banner_set(LIBSSH2_SESSION *session,

src/channel.c

@@ -1932,9 +1932,7 @@ _libssh2_channel_receive_window_adjust(LIBSSH2_CHANNEL * channel,
 
 #ifndef LIBSSH2_NO_DEPRECATED
 /*
- * libssh2_channel_receive_window_adjust
+ * libssh2_channel_receive_window_adjust (DEPRECATED, DO NOT USE!)
- *
- * DEPRECATED
  *
  * Adjust the receive window for a channel by adjustment bytes. If the amount
  * to be adjusted is less than LIBSSH2_CHANNEL_MINADJUST and force is 0 the
@@ -2042,9 +2040,7 @@ libssh2_channel_handle_extended_data2(LIBSSH2_CHANNEL *channel,
 
 #ifndef LIBSSH2_NO_DEPRECATED
 /*
- * libssh2_channel_handle_extended_data
+ * libssh2_channel_handle_extended_data (DEPRECATED, DO NOT USE!)
- *
- * DEPRECATED DO NOTE USE!
  *
  * How should extended data look to the calling app?  Keep it in separate
  * channels[_read() _read_stdder()]? (NORMAL) Merge the extended data to the

src/session.c

@@ -480,7 +480,78 @@ libssh2_session_init_ex(LIBSSH2_ALLOC_FUNC((*my_alloc)),
 }
 
 /*
- * libssh2_session_callback_set
+ * libssh2_session_callback_set2
+ *
+ * Set (or reset) a callback function
+ * Returns the prior address
+ */
+LIBSSH2_API libssh2_cb_generic *
+libssh2_session_callback_set2(LIBSSH2_SESSION *session, int cbtype,
+                              libssh2_cb_generic *callback)
+{
+    libssh2_cb_generic *oldcb;
+
+    switch(cbtype) {
+    case LIBSSH2_CALLBACK_IGNORE:
+        oldcb = (libssh2_cb_generic *)session->ssh_msg_ignore;
+        session->ssh_msg_ignore = (LIBSSH2_IGNORE_FUNC((*)))callback;
+        return oldcb;
+
+    case LIBSSH2_CALLBACK_DEBUG:
+        oldcb = (libssh2_cb_generic *)session->ssh_msg_debug;
+        session->ssh_msg_debug = (LIBSSH2_DEBUG_FUNC((*)))callback;
+        return oldcb;
+
+    case LIBSSH2_CALLBACK_DISCONNECT:
+        oldcb = (libssh2_cb_generic *)session->ssh_msg_disconnect;
+        session->ssh_msg_disconnect = (LIBSSH2_DISCONNECT_FUNC((*)))callback;
+        return oldcb;
+
+    case LIBSSH2_CALLBACK_MACERROR:
+        oldcb = (libssh2_cb_generic *)session->macerror;
+        session->macerror = (LIBSSH2_MACERROR_FUNC((*)))callback;
+        return oldcb;
+
+    case LIBSSH2_CALLBACK_X11:
+        oldcb = (libssh2_cb_generic *)session->x11;
+        session->x11 = (LIBSSH2_X11_OPEN_FUNC((*)))callback;
+        return oldcb;
+
+    case LIBSSH2_CALLBACK_SEND:
+        oldcb = (libssh2_cb_generic *)session->send;
+        session->send = (LIBSSH2_SEND_FUNC((*)))callback;
+        return oldcb;
+
+    case LIBSSH2_CALLBACK_RECV:
+        oldcb = (libssh2_cb_generic *)session->recv;
+        session->recv = (LIBSSH2_RECV_FUNC((*)))callback;
+        return oldcb;
+
+    case LIBSSH2_CALLBACK_AUTHAGENT:
+        oldcb = (libssh2_cb_generic *)session->authagent;
+        session->authagent = (LIBSSH2_AUTHAGENT_FUNC((*)))callback;
+        return oldcb;
+
+    case LIBSSH2_CALLBACK_AUTHAGENT_IDENTITIES:
+        oldcb = (libssh2_cb_generic *)session->addLocalIdentities;
+        session->addLocalIdentities =
+            (LIBSSH2_ADD_IDENTITIES_FUNC((*)))callback;
+        return oldcb;
+
+    case LIBSSH2_CALLBACK_AUTHAGENT_SIGN:
+        oldcb = (libssh2_cb_generic *)session->agentSignCallback;
+        session->agentSignCallback =
+            (LIBSSH2_AUTHAGENT_SIGN_FUNC((*)))callback;
+        return oldcb;
+    }
+    _libssh2_debug((session, LIBSSH2_TRACE_TRANS, "Setting Callback %d",
+                   cbtype));
+
+    return NULL;
+}
+
+/*
+ * libssh2_session_callback_set (DEPRECATED, DO NOT USE!)
  *
  * Set (or reset) a callback function
  * Returns the prior address
@@ -490,8 +561,10 @@ libssh2_session_init_ex(LIBSSH2_ALLOC_FUNC((*my_alloc)),
  */
 #ifdef _MSC_VER
 #pragma warning(push)
-/* nonstandard extension, function/data pointer conversion in expression */
+/* 'type cast': from data pointer to function pointer */
-#pragma warning(disable:4152)
+#pragma warning(disable:4054)
+/* 'type cast': from function pointer to data pointer */
+#pragma warning(disable:4055)
 #else
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Wpedantic"
@@ -500,63 +573,8 @@ LIBSSH2_API void *
 libssh2_session_callback_set(LIBSSH2_SESSION * session,
                              int cbtype, void *callback)
 {
-    void *oldcb;
+    return (void *)libssh2_session_callback_set2(session, cbtype,
-
+                                               (libssh2_cb_generic *)callback);
-    switch(cbtype) {
-    case LIBSSH2_CALLBACK_IGNORE:
-        oldcb = session->ssh_msg_ignore;
-        session->ssh_msg_ignore = callback;
-        return oldcb;
-
-    case LIBSSH2_CALLBACK_DEBUG:
-        oldcb = session->ssh_msg_debug;
-        session->ssh_msg_debug = callback;
-        return oldcb;
-
-    case LIBSSH2_CALLBACK_DISCONNECT:
-        oldcb = session->ssh_msg_disconnect;
-        session->ssh_msg_disconnect = callback;
-        return oldcb;
-
-    case LIBSSH2_CALLBACK_MACERROR:
-        oldcb = session->macerror;
-        session->macerror = callback;
-        return oldcb;
-
-    case LIBSSH2_CALLBACK_X11:
-        oldcb = session->x11;
-        session->x11 = callback;
-        return oldcb;
-
-    case LIBSSH2_CALLBACK_SEND:
-        oldcb = session->send;
-        session->send = callback;
-        return oldcb;
-
-    case LIBSSH2_CALLBACK_RECV:
-        oldcb = session->recv;
-        session->recv = callback;
-        return oldcb;
-
-    case LIBSSH2_CALLBACK_AUTHAGENT:
-        oldcb = session->authagent;
-        session->authagent = callback;
-        return oldcb;
-
-    case LIBSSH2_CALLBACK_AUTHAGENT_IDENTITIES:
-        oldcb = session->addLocalIdentities;
-        session->addLocalIdentities = callback;
-        return oldcb;
-
-    case LIBSSH2_CALLBACK_AUTHAGENT_SIGN:
-        oldcb = session->agentSignCallback;
-        session->agentSignCallback = callback;
-        return oldcb;
-    }
-    _libssh2_debug((session, LIBSSH2_TRACE_TRANS, "Setting Callback %d",
-                   cbtype));
-
-    return NULL;
 }
 #ifdef _MSC_VER
 #pragma warning(pop)