os400: implement character encoding conversion support

Makefile.am

@@ -34,9 +34,11 @@ win32/msvcproj.head win32/msvcproj.foot win32/libssh2.rc
 
 OS400FILES = os400/README400 os400/initscript.sh os400/make.sh		\
 os400/make-src.sh os400/make-rpg.sh os400/make-include.sh		\
-os400/os400sys.c os400/libssh2_config.h os400/macros.h			\
+os400/os400sys.c os400/ccsid.c						\
+os400/libssh2_config.h os400/macros.h os400/libssh2_ccsid.h		\
 os400/include/alloca.h os400/include/sys/socket.h os400/include/stdio.h	\
 os400/libssh2rpg/libssh2.rpgle.in					\
+os400/libssh2rpg/libssh2_ccsid.rpgle.in					\
 os400/libssh2rpg/libssh2_publickey.rpgle				\
 os400/libssh2rpg/libssh2_sftp.rpgle					\
 Makefile.os400qc3.inc

os400/README400

@@ -13,9 +13,10 @@ QADRT may use libssh2 functions as on any other platform.
   QADRT does not define ASCII wrappers for all C/system procedures: an
 additional module (os400sys.c) define some more of them, that are used by
 libssh2 and that QADRT left out.
-  Presently, there is no EBCDIC support wrappers provided. It is the
+  Since standard library entry points expect and return ASCII character strings,
-responsibility of the caller to convert procedure string values and arguments
+additional procedures are provided for string transcoding (see below). No
-between ASCII and EBCDIC.
+wrappers to standard procedures are provided: however, nested calls to
+transcoding procedures may be used.
 
 Crypto API is provided by the IBM QC3 API library. It supports RSA, but not DSA.
 
@@ -89,6 +90,62 @@ _ Do not use original source include files unless you know what you are doing.
 
 
 
+String transcoding support:
+
+  To help passing arbitrarily encoded string arguments and/or receiving string
+values from/to the libssh2 API, three non-standard additional procedures are
+provided. They use a session pointer and a "string cache" pointer.
+  Each time a string is transcoded, it is cached in the given cache. It is
+the responsibility of the caller to release the cache when its associted strings
+are no longer needed. These procedures and the string cache type are defined
+in a new libssh2_ccsid.h header file.
+  To create a string cache, use:
+
+#include <libssh2_ccsid.h>
+libssh2_string_cache *  cache = NULL;
+
+  To release all strings in a cache, call:
+
+libssh2_release_string_cache(session, &cache);
+
+  The transcoding procedures are:
+
+char * libssh2_from_ccsid(LIBSSH2_SESSION *session,
+                          libssh2_string_cache **cache,
+                          unsigned short ccsid,
+                          const char *string, ssize_t inlen,
+                          size_t *outlen);
+char * libssh2_to_ccsid(LIBSSH2_SESSION *session,
+                        libssh2_string_cache **cache,
+                        unsigned short ccsid,
+                        const char *string, ssize_t inlen,
+                        size_t *outlen);
+
+where:
+    session      is a libssh2 session used for memory allocation.
+    cache        is the address of a string cache.
+    ccsid        is the external (i.e.: non libssh2) coded character set id.
+                 65535 means no conversion and 0 means the current job's CCSID.
+    string       is the string to convert.
+    inlen        is the source string length in bytes: set to -1 if
+                 null-terminated.
+    outlen       if not NULL, is the address of a variable that will receive
+                 the transcoded string length upon return.
+
+  libssh2_from_ccsid() transcodes the string from the given CCSID to libssh2
+internal encoding (UTF-8). It is intended to be used to convert API input
+parameters.
+  libssh2_to_ccsid() transcodes the string from libssh2 internal encoding
+(UTF-8) to the given CCSID. This has been implemented to get standard API
+string results in a program's native encoding.
+
+  Both these functions return a pointer to the null-terminated converted string,
+or NULL if an error occurred. In addition, the variable pointed by outlen
+receives the effective byte length of the (cached) translated string, or -1
+in case of error.
+
+
+
 ILE/RPG support:
 
   Since 95% of the OS/400 programmers use ILE/RPG exclusively, a definition
@@ -102,7 +159,7 @@ must figure in the program header, and line
      d/include libssh2/libssh2rpg,libssh2
 
 in the global data section of the module's source code.
-If required, members ssh2_sftp and ssh2_pkey may also be included.
+If required, members ssh2_sftp, ssh2_pkey and ssh2_ccsid may also be included.
 
 For IFS source compilations, include members are located in directory
 /libssh2/include/libssh2rpg and have their original names retained.

os400/ccsid.c

@@ -0,0 +1,252 @@
+/*
+ * Copyright (C) 2015 Patrick Monnerat, D+H <patrick.monnerat@dh.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms,
+ * with or without modification, are permitted provided
+ * that the following conditions are met:
+ *
+ *   Redistributions of source code must retain the above
+ *   copyright notice, this list of conditions and the
+ *   following disclaimer.
+ *
+ *   Redistributions in binary form must reproduce the above
+ *   copyright notice, this list of conditions and the following
+ *   disclaimer in the documentation and/or other materials
+ *   provided with the distribution.
+ *
+ *   Neither the name of the copyright holder nor the names
+ *   of any other contributors may be used to endorse or
+ *   promote products derived from this software without
+ *   specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+ * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
+ * OF SUCH DAMAGE.
+ */
+
+/* Character encoding wrappers. */
+
+#include "libssh2_priv.h"
+#include "libssh2_ccsid.h"
+
+#include <qtqiconv.h>
+#include <iconv.h>
+#include <errno.h>
+#include <string.h>
+#include <stdio.h>
+
+
+
+#define CCSID_UTF8      1208
+#define CCSID_UTF16BE   13488
+#define STRING_GRANULE  256
+#define MAX_CHAR_SIZE   4
+
+#define OFFSET_OF(t, f) ((size_t) ((char *) &((t *) 0)->f - (char *) 0))
+
+
+struct _libssh2_string_cache {
+    libssh2_string_cache *  next;
+    char                    string[1];
+};
+
+
+static const QtqCode_T  utf8code = { CCSID_UTF8 };
+
+
+static ssize_t
+terminator_size(unsigned short ccsid)
+{
+    QtqCode_T outcode;
+    iconv_t cd;
+    char *inp;
+    char *outp;
+    size_t ilen;
+    size_t olen;
+    char buf[MAX_CHAR_SIZE];
+
+    /* Return the null-terminator size for the given CCSID. */
+
+    /* Fast check usual CCSIDs. */
+    switch (ccsid) {
+    case CCSID_UTF8:
+    case 0:                                 /* Job CCSID is SBCS EBCDIC. */
+        return 1;
+    case CCSID_UTF16BE:
+        return 2;
+    }
+
+    /* Convert an UTF-8 NUL to the target CCSID: use the converted size as
+       result. */
+    memset((void *) &outcode, 0, sizeof outcode);
+    outcode.CCSID = ccsid;
+    cd = QtqIconvOpen(&outcode, (QtqCode_T *) &utf8code);
+    if (cd.return_value == -1)
+        return -1;
+    inp = "";
+    ilen = 1;
+    outp = buf;
+    olen = sizeof buf;
+    iconv(cd, &inp, &ilen, &outp, &olen);
+    iconv_close(cd);
+    olen = sizeof buf - olen;
+    return olen? olen: -1;
+}
+
+static char *
+convert_ccsid(LIBSSH2_SESSION *session, libssh2_string_cache **cache,
+              unsigned short outccsid, unsigned short inccsid,
+              const char *instring, ssize_t inlen, size_t *outlen)
+{
+    char *inp;
+    char *outp;
+    size_t olen;
+    size_t ilen;
+    size_t buflen;
+    size_t curlen;
+    ssize_t termsize;
+    int i;
+    char *dst;
+    libssh2_string_cache *outstring;
+    QtqCode_T incode;
+    QtqCode_T outcode;
+    iconv_t cd;
+
+    if (!instring) {
+        if (outlen)
+            *outlen = 0;
+        return NULL;
+    }
+    if (outlen)
+        *outlen = -1;
+    if (!session || !cache)
+        return NULL;
+
+    /* Get terminator size. */
+    termsize = terminator_size(outccsid);
+    if (termsize < 0)
+        return NULL;
+ 
+    /* Prepare conversion parameters. */
+    memset((void *) &incode, 0, sizeof incode);
+    memset((void *) &outcode, 0, sizeof outcode);
+    incode.CCSID = inccsid;
+    outcode.CCSID = outccsid;
+    curlen = OFFSET_OF(libssh2_string_cache, string);
+    inp = (char *) instring;
+    ilen = inlen;
+    buflen = inlen + curlen;
+    if (inlen < 0) {
+        incode.length_option = 1;
+        buflen = STRING_GRANULE;
+        ilen = 0;
+    }
+
+    /* Allocate output string buffer and open conversion descriptor. */
+    dst = LIBSSH2_ALLOC(session, buflen + termsize);
+    if (!dst)
+        return NULL;
+    cd = QtqIconvOpen(&outcode, &incode);
+    if (cd.return_value == -1) {
+        LIBSSH2_FREE(session, (char *) dst);
+        return NULL;
+    }
+
+    /* Convert string. */
+    for (;;) {
+        outp = dst + curlen;
+        olen = buflen - curlen;
+        i = iconv(cd, &inp, &ilen, &outp, &olen);
+        if (inlen < 0 && olen == buflen - curlen) {
+            /* Special case: converted 0-length (sub)strings do not store the
+               terminator. */
+            if (termsize) {
+                memset(outp, 0, termsize);
+                olen -= termsize;
+            }
+        }
+        curlen = buflen - olen;
+        if (i >= 0 || errno != E2BIG)
+            break;
+        /* Must expand buffer. */
+        buflen += STRING_GRANULE;
+        outp = LIBSSH2_REALLOC(session, dst, buflen + termsize);
+        if (!outp)
+            break;
+        dst = outp;
+    }
+
+    iconv_close(cd);
+
+    /* Check for error. */
+    if (i < 0 || !outp) {
+        LIBSSH2_FREE(session, dst);
+        return NULL;
+    }
+
+    /* Process terminator. */
+    if (inlen < 0)
+        curlen -= termsize;
+    else if (termsize)
+        memset(dst + curlen, 0, termsize);
+
+    /* Shorten buffer if possible. */
+    if (curlen < buflen)
+        dst = LIBSSH2_REALLOC(session, dst, curlen + termsize);
+
+    /* Link to cache. */
+    outstring = (libssh2_string_cache *) dst;
+    outstring->next = *cache;
+    *cache = outstring;
+
+    /* Return length if required. */
+    if (outlen)
+        *outlen = curlen - OFFSET_OF(libssh2_string_cache, string);
+
+    return outstring->string;
+}
+
+LIBSSH2_API char *
+libssh2_from_ccsid(LIBSSH2_SESSION *session, libssh2_string_cache **cache,
+                   unsigned short ccsid, const char *string, ssize_t inlen,
+                   size_t *outlen)
+{
+    return convert_ccsid(session, cache,
+                         CCSID_UTF8, ccsid, string, inlen, outlen);
+}
+
+LIBSSH2_API char *
+libssh2_to_ccsid(LIBSSH2_SESSION *session, libssh2_string_cache **cache,
+                 unsigned short ccsid, const char *string, ssize_t inlen,
+                 size_t *outlen)
+{
+    return convert_ccsid(session, cache,
+                         ccsid, CCSID_UTF8, string, inlen, outlen);
+}
+
+LIBSSH2_API void
+libssh2_release_string_cache(LIBSSH2_SESSION *session,
+                             libssh2_string_cache **cache)
+{
+    libssh2_string_cache *p;
+
+    if (session && cache)
+        while ((p = *cache)) {
+            *cache = p->next;
+            LIBSSH2_FREE(session, (char *) p);
+        }
+}
+
+/* vim: set expandtab ts=4 sw=4: */

os400/libssh2_ccsid.h

@@ -0,0 +1,63 @@
+/*
+ * Copyright (C) 2015 Patrick Monnerat, D+H <patrick.monnerat@dh.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms,
+ * with or without modification, are permitted provided
+ * that the following conditions are met:
+ *
+ *   Redistributions of source code must retain the above
+ *   copyright notice, this list of conditions and the
+ *   following disclaimer.
+ *
+ *   Redistributions in binary form must reproduce the above
+ *   copyright notice, this list of conditions and the following
+ *   disclaimer in the documentation and/or other materials
+ *   provided with the distribution.
+ *
+ *   Neither the name of the copyright holder nor the names
+ *   of any other contributors may be used to endorse or
+ *   promote products derived from this software without
+ *   specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+ * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
+ * OF SUCH DAMAGE.
+ */
+
+/* CCSID conversion support. */
+
+#ifndef LIBSSH2_CCSID_H_
+#define LIBSSH2_CCSID_H_
+
+#include "libssh2.h"
+
+typedef struct _libssh2_string_cache    libssh2_string_cache;
+
+
+LIBSSH2_API char *
+libssh2_from_ccsid(LIBSSH2_SESSION *session, libssh2_string_cache **cache,
+                   unsigned short ccsid, const char *string, ssize_t inlen,
+                   size_t *outlen);
+LIBSSH2_API char *
+libssh2_to_ccsid(LIBSSH2_SESSION *session, libssh2_string_cache **cache,
+                 unsigned short ccsid, const char *string, ssize_t inlen,
+                 size_t *outlen);
+LIBSSH2_API void
+libssh2_release_string_cache(LIBSSH2_SESSION *session,
+                             libssh2_string_cache **cache);
+
+#endif
+
+/* vim: set expandtab ts=4 sw=4: */

os400/libssh2rpg/libssh2_ccsid.rpgle.in

@@ -0,0 +1,69 @@
+      * Copyright (c) 2015 Patrick Monnerat, D+H <patrick.monnerat@dh.com>
+      * All rights reserved.
+      *
+      * Redistribution and use in source and binary forms,
+      * with or without modification, are permitted provided
+      * that the following conditions are met:
+      *
+      *   Redistributions of source code must retain the above
+      *   copyright notice, this list of conditions and the
+      *   following disclaimer.
+      *
+      *   Redistributions in binary form must reproduce the above
+      *   copyright notice, this list of conditions and the following
+      *   disclaimer in the documentation and/or other materials
+      *   provided with the distribution.
+      *
+      *   Neither the name of the copyright holder nor the names
+      *   of any other contributors may be used to endorse or
+      *   promote products derived from this software without
+      *   specific prior written permission.
+      *
+      * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+      * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+      * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+      * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+      * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+      * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+      * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+      * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+      * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+      * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+      * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+      * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+      * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
+      * OF SUCH DAMAGE.
+
+      /if not defined(LIBSSH2_CCSID_H_)
+      /define LIBSSH2_CCSID_H_
+
+      /include "libssh2rpg/libssh2"
+
+     d libssh2_from_ccsid...
+     d                 pr              *   extproc('libssh2_from_ccsid')        char *
+     d  session                        *   value                                LIBSSH2_SESSION *
+     d  cache                          *   value                                libssh2_string_cache
+     d                                                                          *(*)
+     d  ccsid                              value like(libssh2_Cushort)
+     d  string                         *   value options(*string)               const char *
+     d  inlen                              value like(libssh2_Cssize_t)
+     d  outlen                             like(libssh2_Csize_t) options(*omit)
+
+     d libssh2_to_ccsid...
+     d                 pr              *   extproc('libssh2_to_ccsid')          char *
+     d  session                        *   value                                LIBSSH2_SESSION *
+     d  cache                          *                                        libssh2_string_cache
+     d                                                                          *(*)
+     d  ccsid                              value like(libssh2_Cushort)
+     d  string                         *   value options(*string)               const char *
+     d  inlen                              value like(libssh2_Cssize_t)
+     d  outlen                             like(libssh2_Csize_t) options(*omit)
+
+     d libssh2_release_string_cache...
+     d                 pr                  extproc(
+     d                                     'libssh2_release_string_cache')
+     d  session                        *   value                                LIBSSH2_SESSION *
+     d  cache                          *                                        libssh2_string_cache
+     d                                                                          *(*)
+
+      /endif                                                                    LIBSSH2_CCSID_H_

os400/make-include.sh

@@ -44,7 +44,7 @@ copy_hfile()
 
 #       Copy the header files.
 
-for HFILE in *.h
+for HFILE in *.h "${TOPDIR}/os400/libssh2_ccsid.h"
 do      DEST="${SRCPF}/`db2_name \"${HFILE}\"`.MBR"
 
         if action_needed "${DEST}" "${HFILE}"

os400/make-src.sh

@@ -97,7 +97,8 @@ cat ../Makefile.inc ../Makefile.os400qc3.inc                            |
 
 INCLUDES="'`pwd`'"
 
-for SRC in "${SCRIPTDIR}/os400sys.c" ${CSOURCES} ${CRYPTO_CSOURCES} macros.c
+for SRC in "${TOPDIR}/os400/os400sys.c" "${TOPDIR}/os400/ccsid.c"       \
+           ${CSOURCES} ${CRYPTO_CSOURCES} macros.c
 do      MODULE=`db2_name "${SRC}"`
         make_module "${MODULE}" "${SRC}"
 done
@@ -142,7 +143,8 @@ fi
 
 #       Gather the list of symbols to export.
 
-EXPORTS=`cat "${TOPDIR}"/include/*.h "${TOPDIR}"/os400/macros.h         |
+EXPORTS=`cat "${TOPDIR}"/include/*.h "${TOPDIR}/os400/macros.h"         \
+             "${TOPDIR}/os400/libssh2_ccsid.h"                          |
          extproto                                                       |
          sed -e 's/(.*//;s/[^A-Za-z0-9_]/ /g;s/ *$//;s/^.* //'`