Eliminate query length limitation imposed by pg_client_to_server

and pg_server_to_client.  Eliminate copy.c's restriction on the length
of a single attribute.

src/backend/commands/copy.c

@@ -6,7 +6,7 @@
  *
  *
  * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/backend/commands/copy.c,v 1.86 1999/07/22 02:40:06 tgl Exp $
+ *	  $Header: /cvsroot/pgsql/src/backend/commands/copy.c,v 1.87 1999/09/11 22:28:11 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -26,6 +26,7 @@
 #include "commands/copy.h"
 #include "commands/trigger.h"
 #include "executor/executor.h"
+#include "lib/stringinfo.h"
 #include "libpq/libpq.h"
 #include "miscadmin.h"
 #include "utils/acl.h"
@@ -51,14 +52,10 @@ static void GetIndexRelations(Oid main_relation_oid,
 				  int *n_indices,
 				  Relation **index_rels);
 
-#ifdef COPY_PATCH
 static void CopyReadNewline(FILE *fp, int *newline);
 static char *CopyReadAttribute(FILE *fp, bool *isnull, char *delim, int *newline);
-#else
-static char *CopyReadAttribute(FILE *fp, bool *isnull, char *delim);
-#endif
 
-static void CopyAttributeOut(FILE *fp, char *string, char *delim, int is_array);
+static void CopyAttributeOut(FILE *fp, char *string, char *delim);
 static int	CountTuples(Relation relation);
 
 /*
@@ -431,7 +428,7 @@ CopyTo(Relation rel, bool binary, bool oids, FILE *fp, char *delim)
 				{
 					string = (char *) (*fmgr_faddr(&out_functions[i]))
 						(value, elements[i], typmod[i]);
-					CopyAttributeOut(fp, string, delim, attr[i]->attnelems);
+					CopyAttributeOut(fp, string, delim);
 					pfree(string);
 				}
 				else
@@ -691,18 +688,12 @@ CopyFrom(Relation rel, bool binary, bool oids, FILE *fp, char *delim)
 	{
 		if (!binary)
 		{
-#ifdef COPY_PATCH
 			int			newline = 0;
 
-#endif
 			lineno++;
 			if (oids)
 			{
-#ifdef COPY_PATCH
 				string = CopyReadAttribute(fp, &isnull, delim, &newline);
-#else
-				string = CopyReadAttribute(fp, &isnull, delim);
-#endif
 				if (string == NULL)
 					done = 1;
 				else
@@ -710,19 +701,18 @@ CopyFrom(Relation rel, bool binary, bool oids, FILE *fp, char *delim)
 					loaded_oid = oidin(string);
 					if (loaded_oid < BootstrapObjectIdData)
 						elog(ERROR, "COPY TEXT: Invalid Oid. line: %d", lineno);
+					pfree(string);
 				}
 			}
 			for (i = 0; i < attr_count && !done; i++)
 			{
-#ifdef COPY_PATCH
 				string = CopyReadAttribute(fp, &isnull, delim, &newline);
-#else
-				string = CopyReadAttribute(fp, &isnull, delim);
-#endif
 				if (isnull)
 				{
 					values[i] = PointerGetDatum(NULL);
 					nulls[i] = 'n';
+					if (string)
+						pfree(string);
 				}
 				else if (string == NULL)
 					done = 1;
@@ -739,12 +729,11 @@ CopyFrom(Relation rel, bool binary, bool oids, FILE *fp, char *delim)
 					if (!PointerIsValid(values[i]) &&
 						!(rel->rd_att->attrs[i]->attbyval))
 						elog(ERROR, "copy from line %d: Bad file format", lineno);
+					pfree(string);
 				}
 			}
-#ifdef COPY_PATCH
 			if (!done)
 				CopyReadNewline(fp, &newline);
-#endif
 		}
 		else
 		{						/* binary */
@@ -812,11 +801,6 @@ CopyFrom(Relation rel, bool binary, bool oids, FILE *fp, char *delim)
 		if (done)
 			continue;
 
-		/*
-		 * Does it have any sence ? - vadim 12/14/96
-		 *
-		 * tupDesc = CreateTupleDesc(attr_count, attr);
-		 */
 		tuple = heap_formtuple(tupDesc, values, nulls);
 		if (oids)
 			tuple->t_data->t_oid = loaded_oid;
@@ -1086,30 +1070,18 @@ GetIndexRelations(Oid main_relation_oid,
 	}
 }
 
-#define EXT_ATTLEN	(5 * BLCKSZ)
 
 /*
-   returns 1 is c is in s
+   returns 1 if c is in s
 */
 static bool
 inString(char c, char *s)
 {
-	int			i;
+	if (s && c)
-
+		return strchr(s, c) != NULL;
-	if (s)
-	{
-		i = 0;
-		while (s[i] != '\0')
-		{
-			if (s[i] == c)
-				return 1;
-			i++;
-		}
-	}
 	return 0;
 }
 
-#ifdef COPY_PATCH
 /*
  * Reads input from fp until an end of line is seen.
  */
@@ -1125,64 +1097,57 @@ CopyReadNewline(FILE *fp, int *newline)
 	*newline = 0;
 }
 
-#endif
-
 /*
- * Reads input from fp until eof is seen.  If we are reading from standard
+ * Read the value of a single attribute.
- * input, AND we see a dot on a line by itself (a dot followed immediately
+ *
- * by a newline), we exit as if we saw eof.  This is so that copy pipelines
+ * Result is either a palloc'd string, or NULL (if EOF or a null attribute).
- * can be used as standard input.
+ * *isnull is set true if a null attribute, else false.
+ *
+ * delim is the string of acceptable delimiter characters(s).
+ * *newline remembers whether we've seen a newline ending this tuple.
  */
 
 static char *
-#ifdef COPY_PATCH
 CopyReadAttribute(FILE *fp, bool *isnull, char *delim, int *newline)
-#else
-CopyReadAttribute(FILE *fp, bool *isnull, char *delim)
-#endif
 {
-	static char attribute[EXT_ATTLEN];
+	StringInfoData	attribute_buf;
 	char		c;
-	int			done = 0;
-	int			i = 0;
-
 #ifdef MULTIBYTE
 	int			mblen;
 	int			encoding;
 	unsigned char s[2];
+	char	   *cvt;
 	int			j;
 
-#endif
-
-#ifdef MULTIBYTE
 	encoding = pg_get_client_encoding();
 	s[1] = 0;
 #endif
 
-#ifdef COPY_PATCH
 	/* if last delimiter was a newline return a NULL attribute */
 	if (*newline)
 	{
 		*isnull = (bool) true;
 		return NULL;
 	}
-#endif
 
 	*isnull = (bool) false;		/* set default */
-	if (CopyGetEof(fp))
-		return NULL;
 
-	while (!done)
+	initStringInfo(&attribute_buf);
-	{
-		c = CopyGetChar(fp);
 
 	if (CopyGetEof(fp))
-			return NULL;
+		goto endOfFile;
-		else if (c == '\\')
+
+	for (;;)
 	{
 		c = CopyGetChar(fp);
 		if (CopyGetEof(fp))
-				return NULL;
+			goto endOfFile;
+
+		if (c == '\\')
+		{
+			c = CopyGetChar(fp);
+			if (CopyGetEof(fp))
+				goto endOfFile;
 			switch (c)
 			{
 				case '0':
@@ -1212,14 +1177,14 @@ CopyReadAttribute(FILE *fp, bool *isnull, char *delim)
 							else
 							{
 								if (CopyGetEof(fp))
-									return NULL;
+									goto endOfFile;
 								CopyDonePeek(fp, c, 0); /* Return to stream! */
 							}
 						}
 						else
 						{
 							if (CopyGetEof(fp))
-								return NULL;
+								goto endOfFile;
 							CopyDonePeek(fp, c, 0);		/* Return to stream! */
 						}
 						c = val & 0377;
@@ -1244,66 +1209,70 @@ CopyReadAttribute(FILE *fp, bool *isnull, char *delim)
 					c = '\v';
 					break;
 				case 'N':
-					attribute[0] = '\0';		/* just to be safe */
 					*isnull = (bool) true;
 					break;
 				case '.':
 					c = CopyGetChar(fp);
 					if (c != '\n')
 						elog(ERROR, "CopyReadAttribute - end of record marker corrupted. line: %d", lineno);
-					return NULL;
+					goto endOfFile;
 					break;
 			}
 		}
-		else if (inString(c, delim) || c == '\n')
+		else if (c == '\n' || inString(c, delim))
 		{
-#ifdef COPY_PATCH
 			if (c == '\n')
 				*newline = 1;
-#endif
+			break;
-			done = 1;
 		}
-		if (!done)
+		appendStringInfoChar(&attribute_buf, c);
-			attribute[i++] = c;
 #ifdef MULTIBYTE
+		/* get additional bytes of the char, if any */
 		s[0] = c;
 		mblen = pg_encoding_mblen(encoding, s);
-		mblen--;
+		for (j = 1; j < mblen; j++)
-		for (j = 0; j < mblen; j++)
 		{
 			c = CopyGetChar(fp);
 			if (CopyGetEof(fp))
-				return NULL;
+				goto endOfFile;
-			attribute[i++] = c;
+			appendStringInfoChar(&attribute_buf, c);
 		}
 #endif
-		if (i == EXT_ATTLEN - 1)
-			elog(ERROR, "CopyReadAttribute - attribute length too long. line: %d", lineno);
 	}
-	attribute[i] = '\0';
+
 #ifdef MULTIBYTE
-	return (pg_client_to_server((unsigned char *) attribute, strlen(attribute)));
+	cvt = (char *) pg_client_to_server((unsigned char *) attribute_buf.data,
-#else
+									   attribute_buf.len);
-	return &attribute[0];
+	if (cvt != attribute_buf.data)
+	{
+		pfree(attribute_buf.data);
+		return cvt;
+	}
 #endif
+	return attribute_buf.data;
+
+endOfFile:
+	pfree(attribute_buf.data);
+	return NULL;
 }
 
 static void
-CopyAttributeOut(FILE *fp, char *server_string, char *delim, int is_array)
+CopyAttributeOut(FILE *fp, char *server_string, char *delim)
 {
 	char	   *string;
 	char		c;
-
 #ifdef MULTIBYTE
-	int			mblen;
+	char	   *string_start;
 	int			encoding;
+	int			mblen;
 	int			i;
-
 #endif
 
 #ifdef MULTIBYTE
-	string = pg_server_to_client(server_string, strlen(server_string));
 	encoding = pg_get_client_encoding();
+	string = (char *) pg_server_to_client((unsigned char *) server_string,
+										  strlen(server_string));
+	string_start = string;
 #else
 	string = server_string;
 #endif
@@ -1315,33 +1284,20 @@ CopyAttributeOut(FILE *fp, char *server_string, char *delim, int is_array)
 	for (; (c = *string) != '\0'; string++)
 #endif
 	{
-		if (c == delim[0] || c == '\n' ||
+		if (c == delim[0] || c == '\n' || c == '\\')
-			(c == '\\' && !is_array))
 			CopySendChar('\\', fp);
-		else if (c == '\\' && is_array)
-		{
-			if (*(string + 1) == '\\')
-			{
-				/* translate \\ to \\\\ */
-				CopySendChar('\\', fp);
-				CopySendChar('\\', fp);
-				CopySendChar('\\', fp);
-				string++;
-			}
-			else if (*(string + 1) == '"')
-			{
-				/* translate \" to \\\" */
-				CopySendChar('\\', fp);
-				CopySendChar('\\', fp);
-			}
-		}
 #ifdef MULTIBYTE
 		for (i = 0; i < mblen; i++)
 			CopySendChar(*(string + i), fp);
 #else
-		CopySendChar(*string, fp);
+		CopySendChar(c, fp);
 #endif
 	}
+
+#ifdef MULTIBYTE
+	if (string_start != server_string)
+		pfree(string_start);	/* pfree pg_server_to_client result */
+#endif
 }
 
 /*

src/backend/libpq/pqformat.c

@@ -15,7 +15,7 @@
  *
  * Copyright (c) 1994, Regents of the University of California
  *
- *	$Id: pqformat.c,v 1.8 1999/08/31 04:26:37 tgl Exp $
+ *	$Id: pqformat.c,v 1.9 1999/09/11 22:28:05 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -125,13 +125,16 @@ void
 pq_sendcountedtext(StringInfo buf, const char *str, int slen)
 {
 #ifdef MULTIBYTE
-	const char *p;
+	char	   *p;
 
-	p = (const char *) pg_server_to_client((unsigned char *) str, slen);
+	p = (char *) pg_server_to_client((unsigned char *) str, slen);
 	if (p != str)				/* actual conversion has been done? */
 	{
-		str = p;
+		slen = strlen(p);
-		slen = strlen(str);
+		pq_sendint(buf, slen + 4, 4);
+		appendBinaryStringInfo(buf, p, slen);
+		pfree(p);
+		return;
 	}
 #endif
 	pq_sendint(buf, slen + 4, 4);
@@ -149,15 +152,16 @@ void
 pq_sendstring(StringInfo buf, const char *str)
 {
 	int			slen = strlen(str);
-
 #ifdef MULTIBYTE
-	const char *p;
+	char	   *p;
 
-	p = (const char *) pg_server_to_client((unsigned char *) str, slen);
+	p = (char *) pg_server_to_client((unsigned char *) str, slen);
 	if (p != str)				/* actual conversion has been done? */
 	{
-		str = p;
+		slen = strlen(p);
-		slen = strlen(str);
+		appendBinaryStringInfo(buf, p, slen + 1);
+		pfree(p);
+		return;
 	}
 #endif
 	appendBinaryStringInfo(buf, str, slen + 1);
@@ -229,15 +233,15 @@ int
 pq_puttextmessage(char msgtype, const char *str)
 {
 	int			slen = strlen(str);
-
 #ifdef MULTIBYTE
-	const char *p;
+	char	   *p;
 
-	p = (const char *) pg_server_to_client((unsigned char *) str, slen);
+	p = (char *) pg_server_to_client((unsigned char *) str, slen);
 	if (p != str)				/* actual conversion has been done? */
 	{
-		str = p;
+		int result = pq_putmessage(msgtype, p, strlen(p) + 1);
-		slen = strlen(str);
+		pfree(p);
+		return result;
 	}
 #endif
 	return pq_putmessage(msgtype, str, slen + 1);
@@ -299,12 +303,12 @@ pq_getint(int *result, int b)
 int
 pq_getstr(StringInfo s)
 {
-	int			c;
+	int			result;
 #ifdef MULTIBYTE
 	char	   *p;
 #endif
 
-	c = pq_getstring(s);
+	result = pq_getstring(s);
 
 #ifdef MULTIBYTE
 	p = (char *) pg_client_to_server((unsigned char *) s->data, s->len);
@@ -314,8 +318,9 @@ pq_getstr(StringInfo s)
 		s->len = 0;
 		s->data[0] = '\0';
 		appendBinaryStringInfo(s, p, strlen(p));
+		pfree(p);
 	}
 #endif
 
-	return c;
+	return result;
 }

src/backend/utils/mb/mbutils.c

@@ -3,7 +3,7 @@
  * client encoding and server internal encoding.
  * (currently mule internal code (mic) is used)
  * Tatsuo Ishii
- * $Id: mbutils.c,v 1.8 1999/07/17 20:18:10 momjian Exp $ */
+ * $Id: mbutils.c,v 1.9 1999/09/11 22:28:00 tgl Exp $ */
 
 
 #include "postgres.h"
@@ -93,59 +93,81 @@ pg_get_client_encoding()
 }
 
 /*
- * convert client encoding to server encoding. if server_encoding ==
+ * convert client encoding to server encoding.
- * client_encoding or no conversion function exists,
+ *
- * returns s. So be careful.
+ * CASE 1: if no conversion is required, then the given pointer s is returned.
+ *
+ * CASE 2: if conversion is required, a palloc'd string is returned.
+ *
+ * Callers must check whether return value differs from passed value
+ * to determine whether to pfree the result or not!
+ *
+ * Note: we assume that conversion cannot cause more than a 4-to-1 growth
+ * in the length of the string --- is this enough?
  */
 unsigned char *
 pg_client_to_server(unsigned char *s, int len)
 {
-	static unsigned char b1[MAX_PARSE_BUFFER * 4];		/* is this enough? */
+	unsigned char *result = s;
-	static unsigned char b2[MAX_PARSE_BUFFER * 4];		/* is this enough? */
+	unsigned char *buf;
-	unsigned char *p = s;
 
 	if (client_encoding == GetDatabaseEncoding())
-		return (p);
+		return result;
 	if (client_to_mic)
 	{
-		(*client_to_mic) (s, b1, len);
+		buf = (unsigned char *) palloc(len * 4 + 1);
-		len = strlen(b1);
+		(*client_to_mic) (result, buf, len);
-		p = b1;
+		result = buf;
+		len = strlen(result);
 	}
 	if (server_from_mic)
 	{
-		(*server_from_mic) (p, b2, len);
+		buf = (unsigned char *) palloc(len * 4 + 1);
-		p = b2;
+		(*server_from_mic) (result, buf, len);
+		if (result != s)
+			pfree(result);		/* release first buffer */
+		result = buf;
 	}
-	return (p);
+	return result;
 }
 
 /*
- * convert server encoding to client encoding. if server_encoding ==
+ * convert server encoding to client encoding.
- * client_encoding or no conversion function exists,
+ *
- * returns s. So be careful.
+ * CASE 1: if no conversion is required, then the given pointer s is returned.
+ *
+ * CASE 2: if conversion is required, a palloc'd string is returned.
+ *
+ * Callers must check whether return value differs from passed value
+ * to determine whether to pfree the result or not!
+ *
+ * Note: we assume that conversion cannot cause more than a 4-to-1 growth
+ * in the length of the string --- is this enough?
  */
 unsigned char *
 pg_server_to_client(unsigned char *s, int len)
 {
-	static unsigned char b1[MAX_PARSE_BUFFER * 4];		/* is this enough? */
+	unsigned char *result = s;
-	static unsigned char b2[MAX_PARSE_BUFFER * 4];		/* is this enough? */
+	unsigned char *buf;
-	unsigned char *p = s;
 
 	if (client_encoding == GetDatabaseEncoding())
-		return (p);
+		return result;
 	if (server_to_mic)
 	{
-		(*server_to_mic) (s, b1, len);
+		buf = (unsigned char *) palloc(len * 4 + 1);
-		len = strlen(b1);
+		(*server_to_mic) (result, buf, len);
-		p = b1;
+		result = buf;
+		len = strlen(result);
 	}
 	if (client_from_mic)
 	{
-		(*client_from_mic) (p, b2, len);
+		buf = (unsigned char *) palloc(len * 4 + 1);
-		p = b2;
+		(*client_from_mic) (result, buf, len);
+		if (result != s)
+			pfree(result);		/* release first buffer */
+		result = buf;
 	}
-	return (p);
+	return result;
 }
 
 /* convert a multi-byte string to a wchar */

src/include/config.h.in

@@ -123,14 +123,6 @@
  */
 /* #define TCL_ARRAYS */
 
-/*
- * The following flag allows copying tables from files with number of columns
- * different than the number of attributes setting missing attributes to NULL
- * and ignoring extra columns.  This also avoids the shift of the attributes
- * of the rest of the file if one line has a wrong column count.
- */
-#define COPY_PATCH
-
 /*
  * User locks are handled totally on the application side as long term
  * cooperative locks which extend beyond the normal transaction boundaries.