PyGreSQL inserttable patch

=====================

I suggested an improvement of the inserttable in the PyGreSQL interface
already in January, but seemingly it was never implemented. I was told this
is the right place to get patches in for PyGreSQL, so I'm reposting my patch
here.

I consider the inserttable methode essential in populating the database
because of its benefits in performance compared to insert, so I think this
patch is quite essential. The attachment is an improved version of the
corresponding pg_inserttable function in pgmodule.c, which fixes the
following problems:

* The function raised exceptions because PyList_GetItem was used beyond the
size of the list. This was checked by comparing the result with NULL, but
the exception was not cleaned up, which could result in mysterious errors in
the following Python code. Instead of clearing the exception using
PyErr_Clear or something like that, I avoided throwing the exception at all
by at first requesting the size of the list. Using this opportunity, I also
checked the uniformity of the size of the rows passed in the lists/tuples.
The function also accepts (and silently ignores) empty lists and sublists.
* Python "None" values are now accepted and properly converted to PostgreSQL
NULL values
* The function now generates an error message in case of a line buffer
overflow
* It copes with tabulators, newlines and backslashes in strings now
* Rewrote the buffer filling code which should now run faster by avoiding
unnecessary string copy operations forth and back

Christoph Zwerschke

src/interfaces/python/pgmodule.c

@@ -2327,15 +2327,18 @@ pg_inserttable(pgobject * self, PyObject * args)
 	PGresult	*result;
 	char		*table,
 				*buffer,
+				*bufpt,
 				*temp;
-	char		temp_buffer[256];
+	size_t		bufsiz;
 	PyObject	*list,
 				*sublist,
 				*item;
 	PyObject	*(*getitem) (PyObject *, int);
 	PyObject	*(*getsubitem) (PyObject *, int);
 	int			i,
-				j;
+				j,
+				m,
+				n;
 
 	if (!self->cnx)
 	{
@@ -2354,9 +2357,15 @@ pg_inserttable(pgobject * self, PyObject * args)
 
 	/* checks list type */
 	if (PyTuple_Check(list))
+	{
+		m = PyTuple_Size(list);
 		getitem = PyTuple_GetItem;
+	}
 	else if (PyList_Check(list))
+	{
+		m = PyList_Size(list);
 		getitem = PyList_GetItem;
+	}
 	else
 	{
 		PyErr_SetString(PyExc_TypeError,
@@ -2364,21 +2373,41 @@ pg_inserttable(pgobject * self, PyObject * args)
 		return NULL;
 	}
 
-	/* checks sublists type */
-	for (i = 0; (sublist = getitem(list, i)) != NULL; i++)
+	if (m)
+	{
+		/* checks sublists type and size */
+		for (i = 0; i < m; i++)
 		{
-		if (!PyTuple_Check(sublist) && !PyList_Check(sublist))
+			sublist = getitem(list, i);
+			if (PyTuple_Check(sublist))
+				j = PyTuple_Size(sublist);
+			else if (PyList_Check(sublist))
+				j = PyList_Size(sublist);
+			else
 			{
 				PyErr_SetString(PyExc_TypeError,
 					"second arg must contain some kind of arrays.");
 				return NULL;
 			}
+			if (i)
+			{
+				if (j != n)
+				{
+					PyErr_SetString(PyExc_TypeError,
+						"arrays contained in second arg must have same size.");
+					return NULL;
+				}
 			}
-
+			else
+				n=j;
+		}
+		if (n)
+		{	
 			/* allocate buffer */
 			if (!(buffer = malloc(MAX_BUFFER_SIZE)))
 			{		
-		PyErr_SetString(PyExc_MemoryError, "can't allocate insert buffer.");
+				PyErr_SetString(PyExc_MemoryError,
+					"can't allocate insert buffer.");
 				return NULL;
 			}
 
@@ -2399,44 +2428,88 @@ pg_inserttable(pgobject * self, PyObject * args)
 			PQclear(result);
 
 			/* feeds table */
-	for (i = 0; (sublist = getitem(list, i)) != NULL; i++)
+			for (i = 0; i < m; i++)
 			{
+				sublist = getitem(list, i);
 				if (PyTuple_Check(sublist))
 					getsubitem = PyTuple_GetItem;
 				else
 					getsubitem = PyList_GetItem;
 	
 				/* builds insert line */
-		buffer[0] = 0;
+				bufpt=buffer;
+				bufsiz = MAX_BUFFER_SIZE - 1;
 
-		for (j = 0; (item = getsubitem(sublist, j)) != NULL; j++)
+				for (j = 0; j < n; j++)
 				{
+					item = getsubitem(sublist, j);
 					/* converts item to string */
-			if (PyString_Check(item))
+					if (item == Py_None)
+					{
+						if (bufsiz > 2)
+						{
+							*bufpt++ = '\\'; *bufpt++ = 'N';
+							bufsiz -= 2;
+						}
+						else
+							bufsiz = 0;
+					}
+					else if (PyString_Check(item))
+					{
 						temp = PyString_AS_STRING(item);
+						while (*temp && bufsiz)
+						{
+						     if (*temp == '\\'
+						      || *temp == '\t'
+						      || *temp == '\n')
+						     {
+						        *bufpt++='\\'; --bufsiz;
+						        if (!bufsiz) break;
+						     }
+						     *bufpt++=*temp++; --bufsiz;
+						}
+					}
 					else if (PyInt_Check(item))
 					{
 						long	k;
+						int 	h;
 
 						k = PyInt_AsLong(item);
-				sprintf(temp_buffer, "%ld", k);
-				temp = temp_buffer;
+						h = snprintf(bufpt, bufsiz, "%ld", k);
+						if (h > 0)
+						{
+							bufpt += h; bufsiz -= h;
+						}
+						else
+							bufsiz = 0;
 					}
 					else if (PyLong_Check(item))
 					{
 						long	k;
+						int 	h;
 
 						k = PyLong_AsLong(item);
-				sprintf(temp_buffer, "%ld", k);
-				temp = temp_buffer;
+						h = snprintf(bufpt, bufsiz, "%ld", k);
+						if (h > 0)
+						{
+							bufpt += h; bufsiz -= h;
+						}
+						else
+							bufsiz = 0;
 					}
 					else if (PyFloat_Check(item))
 					{
 						double	k;
+						int		h;
 
 						k = PyFloat_AS_DOUBLE(item);
-				sprintf(temp_buffer, "%g", k);
-				temp = temp_buffer;
+						h = snprintf(bufpt, bufsiz, "%g", k);
+						if (h > 0)
+						{
+							bufpt += h; bufsiz -= h;
+						}
+						else
+							bufsiz = 0;
 					}
 					else
 					{
@@ -2447,14 +2520,22 @@ pg_inserttable(pgobject * self, PyObject * args)
 						return NULL;
 					}
 
-			/* concats buffer */
-			if (strlen(buffer))
-				strncat(buffer, "\t", MAX_BUFFER_SIZE - strlen(buffer));
+					if (j < n-1)
+					{
+						*bufpt++ = '\t'; --bufsiz;
+					}
+			
+					if (bufsiz <= 0)
+					{
+						free(buffer);
+						PyErr_SetString(PyExc_MemoryError,
+							"insert buffer overflow.");
+						return NULL;
+					}
 
-			strncat(buffer, temp, MAX_BUFFER_SIZE - strlen(buffer));
 				}
 
-		strncat(buffer, "\n", MAX_BUFFER_SIZE - strlen(buffer));
+				*bufpt++ = '\n'; *bufpt = '\0';
 
 				/* sends data */
 				PQputline(self->cnx, buffer);
@@ -2464,6 +2545,8 @@ pg_inserttable(pgobject * self, PyObject * args)
 			PQputline(self->cnx, "\\.\n");
 			PQendcopy(self->cnx);
 			free(buffer);
+		}
+	}
 
 	/* no error : returns nothing */
 	Py_INCREF(Py_None);