add streaming on memory regression tests, found bad bugs in the reader

* Makefile.am: add streaming on memory regression tests, found
  bad bugs in the reader interface
* xmlreader.c: fixing bugs w.r.t. very large names, and special
  condition in end of file.
* xmlIO.c tree.c include/libxml/tree.h include/libxml/xmlIO.h:
  adding immutable buffers, and parser input based on those,
  but this should not be used (yet) for general parsing
* parser.c: added a comment about using immutable buffers for
  general parsing.
* result/bigname.xml.rdr result/bigname2.xml.rdr: fixing the
  output of the regression tests
* xmllint.c: using the immutable buffers when streaming on
  mmaped file (--stream --memory)
Daniel

ChangeLog

@@ -1,3 +1,19 @@
+Thu Sep 18 15:29:46 CEST 2003 Daniel Veillard <daniel@veillard.com>
+
+	* Makefile.am: add streaming on memory regression tests, found
+	  bad bugs in the reader interface
+	* xmlreader.c: fixing bugs w.r.t. very large names, and special
+	  condition in end of file.
+	* xmlIO.c tree.c include/libxml/tree.h include/libxml/xmlIO.h:
+	  adding immutable buffers, and parser input based on those,
+	  but this should not be used (yet) for general parsing
+	* parser.c: added a comment about using immutable buffers for
+	  general parsing.
+	* result/bigname.xml.rdr result/bigname2.xml.rdr: fixing the
+	  output of the regression tests
+	* xmllint.c: using the immutable buffers when streaming on
+	  mmaped file (--stream --memory)
+
 Thu Sep 18 12:04:50 CEST 2003 Daniel Veillard <daniel@veillard.com>
 
 	* dict.c: the last patch broke unicity of returned strings, removed

Makefile.am

@@ -588,6 +588,23 @@ Readertests : xmllint$(EXEEXT)
 	      diff $(srcdir)/result/$$name.rdr result.$$name ; \
 	      rm result.$$name ; \
 	  fi ; fi ; done)
+	@echo "##"
+	@echo "## Reader on memory regression tests"
+	@echo "##"
+	-@(for i in $(srcdir)/test/* ; do \
+	  name=`basename $$i`; \
+	  if [ ! -d $$i ] ; then \
+	  if [ ! -f $(srcdir)/result/$$name.rdr ] ; then \
+	      echo New test file $$name ; \
+	      $(CHECKER) $(top_builddir)/xmllint --memory --nonet --debug --stream $$i > $(srcdir)/result/$$name.rdr 2>/dev/null ; \
+	      grep "MORY ALLO" .memdump  | grep -v "MEMORY ALLOCATED : 0";\
+	  else \
+	      echo Testing $$name ; \
+	      $(CHECKER) $(top_builddir)/xmllint --memory --nonet --debug --stream $$i > result.$$name 2>/dev/null ; \
+	      grep "MORY ALLO" .memdump  | grep -v "MEMORY ALLOCATED : 0";\
+	      diff $(srcdir)/result/$$name.rdr result.$$name ; \
+	      rm result.$$name ; \
+	  fi ; fi ; done)
 
 
 SAXtests : testSAX$(EXEEXT)

include/libxml/tree.h

@@ -416,7 +416,8 @@ struct _xmlRef {
 
 typedef enum {
     XML_BUFFER_ALLOC_DOUBLEIT,
-    XML_BUFFER_ALLOC_EXACT
+    XML_BUFFER_ALLOC_EXACT,
+    XML_BUFFER_ALLOC_IMMUTABLE
 } xmlBufferAllocationScheme;
 
 /**
@@ -581,6 +582,9 @@ XMLPUBFUN xmlBufferPtr XMLCALL
 		xmlBufferCreate		(void);
 XMLPUBFUN xmlBufferPtr XMLCALL	
 		xmlBufferCreateSize	(size_t size);
+XMLPUBFUN xmlBufferPtr XMLCALL	
+		xmlBufferCreateStatic	(void *mem,
+					 size_t size);
 XMLPUBFUN int XMLCALL		
 		xmlBufferResize		(xmlBufferPtr buf,
 					 unsigned int size);

include/libxml/xmlIO.h

@@ -170,6 +170,9 @@ XMLPUBFUN xmlParserInputBufferPtr XMLCALL
 XMLPUBFUN xmlParserInputBufferPtr XMLCALL
 	xmlParserInputBufferCreateMem		(const char *mem, int size,
 	                                         xmlCharEncoding enc);
+XMLPUBFUN xmlParserInputBufferPtr XMLCALL
+	xmlParserInputBufferCreateStatic	(const char *mem, int size,
+	                                         xmlCharEncoding enc);
 XMLPUBFUN xmlParserInputBufferPtr XMLCALL
 	xmlParserInputBufferCreateIO		(xmlInputReadCallback   ioread,
 						 xmlInputCloseCallback  ioclose,

parser.c

@@ -11568,6 +11568,7 @@ xmlCreateMemoryParserCtxt(const char *buffer, int size) {
     if (ctxt == NULL)
 	return(NULL);
 
+    /* TODO: xmlParserInputBufferCreateStatic, requires some serious changes */
     buf = xmlParserInputBufferCreateMem(buffer, size, XML_CHAR_ENCODING_NONE);
     if (buf == NULL) {
 	xmlFreeParserCtxt(ctxt);

tree.c

@@ -6332,6 +6332,37 @@ xmlBufferCreateSize(size_t size) {
     return(ret);
 }
 
+/**
+ * xmlBufferCreateStatic:
+ * @mem: the memory area
+ * @size:  the size in byte
+ *
+ * routine to create an XML buffer from an immutable memory area,
+ * The are won't be modified nor copied, and is expected to be
+ * present until the end of the buffer lifetime.
+ *
+ * returns the new structure.
+ */
+xmlBufferPtr
+xmlBufferCreateStatic(void *mem, size_t size) {
+    xmlBufferPtr ret;
+
+    if ((mem == NULL) || (size == 0))
+        return(NULL);
+
+    ret = (xmlBufferPtr) xmlMalloc(sizeof(xmlBuffer));
+    if (ret == NULL) {
+	xmlGenericError(xmlGenericErrorContext,
+		"xmlBufferCreate : out of memory!\n");
+        return(NULL);
+    }
+    ret->use = size;
+    ret->size = size;
+    ret->alloc = XML_BUFFER_ALLOC_IMMUTABLE;
+    ret->content = (xmlChar *) mem;
+    return(ret);
+}
+
 /**
  * xmlBufferSetAllocationScheme:
  * @buf:  the buffer to tune
@@ -6349,6 +6380,7 @@ xmlBufferSetAllocationScheme(xmlBufferPtr buf,
 #endif
         return;
     }
+    if (buf->alloc == XML_BUFFER_ALLOC_IMMUTABLE) return;
 
     buf->alloc = scheme;
 }
@@ -6369,7 +6401,9 @@ xmlBufferFree(xmlBufferPtr buf) {
 #endif
 	return;
     }
-    if (buf->content != NULL) {
+
+    if ((buf->content != NULL) &&
+        (buf->alloc != XML_BUFFER_ALLOC_IMMUTABLE)) {
         xmlFree(buf->content);
     }
     xmlFree(buf);
@@ -6385,7 +6419,11 @@ void
 xmlBufferEmpty(xmlBufferPtr buf) {
     if (buf->content == NULL) return;
     buf->use = 0;
-    memset(buf->content, 0, buf->size);
+    if (buf->alloc == XML_BUFFER_ALLOC_IMMUTABLE) {
+        buf->content="";
+    } else {
+	memset(buf->content, 0, buf->size);
+    }
 }
 
 /**
@@ -6403,9 +6441,12 @@ xmlBufferShrink(xmlBufferPtr buf, unsigned int len) {
     if (len > buf->use) return(-1);
 
     buf->use -= len;
-    memmove(buf->content, &buf->content[len], buf->use * sizeof(xmlChar));
+    if (buf->alloc == XML_BUFFER_ALLOC_IMMUTABLE) {
-
+        buf->content += len;
-    buf->content[buf->use] = 0;
+    } else {
+	memmove(buf->content, &buf->content[len], buf->use * sizeof(xmlChar));
+	buf->content[buf->use] = 0;
+    }
     return(len);
 }
 
@@ -6423,6 +6464,7 @@ xmlBufferGrow(xmlBufferPtr buf, unsigned int len) {
     int size;
     xmlChar *newbuf;
 
+    if (buf->alloc == XML_BUFFER_ALLOC_IMMUTABLE) return(0);
     if (len + buf->use < buf->size) return(0);
 
     size = buf->use + len + 100;
@@ -6517,6 +6559,8 @@ xmlBufferResize(xmlBufferPtr buf, unsigned int size)
     unsigned int newSize;
     xmlChar* rebuf = NULL;
 
+    if (buf->alloc == XML_BUFFER_ALLOC_IMMUTABLE) return(0);
+
     /*take care of empty case*/
     newSize = (buf->size ? buf->size*2 : size);
 
@@ -6586,6 +6630,7 @@ xmlBufferAdd(xmlBufferPtr buf, const xmlChar *str, int len) {
 #endif
 	return;
     }
+    if (buf->alloc == XML_BUFFER_ALLOC_IMMUTABLE) return;
     if (len < -1) {
 #ifdef DEBUG_BUFFER
         xmlGenericError(xmlGenericErrorContext,
@@ -6627,6 +6672,7 @@ void
 xmlBufferAddHead(xmlBufferPtr buf, const xmlChar *str, int len) {
     unsigned int needSize;
 
+    if (buf->alloc == XML_BUFFER_ALLOC_IMMUTABLE) return;
     if (str == NULL) {
 #ifdef DEBUG_BUFFER
         xmlGenericError(xmlGenericErrorContext,
@@ -6672,6 +6718,7 @@ xmlBufferAddHead(xmlBufferPtr buf, const xmlChar *str, int len) {
  */
 void
 xmlBufferCat(xmlBufferPtr buf, const xmlChar *str) {
+    if (buf->alloc == XML_BUFFER_ALLOC_IMMUTABLE) return;
     if (str != NULL)
 	xmlBufferAdd(buf, str, -1);
 }
@@ -6687,6 +6734,7 @@ void
 xmlBufferCCat(xmlBufferPtr buf, const char *str) {
     const char *cur;
 
+    if (buf->alloc == XML_BUFFER_ALLOC_IMMUTABLE) return;
     if (str == NULL) {
 #ifdef DEBUG_BUFFER
         xmlGenericError(xmlGenericErrorContext,
@@ -6716,8 +6764,8 @@ xmlBufferCCat(xmlBufferPtr buf, const char *str) {
  * xmlChars at the end of the buffer.
  */
 void
-xmlBufferWriteCHAR
+xmlBufferWriteCHAR(xmlBufferPtr buf, const xmlChar *string) {
-(xmlBufferPtr buf, const xmlChar *string) {
+    if (buf->alloc == XML_BUFFER_ALLOC_IMMUTABLE) return;
     xmlBufferCat(buf, string);
 }
 
@@ -6731,6 +6779,7 @@ xmlBufferWriteCHAR
  */
 void
 xmlBufferWriteChar(xmlBufferPtr buf, const char *string) {
+    if (buf->alloc == XML_BUFFER_ALLOC_IMMUTABLE) return;
     xmlBufferCCat(buf, string);
 }
 
@@ -6747,6 +6796,7 @@ xmlBufferWriteChar(xmlBufferPtr buf, const char *string) {
 void
 xmlBufferWriteQuotedString(xmlBufferPtr buf, const xmlChar *string) {
     const xmlChar *cur, *base;
+    if (buf->alloc == XML_BUFFER_ALLOC_IMMUTABLE) return;
     if (xmlStrchr(string, '\"')) {
         if (xmlStrchr(string, '\'')) {
 #ifdef DEBUG_BUFFER

xmlIO.c

@@ -2011,6 +2011,52 @@ xmlParserInputBufferCreateMem(const char *mem, int size, xmlCharEncoding enc) {
     return(ret);
 }
 
+/**
+ * xmlParserInputBufferCreateStatic:
+ * @mem:  the memory input
+ * @size:  the length of the memory block
+ * @enc:  the charset encoding if known
+ *
+ * Create a buffered parser input for the progressive parsing for the input
+ * from an immutable memory area. This will not copy the memory area to
+ * the buffer, but the memory is expected to be available until the end of
+ * the parsing, this is useful for example when using mmap'ed file.
+ *
+ * Returns the new parser input or NULL
+ */
+xmlParserInputBufferPtr
+xmlParserInputBufferCreateStatic(const char *mem, int size,
+                                 xmlCharEncoding enc) {
+    xmlParserInputBufferPtr ret;
+
+    if (size <= 0) return(NULL);
+    if (mem == NULL) return(NULL);
+
+    ret = (xmlParserInputBufferPtr) xmlMalloc(sizeof(xmlParserInputBuffer));
+    if (ret == NULL) {
+        xmlGenericError(xmlGenericErrorContext,
+		"xmlParserInputBufferCreateStatic : out of memory!\n");
+	return(NULL);
+    }
+    memset(ret, 0, (size_t) sizeof(xmlParserInputBuffer));
+    ret->buffer = xmlBufferCreateStatic(mem, size);
+    if (ret->buffer == NULL) {
+        xmlFree(ret);
+	return(NULL);
+    }
+    ret->encoder = xmlGetCharEncodingHandler(enc);
+    if (ret->encoder != NULL)
+        ret->raw = xmlBufferCreateSize(2 * xmlDefaultBufferSize);
+    else
+        ret->raw = NULL;
+    ret->compressed = -1;
+    ret->context = (void *) mem;
+    ret->readcallback = NULL;
+    ret->closecallback = NULL;
+
+    return(ret);
+}
+
 /**
  * xmlOutputBufferCreateFd:
  * @fd:  a file descriptor number
@@ -2265,6 +2311,9 @@ xmlParserInputBufferRead(xmlParserInputBufferPtr in, int len) {
     /* xmlBufferEmpty(in->buffer); */
     if (in->readcallback != NULL)
 	return(xmlParserInputBufferGrow(in, len));
+    else if ((in->buffer != NULL) &&
+             (in->buffer->alloc == XML_BUFFER_ALLOC_IMMUTABLE))
+	return(0);
     else
         return(-1);
 }

xmllint.c

@@ -624,7 +624,7 @@ static void streamFile(char *filename) {
 	if (base == (void *) MAP_FAILED)
 	    return;
 
-	input = xmlParserInputBufferCreateMem((char *) base, info.st_size,
+	input = xmlParserInputBufferCreateStatic((char *) base, info.st_size,
 	                                      XML_CHAR_ENCODING_NONE);
 	reader = xmlNewTextReader(input, filename);
     } else

xmlreader.c

@@ -765,12 +765,25 @@ xmlTextReaderPushData(xmlTextReaderPtr reader) {
 	     */
 	    if (reader->mode != XML_TEXTREADER_MODE_EOF) {
 		val = xmlParserInputBufferRead(reader->input, 4096);
-		if (val <= 0) {
+		if ((val == 0) &&
+		    (inbuf->alloc == XML_BUFFER_ALLOC_IMMUTABLE)) {
+		    if (inbuf->use == reader->cur) {
+			reader->mode = XML_TEXTREADER_MODE_EOF;
+			reader->state = oldstate;
+			if ((oldstate != XML_TEXTREADER_START) ||
+			    (reader->ctxt->myDoc != NULL))
+			    return(val);
+		    }
+		} else if (val < 0) {
 		    reader->mode = XML_TEXTREADER_MODE_EOF;
 		    reader->state = oldstate;
 		    if ((oldstate != XML_TEXTREADER_START) ||
 			(reader->ctxt->myDoc != NULL))
 			return(val);
+		} else if (val == 0) {
+		    /* mark the end of the stream and process the remains */
+		    reader->mode = XML_TEXTREADER_MODE_EOF;
+		    break;
 		}
 
 	    } else 
@@ -1572,8 +1585,10 @@ xmlNewTextReader(xmlParserInputBufferPtr input, const char *URI) {
     ret->mode = XML_TEXTREADER_MODE_INITIAL;
     ret->node = NULL;
     ret->curnode = NULL;
-    val = xmlParserInputBufferRead(input, 4);
+    if (ret->input->buffer->use < 4) {
-    if (val >= 4) {
+	val = xmlParserInputBufferRead(input, 4);
+    }
+    if (ret->input->buffer->use >= 4) {
 	ret->ctxt = xmlCreatePushParserCtxt(ret->sax, NULL,
 			(const char *) ret->input->buffer->content, 4, URI);
 	ret->base = 0;