To: vim-dev@vim.org
Subject: Patch 7.2.312
Fcc: outbox
From: Bram Moolenaar <Bram@moolenaar.net>
Mime-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
------------

Patch 7.2.312
Problem:    iconv() returns an invalid character sequence when conversion
	    fails.  It should return an empty string. (Yongwei Wu)
Solution:   Be more strict about invalid characters in the input.
Files:	    src/mbyte.c


*** ../vim-7.2.311/src/mbyte.c	2009-06-16 15:23:07.000000000 +0200
--- src/mbyte.c	2009-11-25 16:10:44.000000000 +0100
***************
*** 133,154 ****
  static int dbcs_ptr2cells_len __ARGS((char_u *p, int size));
  static int dbcs_ptr2char __ARGS((char_u *p));
  
! /* Lookup table to quickly get the length in bytes of a UTF-8 character from
!  * the first byte of a UTF-8 string.  Bytes which are illegal when used as the
!  * first byte have a one, because these will be used separately. */
  static char utf8len_tab[256] =
  {
      1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
      1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
      1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
      1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
!     1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, /*bogus*/
!     1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, /*bogus*/
      2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
      3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,5,5,5,5,6,6,1,1,
  };
  
  /*
   * XIM often causes trouble.  Define XIM_DEBUG to get a log of XIM callbacks
   * in the "xim.log" file.
   */
--- 133,172 ----
  static int dbcs_ptr2cells_len __ARGS((char_u *p, int size));
  static int dbcs_ptr2char __ARGS((char_u *p));
  
! /*
!  * Lookup table to quickly get the length in bytes of a UTF-8 character from
!  * the first byte of a UTF-8 string.
!  * Bytes which are illegal when used as the first byte have a 1.
!  * The NUL byte has length 1.
!  */
  static char utf8len_tab[256] =
  {
      1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
      1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
      1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
      1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
!     1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
!     1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
      2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
      3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,5,5,5,5,6,6,1,1,
  };
  
  /*
+  * Like utf8len_tab above, but using a zero for illegal lead bytes.
+  */
+ static char utf8len_tab_zero[256] =
+ {
+     1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
+     1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
+     1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
+     1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
+     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+     2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
+     3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,5,5,5,5,6,6,0,0,
+ };
+ 
+ /*
   * XIM often causes trouble.  Define XIM_DEBUG to get a log of XIM callbacks
   * in the "xim.log" file.
   */
***************
*** 1352,1358 ****
      if (size > 0 && *p >= 0x80)
      {
  	if (utf_ptr2len_len(p, size) < utf8len_tab[*p])
! 	    return 1;
  	c = utf_ptr2char(p);
  	/* An illegal byte is displayed as <xx>. */
  	if (utf_ptr2len(p) == 1 || c == NUL)
--- 1370,1376 ----
      if (size > 0 && *p >= 0x80)
      {
  	if (utf_ptr2len_len(p, size) < utf8len_tab[*p])
! 	    return 1;  /* truncated */
  	c = utf_ptr2char(p);
  	/* An illegal byte is displayed as <xx>. */
  	if (utf_ptr2len(p) == 1 || c == NUL)
***************
*** 1473,1479 ****
      if (p[0] < 0x80)	/* be quick for ASCII */
  	return p[0];
  
!     len = utf8len_tab[p[0]];
      if (len > 1 && (p[1] & 0xc0) == 0x80)
      {
  	if (len == 2)
--- 1491,1497 ----
      if (p[0] < 0x80)	/* be quick for ASCII */
  	return p[0];
  
!     len = utf8len_tab_zero[p[0]];
      if (len > 1 && (p[1] & 0xc0) == 0x80)
      {
  	if (len == 2)
***************
*** 1723,1728 ****
--- 1741,1747 ----
  /*
   * Return length of UTF-8 character, obtained from the first byte.
   * "b" must be between 0 and 255!
+  * Returns 1 for an invalid first byte value.
   */
      int
  utf_byte2len(b)
***************
*** 1737,1742 ****
--- 1756,1762 ----
   * Returns 1 for "".
   * Returns 1 for an illegal byte sequence (also in incomplete byte seq.).
   * Returns number > "size" for an incomplete byte sequence.
+  * Never returns zero.
   */
      int
  utf_ptr2len_len(p, size)
***************
*** 1747,1757 ****
      int		i;
      int		m;
  
!     if (*p == NUL)
! 	return 1;
!     m = len = utf8len_tab[*p];
      if (len > size)
  	m = size;	/* incomplete byte sequence. */
      for (i = 1; i < m; ++i)
  	if ((p[i] & 0xc0) != 0x80)
  	    return 1;
--- 1767,1779 ----
      int		i;
      int		m;
  
!     len = utf8len_tab[*p];
!     if (len == 1)
! 	return 1;	/* NUL, ascii or illegal lead byte */
      if (len > size)
  	m = size;	/* incomplete byte sequence. */
+     else
+ 	m = len;
      for (i = 1; i < m; ++i)
  	if ((p[i] & 0xc0) != 0x80)
  	    return 1;
***************
*** 2505,2510 ****
--- 2527,2533 ----
  /*
   * mb_head_off() function pointer.
   * Return offset from "p" to the first byte of the character it points into.
+  * If "p" points to the NUL at the end of the string return 0.
   * Returns 0 when already at the first byte of a character.
   */
      int
***************
*** 2524,2530 ****
  
      /* It can't be a trailing byte when not using DBCS, at the start of the
       * string or the previous byte can't start a double-byte. */
!     if (p <= base || MB_BYTE2LEN(p[-1]) == 1)
  	return 0;
  
      /* This is slow: need to start at the base and go forward until the
--- 2547,2553 ----
  
      /* It can't be a trailing byte when not using DBCS, at the start of the
       * string or the previous byte can't start a double-byte. */
!     if (p <= base || MB_BYTE2LEN(p[-1]) == 1 || *p == NUL)
  	return 0;
  
      /* This is slow: need to start at the base and go forward until the
***************
*** 2552,2558 ****
       * lead byte in the current cell. */
      if (p <= base
  	    || (enc_dbcs == DBCS_JPNU && p[-1] == 0x8e)
! 	    || MB_BYTE2LEN(p[-1]) == 1)
  	return 0;
  
      /* This is slow: need to start at the base and go forward until the
--- 2575,2582 ----
       * lead byte in the current cell. */
      if (p <= base
  	    || (enc_dbcs == DBCS_JPNU && p[-1] == 0x8e)
! 	    || MB_BYTE2LEN(p[-1]) == 1
! 	    || *p == NUL)
  	return 0;
  
      /* This is slow: need to start at the base and go forward until the
***************
*** 2578,2583 ****
--- 2602,2608 ----
      char_u	*q;
      char_u	*s;
      int		c;
+     int		len;
  #ifdef FEAT_ARABIC
      char_u	*j;
  #endif
***************
*** 2597,2604 ****
  	    --q;
  	/* Check for illegal sequence. Do allow an illegal byte after where we
  	 * started. */
! 	if (utf8len_tab[*q] != (int)(s - q + 1)
! 				       && utf8len_tab[*q] != (int)(p - q + 1))
  	    return 0;
  
  	if (q <= base)
--- 2622,2629 ----
  	    --q;
  	/* Check for illegal sequence. Do allow an illegal byte after where we
  	 * started. */
! 	len = utf8len_tab[*q];
! 	if (len != (int)(s - q + 1) && len != (int)(p - q + 1))
  	    return 0;
  
  	if (q <= base)
***************
*** 2810,2818 ****
  
      while (end == NULL ? *p != NUL : p < end)
      {
! 	if ((*p & 0xc0) == 0x80)
  	    return FALSE;	/* invalid lead byte */
- 	l = utf8len_tab[*p];
  	if (end != NULL && p + l > end)
  	    return FALSE;	/* incomplete byte sequence */
  	++p;
--- 2835,2843 ----
  
      while (end == NULL ? *p != NUL : p < end)
      {
! 	l = utf8len_tab_zero[*p];
! 	if (l == 0)
  	    return FALSE;	/* invalid lead byte */
  	if (end != NULL && p + l > end)
  	    return FALSE;	/* incomplete byte sequence */
  	++p;
***************
*** 6117,6128 ****
  	    d = retval;
  	    for (i = 0; i < len; ++i)
  	    {
! 		l = utf_ptr2len(ptr + i);
  		if (l == 0)
  		    *d++ = NUL;
  		else if (l == 1)
  		{
! 		    if (unconvlenp != NULL && utf8len_tab[ptr[i]] > len - i)
  		    {
  			/* Incomplete sequence at the end. */
  			*unconvlenp = len - i;
--- 6142,6161 ----
  	    d = retval;
  	    for (i = 0; i < len; ++i)
  	    {
! 		l = utf_ptr2len_len(ptr + i, len - i);
  		if (l == 0)
  		    *d++ = NUL;
  		else if (l == 1)
  		{
! 		    int l_w = utf8len_tab_zero[ptr[i]];
! 
! 		    if (l_w == 0)
! 		    {
! 			/* Illegal utf-8 byte cannot be converted */
! 			vim_free(retval);
! 			return NULL;
! 		    }
! 		    if (unconvlenp != NULL && l_w > len - i)
  		    {
  			/* Incomplete sequence at the end. */
  			*unconvlenp = len - i;
*** ../vim-7.2.311/src/version.c	2009-12-02 13:32:10.000000000 +0100
--- src/version.c	2009-12-02 15:00:23.000000000 +0100
***************
*** 683,684 ****
--- 683,686 ----
  {   /* Add new patch number below this line */
+ /**/
+     312,
  /**/

-- 
hundred-and-one symptoms of being an internet addict:
6. You refuse to go to a vacation spot with no electricity and no phone lines.

 /// Bram Moolenaar -- Bram@Moolenaar.net -- http://www.Moolenaar.net   \\\
///        sponsor Vim, vote for features -- http://www.Vim.org/sponsor/ \\\
\\\        download, build and distribute -- http://www.A-A-P.org        ///
 \\\            help me help AIDS victims -- http://ICCF-Holland.org    ///