From 6c75e839e970d9da450856d3048fcbac755d38d6 Mon Sep 17 00:00:00 2001
From: David Malcolm <dmalcolm@redhat.com>
Date: Thu, 7 Jul 2016 11:25:53 -0400
Subject: [PATCH 24/25] FIXME: support generating both strings and ranges

---
 libcpp/charset.c | 130 +++++++++++++++++++++++++++++++++++--------------------
 1 file changed, 82 insertions(+), 48 deletions(-)

diff --git a/libcpp/charset.c b/libcpp/charset.c
index 91d95b3..b293efc 100644
--- a/libcpp/charset.c
+++ b/libcpp/charset.c
@@ -812,7 +812,7 @@ cpp_host_to_exec_charset (cpp_reader *pfile, cppchar_t c)
 
 
 
-/* FIXME.  */
+/* cpp_substring_ranges's constructor. */
 
 cpp_substring_ranges::cpp_substring_ranges () :
   m_ranges (NULL),
@@ -822,7 +822,7 @@ cpp_substring_ranges::cpp_substring_ranges () :
   m_ranges = XNEWVEC (source_range, m_alloc_ranges);
 }
 
-/* FIXME.  */
+/* cpp_substring_ranges's destructor. */
 
 cpp_substring_ranges::~cpp_substring_ranges ()
 {
@@ -848,7 +848,7 @@ cpp_substring_ranges::add_range (source_range range)
 
 void
 cpp_substring_ranges::add_n_ranges (int num,
-				cpp_string_location_reader &loc_reader)
+				    cpp_string_location_reader &loc_reader)
 {
   for (int i = 0; i < num; i++)
     add_range (loc_reader.get_next ());
@@ -1024,7 +1024,10 @@ ucn_valid_in_identifier (cpp_reader *pfile, cppchar_t c,
    one beyond the UCN, or to the syntactically invalid character.
 
    IDENTIFIER_POS is 0 when not in an identifier, 1 for the start of
-   an identifier, or 2 otherwise.  */
+   an identifier, or 2 otherwise.
+
+   If CHAR_RANGE and LOC_READER are non-NULL, then position information is
+   read from *LOC_READER and CHAR_RANGE->m_finish is updated accordingly.  */
 
 bool
 _cpp_valid_ucn (cpp_reader *pfile, const uchar **pstr,
@@ -1038,6 +1041,10 @@ _cpp_valid_ucn (cpp_reader *pfile, const uchar **pstr,
   const uchar *str = *pstr;
   const uchar *base = str - 2;
 
+  /* char_range and loc_reader must either be both NULL, or both be
+     non-NULL.  */
+  gcc_assert ((char_range != NULL) == (loc_reader != NULL));
+
   if (!CPP_OPTION (pfile, cplusplus) && !CPP_OPTION (pfile, c99))
     cpp_error (pfile, CPP_DL_WARNING,
 	       "universal character names are only valid in C++ and C99");
@@ -1134,8 +1141,12 @@ _cpp_valid_ucn (cpp_reader *pfile, const uchar **pstr,
 }
 
 /* Convert an UCN, pointed to by FROM, to UTF-8 encoding, then translate
-   it to the execution character set and write the result into TBUF.
-   An advanced pointer is returned.  Issues all relevant diagnostics.  */
+   it to the execution character set and write the result into TBUF,
+   if TBUF is non-NULL.
+   An advanced pointer is returned.  Issues all relevant diagnostics.
+   If LOC_READER is non-NULL, then RANGES must be non-NULL and CHAR_RANGE
+   contains the location of the character so far: location information
+   is read from *LOC_READER, and *RANGES is updated accordingly.  */
 static const uchar *
 convert_ucn (cpp_reader *pfile, const uchar *from, const uchar *limit,
 	     struct _cpp_strbuf *tbuf, struct cset_converter cvt,
@@ -1150,6 +1161,9 @@ convert_ucn (cpp_reader *pfile, const uchar *from, const uchar *limit,
   int rval;
   struct normalize_state nst = INITIAL_NORMALIZE_STATE;
 
+  /* loc_reader and ranges must either be both NULL, or both be non-NULL.  */
+  gcc_assert ((loc_reader != NULL) == (ranges != NULL));
+
   from++;  /* Skip u/U.  */
 
   if (loc_reader)
@@ -1168,18 +1182,17 @@ convert_ucn (cpp_reader *pfile, const uchar *from, const uchar *limit,
     }
   else
     {
+      if (tbuf)
+	if (!APPLY_CONVERSION (cvt, buf, 6 - bytesleft, tbuf))
+	  cpp_errno (pfile, CPP_DL_ERROR,
+		     "converting UCN to execution character set");
+
       if (loc_reader)
 	{
 	  int num_encoded_bytes = 6 - bytesleft;
 	  for (int i = 0; i < num_encoded_bytes; i++)
 	    ranges->add_range (char_range);
 	}
-      else
-	{
-	  if (!APPLY_CONVERSION (cvt, buf, 6 - bytesleft, tbuf))
-	    cpp_errno (pfile, CPP_DL_ERROR,
-		       "converting UCN to execution character set");
-	}
     }
 
   return from;
@@ -1236,11 +1249,14 @@ emit_numeric_escape (cpp_reader *pfile, cppchar_t n,
 }
 
 /* Convert a hexadecimal escape, pointed to by FROM, to the execution
-   character set and write it into the string buffer TBUF.  Returns an
-   advanced pointer, and issues diagnostics as necessary.
+   character set and write it into the string buffer TBUF (if non-NULL).
+   Returns an advanced pointer, and issues diagnostics as necessary.
    No character set translation occurs; this routine always produces the
    execution-set character with numeric value equal to the given hex
-   number.  You can, e.g. generate surrogate pairs this way.  */
+   number.  You can, e.g. generate surrogate pairs this way.
+   If LOC_READER is non-NULL, then RANGES must be non-NULL and CHAR_RANGE
+   contains the location of the character so far: location information
+   is read from *LOC_READER, and *RANGES is updated accordingly.  */
 static const uchar *
 convert_hex (cpp_reader *pfile, const uchar *from, const uchar *limit,
 	     struct _cpp_strbuf *tbuf, struct cset_converter cvt,
@@ -1253,6 +1269,9 @@ convert_hex (cpp_reader *pfile, const uchar *from, const uchar *limit,
   size_t width = cvt.width;
   size_t mask = width_to_mask (width);
 
+  /* loc_reader and ranges must either be both NULL, or both be non-NULL.  */
+  gcc_assert ((loc_reader != NULL) == (ranges != NULL));
+
   if (CPP_WTRADITIONAL (pfile))
     cpp_warning (pfile, CPP_W_TRADITIONAL,
 	         "the meaning of '\\x' is different in traditional C");
@@ -1291,10 +1310,10 @@ convert_hex (cpp_reader *pfile, const uchar *from, const uchar *limit,
       n &= mask;
     }
 
+  if (tbuf)
+    emit_numeric_escape (pfile, n, tbuf, cvt);
   if (ranges)
     ranges->add_range (char_range);
-  else
-    emit_numeric_escape (pfile, n, tbuf, cvt);
 
   return from;
 }
@@ -1304,7 +1323,10 @@ convert_hex (cpp_reader *pfile, const uchar *from, const uchar *limit,
    advanced pointer, and issues diagnostics as necessary.
    No character set translation occurs; this routine always produces the
    execution-set character with numeric value equal to the given octal
-   number.  */
+   number.
+   If LOC_READER is non-NULL, then RANGES must be non-NULL and CHAR_RANGE
+   contains the location of the character so far: location information
+   is read from *LOC_READER, and *RANGES is updated accordingly.  */
 static const uchar *
 convert_oct (cpp_reader *pfile, const uchar *from, const uchar *limit,
 	     struct _cpp_strbuf *tbuf, struct cset_converter cvt,
@@ -1318,6 +1340,9 @@ convert_oct (cpp_reader *pfile, const uchar *from, const uchar *limit,
   size_t mask = width_to_mask (width);
   bool overflow = false;
 
+  /* loc_reader and ranges must either be both NULL, or both be non-NULL.  */
+  gcc_assert ((loc_reader != NULL) == (ranges != NULL));
+
   while (from < limit && count++ < 3)
     {
       c = *from;
@@ -1337,18 +1362,21 @@ convert_oct (cpp_reader *pfile, const uchar *from, const uchar *limit,
       n &= mask;
     }
 
+  if (tbuf)
+    emit_numeric_escape (pfile, n, tbuf, cvt);
   if (ranges)
     ranges->add_range (char_range);
-  else
-    emit_numeric_escape (pfile, n, tbuf, cvt);
 
   return from;
 }
 
 /* Convert an escape sequence (pointed to by FROM) to its value on
    the target, and to the execution character set.  Do not scan past
-   LIMIT.  Write the converted value into TBUF.  Returns an advanced
-   pointer.  Handles all relevant diagnostics.  */
+   LIMIT.  Write the converted value into TBUF, if TBUF is non-NULL.
+   Returns an advanced pointer.  Handles all relevant diagnostics.
+   If LOC_READER is non-NULL, then RANGES must be non-NULL: location
+   information is read from *LOC_READER, and *RANGES is updated
+   accordingly.  */
 static const uchar *
 convert_escape (cpp_reader *pfile, const uchar *from, const uchar *limit,
 		struct _cpp_strbuf *tbuf, struct cset_converter cvt,
@@ -1439,7 +1467,7 @@ convert_escape (cpp_reader *pfile, const uchar *from, const uchar *limit,
 	}
     }
 
-  if (!loc_reader)
+  if (tbuf)
     /* Now convert what we have to the execution character set.  */
     if (!APPLY_CONVERSION (cvt, &c, 1, tbuf))
       cpp_errno (pfile, CPP_DL_ERROR,
@@ -1477,14 +1505,16 @@ converter_for_type (cpp_reader *pfile, enum cpp_ttype type)
    are to be converted from the source to the execution character set,
    escape sequences translated, and finally all are to be
    concatenated.  WIDE indicates whether or not to produce a wide
-   string.  The result is written into TO.  Returns true for success,
-   false for failure.  */
-#if 1
-bool
-cpp_interpret_string_unified (cpp_reader *pfile, const cpp_string *from, size_t count,
-			      cpp_string *to,  enum cpp_ttype type,
-			      cpp_string_location_reader *loc_readers,
-			      cpp_substring_ranges *out)
+   string.  If TO is non-NULL, the result is written into TO.
+   If LOC_READERS and OUT are non-NULL, then location information
+   is read from LOC_READERS (which must be an array of length COUNT),
+   and location information is written to *RANGES.
+   Returns true for success, false for failure.  */
+static bool
+cpp_interpret_string_1 (cpp_reader *pfile, const cpp_string *from, size_t count,
+			cpp_string *to,  enum cpp_ttype type,
+			cpp_string_location_reader *loc_readers,
+			cpp_substring_ranges *out)
 {
   struct _cpp_strbuf tbuf;
   const uchar *p, *base, *limit;
@@ -1532,16 +1562,17 @@ cpp_interpret_string_unified (cpp_reader *pfile, const cpp_string *from, size_t
 	  if (limit >= p + (p - prefix) + 1)
 	    limit -= (p - prefix) + 1;
 
+	  /* Raw strings are all normal characters; these can be fed
+	     directly to convert_cset.  */
+	  if (to)
+	    if (!APPLY_CONVERSION (cvt, p, limit - p, &tbuf))
+	      goto fail;
+
 	  if (loc_reader)
-	    /* Raw strings are all normal characters; assume UTF-8 to UTF-8
+	    /* If generating source ranges, assume UTF-8 to UTF-8
 	       conversion, hence these can be all treated as one byte in the
 	       result per column.  */
 	    out->add_n_ranges (limit - p, *loc_reader);
-	  else
-	    /* Raw strings are all normal characters; these can be fed
-	       directly to convert_cset.  */
-	    if (!APPLY_CONVERSION (cvt, p, limit - p, &tbuf))
-	      goto fail;
 
 	  continue;
 	}
@@ -1562,17 +1593,17 @@ cpp_interpret_string_unified (cpp_reader *pfile, const cpp_string *from, size_t
 	    {
 	      /* We have a run of normal characters; these can be fed
 		 directly to convert_cset.  */
-	      if (loc_reader)
-		out->add_n_ranges (p - base, *loc_reader);
-	      else
+	      if (to)
 		if (!APPLY_CONVERSION (cvt, base, p - base, &tbuf))
 		  goto fail;
+	      if (loc_reader)
+		out->add_n_ranges (p - base, *loc_reader);
 	    }
 	  if (p == limit)
 	    break;
 
-	  // FIXME: do we need to conditionalize the &tbuf on "to"?
-	  p = convert_escape (pfile, p + 1, limit, &tbuf, cvt,
+	  struct _cpp_strbuf *tbuf_ptr = to ? &tbuf : NULL;
+	  p = convert_escape (pfile, p + 1, limit, tbuf_ptr, cvt,
 			      loc_reader, out);
 	}
     }
@@ -1595,7 +1626,6 @@ cpp_interpret_string_unified (cpp_reader *pfile, const cpp_string *from, size_t
     free (tbuf.text);
   return false;
 }
-#endif
 
 /* FROM is an array of cpp_string structures of length COUNT.  These
    are to be converted from the source to the execution character set,
@@ -1607,11 +1637,15 @@ bool
 cpp_interpret_string (cpp_reader *pfile, const cpp_string *from, size_t count,
 		      cpp_string *to,  enum cpp_ttype type)
 {
-  return cpp_interpret_string_unified (pfile, from, count, to, type,
-				       NULL, NULL);
+  return cpp_interpret_string_1 (pfile, from, count, to, type, NULL, NULL);
 }
 
-/* FIXME.  */
+/* This function mimics the behavior of cpp_interpret_string, but
+   rather than generating a string in the execution character set,
+   *OUT is written to with the source code ranges of the characters
+   in such a string.
+   FROM and LOC_READERS should both be arrays of length COUNT.
+   Returns true for success, false for failure.  */
 
 bool
 cpp_interpret_string_ranges (cpp_reader *pfile, const cpp_string *from,
@@ -1623,8 +1657,8 @@ cpp_interpret_string_ranges (cpp_reader *pfile, const cpp_string *from,
 #if HOST_CHARSET == HOST_CHARSET_ASCII
 
   /* We assume UTF-8 to UTF-8 conversion.  */
-  return cpp_interpret_string_unified (pfile, from, count, NULL, CPP_STRING,
-				       loc_readers, out);
+  return cpp_interpret_string_1 (pfile, from, count, NULL, CPP_STRING,
+				 loc_readers, out);
 #else
   return false;
 #endif
-- 
1.8.5.3

