From b18733af7d7fa6d4196969cc0a62bd0a2328b7b0 Mon Sep 17 00:00:00 2001
From: David Malcolm <dmalcolm@redhat.com>
Date: Wed, 15 Jun 2016 20:54:33 -0400
Subject: [PATCH 8/8] FIXME: fixups to location tracking within string literals

---
 gcc/input.c             | 293 +++++++++++++++++++++++++++++++++++++++++++++---
 gcc/selftest.h          |  14 ++-
 libcpp/charset.c        |  47 ++++----
 libcpp/include/cpplib.h |  36 ++++--
 4 files changed, 337 insertions(+), 53 deletions(-)

diff --git a/gcc/input.c b/gcc/input.c
index 3fb4a25..aedcaba 100644
--- a/gcc/input.c
+++ b/gcc/input.c
@@ -23,6 +23,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "intl.h"
 #include "diagnostic-core.h"
 #include "selftest.h"
+#include "cpplib.h"
 
 /* This is a cache used by get_next_line to store the content of a
    file to be searched for file lines.  */
@@ -1144,6 +1145,45 @@ namespace selftest {
 
 /* Selftests of location handling.  */
 
+/* FIXME.  */
+
+class temp_source_file
+{
+ public:
+  temp_source_file (const location &loc, const char *suffix,
+		    const char *content);
+  ~temp_source_file ();
+
+  const char *get_filename () const { return m_filename; }
+
+ private:
+  char *m_filename;
+};
+
+/* FIXME.  */
+
+temp_source_file::temp_source_file (const location &loc, const char *suffix,
+				    const char *content)
+{
+  m_filename = make_temp_file (suffix);
+  ASSERT_NE (m_filename, NULL);
+
+  FILE *out = fopen (m_filename, "w");
+  if (!out)
+    ::selftest::fail_formatted (loc, "unable to open tempfile: %s",
+				m_filename);
+  fprintf (out, content);
+  fclose (out);
+}
+
+/* FIXME.  */
+
+temp_source_file::~temp_source_file ()
+{
+  unlink (m_filename);
+  free (m_filename);
+}
+
 /* Verify the result of LOCATION_FILE/LOCATION_LINE/LOCATION_COLUMN
    on LOC.  */
 
@@ -1214,27 +1254,252 @@ test_builtins ()
   ASSERT_PRED1 (is_location_from_builtin_token, BUILTINS_LOCATION);
 }
 
+/* Verify that locations within string literals are correctly handled.  */
+
+/* Verify STRLOC's description of locations within a string literal.
+   Assert that the character at index IDX begins at column EXPECTED_START_COL
+   and ends at EXPECTED_FINISH_COL.  */
+
+static void
+assert_char_at_range (const location &loc,
+		      const cpp_string_location *strloc, int idx,
+		      int expected_start_col, int expected_finish_col)
+{
+  source_range actual_range = strloc->get_range_at_index (idx);
+  int actual_start_col = LOCATION_COLUMN (actual_range.m_start);
+  ASSERT_EQ_AT (loc, expected_start_col, actual_start_col);
+
+  //FIXME: we don't properly handle this case yet, so don't test it
+#if 0
+  if (expected_start_col != expected_finish_col)
+    return;
+#endif
+
+  int actual_finish_col = LOCATION_COLUMN (actual_range.m_finish);
+  ASSERT_EQ_AT (loc, expected_finish_col, actual_finish_col);
+}
+
+/* FIXME.  */
+
+static void
+assert_string_location_fragment (const location &loc,
+				 const cpp_string_location *strloc, int idx,
+				 location_t expected_start_loc,
+				 int expected_len, int expected_cols_per_char,
+				 int expected_bits_per_col)
+{
+  const cpp_string_fragment_location &fragment = strloc->m_fragloc_array[idx];
+  ASSERT_EQ_AT (loc, fragment.m_loc, expected_start_loc);
+  ASSERT_EQ_AT (loc, fragment.m_len, expected_len);
+  ASSERT_EQ_AT (loc, fragment.m_cols_per_char, expected_cols_per_char);
+  ASSERT_EQ_AT (loc, fragment.m_bits_per_col, expected_bits_per_col);
+}
+
+#define ASSERT_STRING_LOCATION_FRAGMENT(STRLOC, IDX, EXPECTED_START_LOC, \
+					EXPECTED_LEN, EXPECTED_COLS_PER_CHAR, \
+					EXPECTED_BITS_PER_COL) \
+  assert_string_location_fragment (SELFTEST_LOCATION, (STRLOC), (IDX), \
+				   (EXPECTED_START_LOC), (EXPECTED_LEN), \
+				   (EXPECTED_COLS_PER_CHAR),	\
+				   (EXPECTED_BITS_PER_COL))
+
+
+/* Verify that locations work within a string literal, for the
+   simple case of a single unbroken run of 1 column-per-character
+   characters.  */
+
+static void
+test_simple_substring_locations ()
+{
+  linemap_add (line_table, LC_ENTER, false,
+	       "test-simple-substring-location.c", 0);
+  linemap_line_start (line_table, 1, 100);
+
+  const int start_column = 5;
+  location_t loc = linemap_position_for_column (line_table, start_column);
+
+  cpp_string_location strloc;
+  const int len = 10;
+  strloc.init_raw (loc, len, 1, line_table);
+
+  /* Ensure that the line table contains all of the locations within the
+     string.  Specifically, ensure that they are all
+       <= line_table->highest_location, so that
+     linemap_location_from_macro_expansion_p returns false.  */
+  linemap_position_for_column (line_table, start_column + len);
+
+  /* Verify that we can correctly retrieve the range for each
+     character.  */
+  for (int i = 0; i < len; i++)
+    {
+      int expected_column = start_column + i;
+      assert_char_at_range (SELFTEST_LOCATION, &strloc, i,
+			    expected_column, expected_column);
+    }
+
+  /* Verify that the data was packed as expected.  */
+  ASSERT_TRUE (strloc.trivial_p ());
+  ASSERT_EQ (strloc.m_num_fraglocs, 1);
+  ASSERT_STRING_LOCATION_FRAGMENT (&strloc, 0, loc, len, 1,
+				   line_table->default_range_bits);
+
+  linemap_add (line_table, LC_LEAVE, false, NULL, 0);
+}
+
+/* FIXME.  */
+
+static void
+add_char_at (cpp_string_location *strloc, int start_col, int end_col)
+{
+  source_range range;
+  range.m_start = linemap_position_for_column (line_table, start_col);
+  range.m_finish = linemap_position_for_column (line_table, end_col);
+  strloc->add_char_at (range, line_table);
+}
+
+/* Verify a complicated example of locations within a string literal.
+
+   This uses the example from cpplib.h, with the following column
+   numbering:
+000000000111111111122222222223333333333
+123456789012345678901234567890123456789
+    printf ("foo \x25\151 bar"  "baz\n",
+    .........^^^^.....................: fragment 0: 4 chars at 1 col per char
+    .............^^^^^^^^.............: fragment 1: 2 chars at 4 cols per char
+    .....................^^^^.........: fragment 2: 4 chars at 1 col per char
+    .............................^^^..: fragment 3: 3 chars at 1 col per char
+    ................................^^: fragment 4: 1 char at 2 cols per char
+
+    We populate a cpp_string_location with the above data, then
+    verify reading the data back.  */
+
+static void
+test_complex_substring_locations ()
+{
+  /* Populate strloc as per the example above.  */
+  cpp_string_location strloc;
+  strloc.init ();
+
+  /* Create a tempfile containing the content we're describing.
+     Apart from providing a filename, this isn't directly used by the test,
+     but is useful when debugging locations via "(gdb) call inform" and the
+     like.  */
+  temp_source_file tmp (SELFTEST_LOCATION, ".c",
+			"    printf (\"foo \\x25\\151 bar\"  \"baz\",");
+
+  linemap_add (line_table, LC_ENTER, false, tmp.get_filename (), 0);
+  linemap_line_start (line_table, 1, 100);
+
+  /* "foo ".  */
+  for (int col = 14; col <= 17; col++)
+    add_char_at (&strloc, col, col);
+  /* "\x25\151".  */
+  add_char_at (&strloc, 18, 21);
+  add_char_at (&strloc, 22, 25);
+  /* " bar".  */
+  for (int col = 26; col <= 29; col++)
+    add_char_at (&strloc, col, col);
+  /* "baz".  */
+  for (int col = 34; col <= 36; col++)
+    add_char_at (&strloc, col, col);
+  /* "\n".  */
+  add_char_at (&strloc, 37, 38);
+
+  /* Ensure that the line table contains all of the locations within the
+     string.  Specifically, ensure that they are all
+       <= line_table->highest_location, so that
+     linemap_location_from_macro_expansion_p returns false.  */
+  linemap_position_for_column (line_table, 38);
+
+  /* Verify that we can correctly retrieve the range for each
+     character.  */
+#define ASSERT_CHAR_AT_RANGE(IDX, EXPECTED_START_COL, EXPECTED_FINISH_COL) \
+  assert_char_at_range (SELFTEST_LOCATION, &strloc, (IDX),		\
+			(EXPECTED_START_COL), (EXPECTED_FINISH_COL))
+
+  /* "foo ".  */
+  ASSERT_CHAR_AT_RANGE (0, 14, 14);
+  ASSERT_CHAR_AT_RANGE (1, 15, 15);
+  ASSERT_CHAR_AT_RANGE (2, 16, 16);
+  ASSERT_CHAR_AT_RANGE (3, 17, 17);
+
+  /* "\x25\151".  */
+  ASSERT_CHAR_AT_RANGE (4, 18, 21);
+  ASSERT_CHAR_AT_RANGE (5, 22, 25);
+
+  /* " bar".  */
+  ASSERT_CHAR_AT_RANGE (6, 26, 26);
+  ASSERT_CHAR_AT_RANGE (7, 27, 27);
+  ASSERT_CHAR_AT_RANGE (8, 28, 28);
+  ASSERT_CHAR_AT_RANGE (9, 29, 29);
+
+  /* "baz".  */
+  ASSERT_CHAR_AT_RANGE (10, 34, 34);
+  ASSERT_CHAR_AT_RANGE (11, 35, 35);
+  ASSERT_CHAR_AT_RANGE (12, 36, 36);
+
+  /* "\n".  */
+  ASSERT_CHAR_AT_RANGE (13, 37, 38);
+
+#undef ASSERT_CHAR_AT_RANGE
+
+  /* Verify that the data was packed as expected.  */
+  ASSERT_FALSE (strloc.trivial_p ());
+
+  ASSERT_EQ (strloc.m_num_fraglocs, 5);
+
+  /* "foo ".  */
+  ASSERT_STRING_LOCATION_FRAGMENT
+    (&strloc, 0,
+     linemap_position_for_column (line_table, 14),
+     4, 1, line_table->default_range_bits);
+
+  /* "\x25\151".  */
+  ASSERT_STRING_LOCATION_FRAGMENT
+    (&strloc, 1,
+     linemap_position_for_column (line_table, 18),
+     2, 4, line_table->default_range_bits);
+
+  /* " bar".  */
+  ASSERT_STRING_LOCATION_FRAGMENT
+    (&strloc, 2,
+     linemap_position_for_column (line_table, 26),
+     4, 1, line_table->default_range_bits);
+
+  /* "baz".  */
+  ASSERT_STRING_LOCATION_FRAGMENT
+    (&strloc, 3,
+     linemap_position_for_column (line_table, 34),
+     3, 1, line_table->default_range_bits);
+
+  /* "\n".  */
+  ASSERT_STRING_LOCATION_FRAGMENT
+    (&strloc, 4,
+     linemap_position_for_column (line_table, 37),
+     1, 2, line_table->default_range_bits);
+
+  linemap_add (line_table, LC_LEAVE, false, NULL, 0);
+}
+
+/* TODO: the same, but via the reader interface.  */
+
+/* TODO: a testcase for a linemap where bits per col is not the default.  */
+
 /* Verify reading of input files (e.g. for caret-based diagnostics).  */
 
 static void
 test_reading_source_line ()
 {
   /* Create a tempfile and write some text to it.  */
-  char *filename = make_temp_file (".txt");
-  ASSERT_NE (filename, NULL);
-  FILE *out = fopen (filename, "w");
-  if (!out)
-    ::selftest::fail_formatted (SELFTEST_LOCATION,
-				"unable to open tempfile: %s", filename);
-  fprintf (out,
-	   "01234567890123456789\n"
-	   "This is the test text\n"
-	   "This is the 3rd line\n");
-  fclose (out);
+  temp_source_file tmp (SELFTEST_LOCATION, ".txt",
+			"01234567890123456789\n"
+			"This is the test text\n"
+			"This is the 3rd line\n");
 
   /* Read back a specific line from the tempfile.  */
   int line_size;
-  const char *source_line = location_get_source_line (filename, 2, &line_size);
+  const char *source_line = location_get_source_line (tmp.get_filename (),
+						      2, &line_size);
   ASSERT_TRUE (source_line != NULL);
   ASSERT_EQ (21, line_size);
   if (!strncmp ("This is the test text",
@@ -1245,8 +1510,6 @@ test_reading_source_line ()
     ::selftest::fail (SELFTEST_LOCATION,
 		      "source_line did not match expected value");
 
-  unlink (filename);
-  free (filename);
 }
 
 /* Run all of the selftests within this file.  */
@@ -1257,6 +1520,8 @@ input_c_tests ()
   test_accessing_ordinary_linemaps ();
   test_unknown_location ();
   test_builtins ();
+  test_simple_substring_locations ();
+  test_complex_substring_locations ();
   test_reading_source_line ();
 }
 
diff --git a/gcc/selftest.h b/gcc/selftest.h
index e719f5f..f71c246 100644
--- a/gcc/selftest.h
+++ b/gcc/selftest.h
@@ -127,13 +127,19 @@ extern int num_passes;
    ::selftest::pass if they are equal,
    ::selftest::fail if they are non-equal.  */
 
-#define ASSERT_EQ(EXPECTED, ACTUAL)			       \
+#define ASSERT_EQ(EXPECTED, ACTUAL) \
+  ASSERT_EQ_AT ((SELFTEST_LOCATION), (EXPECTED), (ACTUAL))
+
+/* Like ASSERT_EQ, but treat LOC as the effective location of the
+   selftest.  */
+
+#define ASSERT_EQ_AT(LOC, EXPECTED, ACTUAL)		       \
   SELFTEST_BEGIN_STMT					       \
   const char *desc = "ASSERT_EQ (" #EXPECTED ", " #ACTUAL ")"; \
   if ((EXPECTED) == (ACTUAL))				       \
-    ::selftest::pass (SELFTEST_LOCATION, desc);			       \
+    ::selftest::pass ((LOC), desc);			       \
   else							       \
-    ::selftest::fail (SELFTEST_LOCATION, desc);			       \
+    ::selftest::fail ((LOC), desc);			       \
   SELFTEST_END_STMT
 
 /* Evaluate EXPECTED and ACTUAL and compare them with !=, calling
@@ -159,7 +165,7 @@ extern int num_passes;
 			    (EXPECTED), (ACTUAL));		    \
   SELFTEST_END_STMT
 
-/* Like ASSERT_STREQ_AT, but treat LOC as the effective location of the
+/* Like ASSERT_STREQ, but treat LOC as the effective location of the
    selftest.  */
 
 #define ASSERT_STREQ_AT(LOC, EXPECTED, ACTUAL)			    \
diff --git a/libcpp/charset.c b/libcpp/charset.c
index 7e4846b..7d8c70f 100644
--- a/libcpp/charset.c
+++ b/libcpp/charset.c
@@ -1923,6 +1923,7 @@ cpp_string_location::init_raw (source_location loc, int len, int cols_per_char,
 				LRK_SPELLING_LOCATION, NULL);
   m_fragloc_array[0].m_loc = ordinary_loc;
   m_fragloc_array[0].m_cols_per_char = cols_per_char;
+  m_fragloc_array[0].m_bits_per_col = 5; // FIXME
   m_num_fraglocs = 1;
   m_alloc_fraglocs = 1;
 }
@@ -1944,26 +1945,22 @@ cpp_string_location::add_char_at (source_range range,
 	  && range.m_finish == next_range.m_finish)
 	/* If so, we can simply increase the length of the current
 	   fragment.  */
-	current_fragment->m_len++;
-      else
 	{
-	  /* We need to start a new fragment.  This may require growing
-	     the underlying array.  */
-	  if (++m_num_fraglocs > m_alloc_fraglocs)
-	    {
-	      m_alloc_fraglocs *= 2;
-	      line_map_realloc reallocator = (line_table->reallocator
-					      ? line_table->reallocator
-					      : (line_map_realloc) xrealloc);
-	      m_fragloc_array = (cpp_string_fragment_location *)reallocator
-		(m_fragloc_array,
-		 sizeof (cpp_string_fragment_location) * m_alloc_fraglocs);
-	    }
-	  current_fragment = get_current_fragment ();
-	  current_fragment->m_len = 1;
-	  current_fragment->m_loc = range.m_start;
-	  current_fragment->m_cols_per_char
-	    = range.m_finish + 1 - range.m_start;
+	  current_fragment->m_len++;
+	  return;
+	}
+
+      /* We need to start a new fragment.  This may require growing
+	 the underlying array.  */
+      if (++m_num_fraglocs > m_alloc_fraglocs)
+	{
+	  m_alloc_fraglocs *= 2;
+	  line_map_realloc reallocator = (line_table->reallocator
+					  ? line_table->reallocator
+					  : (line_map_realloc) xrealloc);
+	  m_fragloc_array = (cpp_string_fragment_location *)reallocator
+	    (m_fragloc_array,
+	     sizeof (cpp_string_fragment_location) * m_alloc_fraglocs);
 	}
     }
   else
@@ -1974,13 +1971,17 @@ cpp_string_location::add_char_at (source_range range,
 				      : (line_map_realloc) xrealloc);
       m_fragloc_array = (cpp_string_fragment_location *)reallocator
 	(NULL, sizeof (cpp_string_fragment_location));
-      m_fragloc_array[0].m_len = 1;
-      m_fragloc_array[0].m_loc = range.m_start;
-      m_fragloc_array[0].m_cols_per_char
-	= range.m_finish + 1 - range.m_start;
       m_num_fraglocs = 1;
       m_alloc_fraglocs = 1;
     }
+
+  cpp_string_fragment_location *current_fragment = get_current_fragment ();
+  current_fragment->m_len = 1;
+  current_fragment->m_loc = range.m_start;
+  current_fragment->m_bits_per_col = 5; // FIXME
+  current_fragment->m_cols_per_char
+    = (((range.m_finish - range.m_start)
+	>> current_fragment->m_bits_per_col) + 1);
 }
 
 /* FIXME.  */
diff --git a/libcpp/include/cpplib.h b/libcpp/include/cpplib.h
index 37e977a..4f910fd 100644
--- a/libcpp/include/cpplib.h
+++ b/libcpp/include/cpplib.h
@@ -179,14 +179,25 @@ enum c_lang {CLK_GNUC89 = 0, CLK_GNUC99, CLK_GNUC11,
    See the description below for cpp_string_location.  */
 struct GTY(()) cpp_string_fragment_location {
   source_location m_loc;
-  unsigned int m_len : 12;
-  unsigned int m_cols_per_char : 4;
+  unsigned int m_len : 16;
+  unsigned int m_cols_per_char : 8;
+  unsigned int m_bits_per_col : 8;
+
+  source_location get_char_start (int idx) const
+  {
+    return m_loc + (idx * m_cols_per_char * (1 << m_bits_per_col));
+  }
+
+  source_location get_char_finish (int idx) const
+  {
+    return get_char_start (idx) + ((m_cols_per_char - 1) * (1 << m_bits_per_col));
+  }
 
   source_range get_char_range (int idx) const
   {
     source_range result;
-    result.m_start = m_loc + (idx * m_cols_per_char);
-    result.m_finish = result.m_start + m_cols_per_char - 1;
+    result.m_start = get_char_start (idx);
+    result.m_finish = get_char_finish (idx);
     return result;
   }
   source_range get_next_range () const
@@ -197,7 +208,7 @@ struct GTY(()) cpp_string_fragment_location {
   {
     source_range result;
     result.m_start = m_loc;
-    result.m_finish = m_loc + (m_len * m_cols_per_char) - 1;
+    result.m_finish = get_char_finish (m_len - 1);
     return result;
   }
   void debug (const char *msg) const;
@@ -209,7 +220,7 @@ class cpp_string_location_reader;
    This is stored as a dynamically-allocated array of fragments.
    For example, consider this call to printf:
 
-     printf ("foo \x25\151 bar"  "baz",
+     printf ("foo \x25\151 bar"  "baz\n",
              "not an int");
 
    The string constant for the first parameter is composed of
@@ -217,7 +228,7 @@ class cpp_string_location_reader;
    encoding of a '%' and octal encoding of a 'i', giving a
    resulting STRING_CST of:
 
-     "foo %i barbaz"
+     "foo %i barbaz\n"
 
    We want to efficiently record the range of locations in the
    source file of each character so that we can emit warnings about
@@ -229,11 +240,12 @@ class cpp_string_location_reader;
    a consistent number of columns per character.  In the example
    above the fragments are:
 
-    printf ("foo \x25\151 bar"  "baz",
-    .........^^^^....................: fragment 0: 4 chars at 1 col per char
-    .............^^^^^^^^............: fragment 1: 2 chars at 4 cols per char
-    .....................^^^^........: fragment 2: 4 chars at 1 col per char
-    .............................^^^.: fragment 3: 3 chars at 1 col per char
+    printf ("foo \x25\151 bar"  "baz\n",
+    .........^^^^.....................: fragment 0: 4 chars at 1 col per char
+    .............^^^^^^^^.............: fragment 1: 2 chars at 4 cols per char
+    .....................^^^^.........: fragment 2: 4 chars at 1 col per char
+    .............................^^^..: fragment 3: 3 chars at 1 col per char
+    ................................^^: fragment 4: 1 char at 2 cols per char
 
    Note that the hex and octal chars both happen to be 4 cols per char
    and are contiguous, hence both end up being in fragment 1, whereas the
-- 
1.8.5.3

