From 7bd54ddd5cbd8d16a79f1bc34dbb18c855c05baa Mon Sep 17 00:00:00 2001
From: David Malcolm <dmalcolm@redhat.com>
Date: Wed, 6 Jul 2016 11:22:04 -0400
Subject: [PATCH 08/25] FIXME: more WIP on on-demand substring locations

---
 gcc/input.c | 190 +++++++++++++++++++++++++++++++++++++++++++++---------------
 1 file changed, 143 insertions(+), 47 deletions(-)

diff --git a/gcc/input.c b/gcc/input.c
index 50c29f7..2f06b5c 100644
--- a/gcc/input.c
+++ b/gcc/input.c
@@ -1154,18 +1154,19 @@ get_substring_ranges_for_loc (cpp_reader *pfile,
   /* FIXME: handle concatenated strings by doing this repeatedly, somehow.
      We'd need to capture concatenations when they happen.  */
 
-  /* Get range of strloc, use it to locate the start of the literal token.
-     We can't reliably determine the literal length and truncate to it
-     since we might be near or beyond LINE_MAP_MAX_LOCATION_WITH_COLS.
-     Instead we simply read the line until we reach the terminator.  */
+  /* Get range of strloc.  We will use it to locate the start and finish
+     of the literal token within the line.  */
   source_range src_range = get_range_from_loc (line_table, strloc);
 
-#if 0
   if (src_range.m_start >= LINE_MAP_MAX_LOCATION_WITH_COLS)
     /* If so, we can't reliably determine where the token started within
        its line.  */
     return false;
-#endif
+
+  if (src_range.m_finish >= LINE_MAP_MAX_LOCATION_WITH_COLS)
+    /* If so, we can't reliably determine where the token finished within
+       its line.  */
+    return false;
 
   expanded_location start
     = expand_location_to_spelling_point (src_range.m_start);
@@ -1187,6 +1188,7 @@ get_substring_ranges_for_loc (cpp_reader *pfile,
   /* Determine the location of the literal (including quotes
      and leading FIXME.  */
   const char *literal = line + start.column - 1;
+  int literal_length = finish.column - start.column + 1;
 
   /* Rerun cpp_interpret_string, or rather, a modified version of it.  */
   /* FIXME: CPP_STRING32 captures the unicode in a fixed-width (4-byte
@@ -1195,8 +1197,8 @@ get_substring_ranges_for_loc (cpp_reader *pfile,
      For now, we simply constrain ourselves to the UTF-8 case (for both
      source and execution encoding).  */
   cpp_string from;
-  gcc_assert (line_width >= (literal - line));
-  from.len = line_width - (literal - line);
+  gcc_assert (line_width >= (start.column - 1 + literal_length));
+  from.len = literal_length;
   from.text = (const unsigned char *)literal;
 
   if (!cpp_interpret_string_ranges (pfile, &from, &src_range.m_start, 1,
@@ -1476,7 +1478,13 @@ assert_char_at_range (const location &loc,
   source_range actual_range;
   bool result = get_source_range_for_substring (pfile, strloc, idx, idx,
 						&actual_range);
-  ASSERT_TRUE (result);
+  if (should_have_column_data_p (strloc))
+    ASSERT_TRUE (result);
+  else
+    {
+      ASSERT_FALSE (result);
+      return;
+    }
 
   if (should_have_column_data_p (actual_range.m_start))
     {
@@ -1498,6 +1506,33 @@ assert_char_at_range (const location &loc,
   assert_char_at_range (SELFTEST_LOCATION, (CPP_READER), (STRLOC), (IDX),	\
 			(EXPECTED_START_COL), (EXPECTED_FINISH_COL))
 
+/* FIXME.  */
+
+static void
+assert_num_substring_ranges (const location &loc,
+			     cpp_reader *pfile,
+			     location_t strloc,
+			     int expected_num_ranges)
+{
+  int actual_num_ranges;
+  bool result
+    = get_num_source_ranges_for_substring (pfile, strloc, &actual_num_ranges);
+  if (should_have_column_data_p (strloc))
+    ASSERT_TRUE (result);
+  else
+    {
+      ASSERT_FALSE (result);
+      return;
+    }
+  ASSERT_EQ_AT (loc, expected_num_ranges, actual_num_ranges);
+}
+
+/* FIXME.  */
+
+#define ASSERT_NUM_SUBSTRING_RANGES(CPP_READER, STRLOC, EXPECTED_NUM_RANGES) \
+    assert_num_substring_ranges (SELFTEST_LOCATION, (CPP_READER), (STRLOC), \
+				 (EXPECTED_NUM_RANGES))
+
 #if 0
 
 /* Verify a complicated example of locations within a string literal.
@@ -1871,9 +1906,11 @@ static void
 test_lexer_string_locations_simple (const line_table_case &case_)
 {
   /* Digits 0-9 (with 0 at column 10), the simple way.
-     ....................000000000.11111111112.
-     ....................123456789.01234567890.  */
-  const char *content = "        \"0123456789\"\n";
+     ....................000000000.11111111112.2222222223333333333
+     ....................123456789.01234567890.1234567890123456789
+     We add a trailing comment to ensure that we correctly locate
+     the end of the string literal token.  */
+  const char *content = "        \"0123456789\" /* not a string */\n";
   lexer_test test (case_, content);
 
   /* Verify that we get the expected token back, with the correct
@@ -1893,6 +1930,7 @@ test_lexer_string_locations_simple (const line_table_case &case_)
   for (int i = 0; i < 9; i++)
     ASSERT_CHAR_AT_RANGE (test.m_parser, tok->src_loc, i, 10 + i, 10 + i);
 
+  ASSERT_NUM_SUBSTRING_RANGES (test.m_parser, tok->src_loc, 10);
 }
 
 /* Lex a string literal containing a hex-escaped character.
@@ -1929,9 +1967,9 @@ test_lexer_string_locations_hex (const line_table_case &case_)
   ASSERT_CHAR_AT_RANGE (test.m_parser, tok->src_loc, 5, 15, 18);
   for (int i = 6; i <= 9; i++)
     ASSERT_CHAR_AT_RANGE (test.m_parser, tok->src_loc, i, 13 + i, 13 + i);
-}
 
-#if 1
+  ASSERT_NUM_SUBSTRING_RANGES (test.m_parser, tok->src_loc, 10);
+}
 
 /* Lex a string literal containing an octal-escaped character.
    Verify the substring location data after running cpp_interpret_string
@@ -1943,9 +1981,9 @@ test_lexer_string_locations_oct (const line_table_case &case_)
   /* Digits 0-9, expressing digit 5 in ASCII as "\065"
      and with a space in place of digit 6, to terminate the escaped
      octal code.
-     ....................000000000.111111.11112222.
-     ....................123456789.012345.67890123.  */
-  const char *content = "        \"01234\\065 789\"\n";
+     ....................000000000.111111.11112222.2222223333333333444
+     ....................123456789.012345.67890123.4567890123456789012  */
+  const char *content = "        \"01234\\065 789\" /* not a string */\n";
   lexer_test test (case_, content);
 
   /* Verify that we get the expected token back, with the correct
@@ -1954,18 +1992,6 @@ test_lexer_string_locations_oct (const line_table_case &case_)
   ASSERT_EQ (tok->type, CPP_STRING);
   ASSERT_TOKEN_AS_TEXT_EQ (test.m_parser, tok, "\"01234\\065 789\"");
 
-#if 0
-  /* Use cpp_interpret_string to extract the quoted characters
-     into "dst_string".  */
-  cpp_string dst_string;
-  bool result = cpp_interpret_string (test.m_parser, &tok->val.str, 1,
-				      &dst_string, CPP_STRING);
-  ASSERT_TRUE (result);
-
-  /* Verify dst_string.  */
-  const cpp_string_location *strloc = &dst_string.loc;
-#endif
-
   /* Verify ranges of individual characters.  This no longer includes the
      quotes.  */
   for (int i = 0; i < 5; i++)
@@ -1973,6 +1999,8 @@ test_lexer_string_locations_oct (const line_table_case &case_)
   ASSERT_CHAR_AT_RANGE (test.m_parser, tok->src_loc, 5, 15, 18);
   for (int i = 6; i <= 9; i++)
     ASSERT_CHAR_AT_RANGE (test.m_parser, tok->src_loc, i, 13 + i, 13 + i);
+
+  ASSERT_NUM_SUBSTRING_RANGES (test.m_parser, tok->src_loc, 10);
 }
 
 /* Lex a string literal containing UCN 4 characters.
@@ -1984,9 +2012,9 @@ test_lexer_string_locations_ucn4 (const line_table_case &case_)
 {
   /* Digits 0-9, expressing digits 5 and 6 as Roman numerals expressed
      as UCN 4.
-     ....................000000000.111111.111122.222222223.
-     ....................123456789.012345.678901.234567890.  */
-  const char *content = "        \"01234\\u2174\\u2175789\"\n";
+     ....................000000000.111111.111122.222222223.33333333344444
+     ....................123456789.012345.678901.234567890.12345678901234  */
+  const char *content = "        \"01234\\u2174\\u2175789\" /* non-str */\n";
   lexer_test test (case_, content);
 
   /* Verify that we get the expected token back, with the correct
@@ -2030,6 +2058,8 @@ test_lexer_string_locations_ucn4 (const line_table_case &case_)
   /* '789'.  */
   for (int i = 11; i <= 13; i++)
     ASSERT_CHAR_AT_RANGE (test.m_parser, tok->src_loc, i, 16 + i, 16 + i);
+
+  ASSERT_NUM_SUBSTRING_RANGES (test.m_parser, tok->src_loc, 14);
 }
 
 /* Lex a string literal containing UCN 8 characters.
@@ -2040,9 +2070,9 @@ static void
 test_lexer_string_locations_ucn8 (const line_table_case &case_)
 {
   /* Digits 0-9, expressing digits 5 and 6 as Roman numerals as UCN 8.
-     ....................000000000.111111.1111222222.22223333333333.
-     ....................123456789.012345.6789012345.67890123456789.  */
-  const char *content = "        \"01234\\U00002174\\U00002175789\"\n";
+     ....................000000000.111111.1111222222.2222333333333.344444
+     ....................123456789.012345.6789012345.6789012345678.901234  */
+  const char *content = "        \"01234\\U00002174\\U00002175789\" /* */\n";
   lexer_test test (case_, content);
 
   /* Verify that we get the expected token back, with the correct
@@ -2070,6 +2100,8 @@ test_lexer_string_locations_ucn8 (const line_table_case &case_)
   /* '789' at columns 35-37  */
   for (int i = 11; i <= 13; i++)
     ASSERT_CHAR_AT_RANGE (test.m_parser, tok->src_loc, i, 24 + i, 24 + i);
+
+  ASSERT_NUM_SUBSTRING_RANGES (test.m_parser, tok->src_loc, 14);
 }
 
 /* Lex a u8-string literal.
@@ -2080,9 +2112,9 @@ static void
 test_lexer_string_locations_u8 (const line_table_case &case_)
 {
   /* Digits 0-9.
-     ....................000000000.11111111112.
-     ....................123456789.01234567890.  */
-  const char *content = "      u8\"0123456789\"\n";
+     ....................000000000.11111111112.22222222233333
+     ....................123456789.01234567890.12345678901234  */
+  const char *content = "      u8\"0123456789\" /* non-str */\n";
   lexer_test test (case_, content);
 
   /* Verify that we get the expected token back, with the correct
@@ -2096,7 +2128,78 @@ test_lexer_string_locations_u8 (const line_table_case &case_)
   for (int i = 0; i < 9; i++)
     ASSERT_CHAR_AT_RANGE (test.m_parser, tok->src_loc, i, 10 + i, 10 + i);
 }
-#endif
+
+/* Lex a string literal containing UTF-8 source characters.
+   Verify the substring location data after running cpp_interpret_string
+   on it.  */
+
+static void
+test_lexer_string_locations_utf8_source (const line_table_case &case_)
+{
+
+ /* This string literal is written out to the source file as UTF-8,
+    and is of the form "before mojibake after", where "mojibake"
+    is written as the following four unicode code points:
+       U+6587 CJK UNIFIED IDEOGRAPH-6587
+       U+5B57 CJK UNIFIED IDEOGRAPH-5B57
+       U+5316 CJK UNIFIED IDEOGRAPH-5316
+       U+3051 HIRAGANA LETTER KE.
+     Each of these is 3 bytes wide when encoded in UTF-8, whereas the
+     "before" and "after" are 1 byte per unicode character.
+
+     The numbering shown are "columns", which are *byte* numbers within
+     the line, rather than unicode character numbers.
+
+     .................... 000000000.1111111.
+     .................... 123456789.0123456.  */
+  const char *content = ("        \"before "
+			 /* U+6587 CJK UNIFIED IDEOGRAPH-6587
+			      UTF-8: 0xE6 0x96 0x87
+			      C octal escaped UTF-8: \346\226\207
+			    "column" numbers: 17-19.  */
+			 "\346\226\207"
+
+			 /* U+5B57 CJK UNIFIED IDEOGRAPH-5B57
+			      UTF-8: 0xE5 0xAD 0x97
+			      C octal escaped UTF-8: \345\255\227
+			    "column" numbers: 20-22.  */
+			 "\345\255\227"
+
+			 /* U+5316 CJK UNIFIED IDEOGRAPH-5316
+			      UTF-8: 0xE5 0x8C 0x96
+			      C octal escaped UTF-8: \345\214\226
+			    "column" numbers: 23-25.  */
+			 "\345\214\226"
+
+			 /* U+3051 HIRAGANA LETTER KE
+			      UTF-8: 0xE3 0x81 0x91
+			      C octal escaped UTF-8: \343\201\221
+			    "column" numbers: 26-28.  */
+			 "\343\201\221"
+
+			 /* column numbers 29 onwards
+			  2333333.33334444444444
+			  9012345.67890123456789. */
+			 " after\" /* non-str */\n");
+  lexer_test test (case_, content);
+
+  /* Verify that we get the expected token back, with the correct
+     location information.  */
+  const cpp_token *tok = test.get_token ();
+  ASSERT_EQ (tok->type, CPP_STRING);
+  ASSERT_TOKEN_AS_TEXT_EQ
+    (test.m_parser, tok,
+     "\"before \346\226\207\345\255\227\345\214\226\343\201\221 after\"");
+
+  /* Verify ranges of individual characters.  This no longer includes the
+     quotes.
+     Assuming that both source and execution encodings are UTF-8, we have
+     a run of 25 octets in each.  */
+  for (int i = 0; i < 25; i++)
+    ASSERT_CHAR_AT_RANGE (test.m_parser, tok->src_loc, i, 10 + i, 10 + i);
+
+  ASSERT_NUM_SUBSTRING_RANGES (test.m_parser, tok->src_loc, 25);
+}
 
 /* A table of interesting location_t values, giving one axis of our test
    matrix.  */
@@ -2133,10 +2236,6 @@ input_c_tests ()
   test_unknown_location ();
   test_builtins ();
 
-#if 0
-  test_location_column_from_byte_offset ();
-#endif
-
   /* As noted above in the description of struct line_table_case,
      we want to explore a test matrix of interesting line_table
      situations, running various selftests for each case within the
@@ -2159,17 +2258,14 @@ input_c_tests ()
 
 	  /* Run all tests for the given case within the test matrix.  */
 	  test_accessing_ordinary_linemaps (c);
-#if 0
-	  test_complex_substring_locations (c);
-#endif
 	  test_lexer (c);
-
 	  test_lexer_string_locations_simple (c);
 	  test_lexer_string_locations_hex (c);
 	  test_lexer_string_locations_oct (c);
 	  test_lexer_string_locations_ucn4 (c);
 	  test_lexer_string_locations_ucn8 (c);
 	  test_lexer_string_locations_u8 (c);
+	  test_lexer_string_locations_utf8_source (c);
 	  num_cases_tested++;
 	}
     }
-- 
1.8.5.3

