From 21c19906f4e9b7c36010efd14751ed23ff16a04b Mon Sep 17 00:00:00 2001
From: David Malcolm <dmalcolm@redhat.com>
Date: Thu, 7 Jul 2016 03:50:10 -0400
Subject: [PATCH 10/25] FIXME: enough WIP on on-demand substring locations to
 get it fully passing

---
 gcc/c-family/c-lex.c |   2 +-
 gcc/input.c          | 344 ++++++++++++++++++++++-----------------------------
 2 files changed, 149 insertions(+), 197 deletions(-)

diff --git a/gcc/c-family/c-lex.c b/gcc/c-family/c-lex.c
index c18e8ee..5806bc9 100644
--- a/gcc/c-family/c-lex.c
+++ b/gcc/c-family/c-lex.c
@@ -1189,7 +1189,7 @@ lex_string (const cpp_token *tok, tree *valp, bool objc_string, bool translate)
        ? cpp_interpret_string : cpp_interpret_string_notranslate)
       (parse_in, strs, concats + 1, &istr, type))
     {
-      value = build_string (istr.len, (const char *) istr.text);//, &istr.loc);
+      value = build_string (istr.len, (const char *) istr.text);
       free (CONST_CAST (unsigned char *, istr.text));
 
       // FIXME: what about the notranslate case?
diff --git a/gcc/input.c b/gcc/input.c
index ed28ae1..73a3804 100644
--- a/gcc/input.c
+++ b/gcc/input.c
@@ -1188,7 +1188,12 @@ string_concat_db::get_string_concatenation (location_t loc,
 location_t
 string_concat_db::get_key_loc (location_t loc)
 {
-  return get_range_from_loc (line_table, loc).m_start;
+  loc = linemap_resolve_location (line_table, loc, LRK_SPELLING_LOCATION,
+				  NULL);
+
+  loc = get_range_from_loc (line_table, loc).m_start;
+
+  return loc;
 }
 
 
@@ -1205,8 +1210,9 @@ get_substring_ranges_for_loc (cpp_reader *pfile,
   if (strloc == UNKNOWN_LOCATION)
     return false;
 
-  /* FIXME: handle concatenated strings by doing this repeatedly, somehow.
-     We'd need to capture concatenations when they happen.  */
+  /* If string concatenation has occurred at STRLOC, get the locations
+     of all of the literal tokens making up the compound string.
+     Otherwise, just use STRLOC.  */
   int num_locs = 1;
   location_t *strlocs = &strloc;
   if (concats)
@@ -1215,10 +1221,9 @@ get_substring_ranges_for_loc (cpp_reader *pfile,
   auto_vec <cpp_string> strs (num_locs);
   for (int i = 0; i < num_locs; i++)
     {
-
       /* Get range of strloc.  We will use it to locate the start and finish
 	 of the literal token within the line.  */
-      source_range src_range = get_range_from_loc (line_table, strloc);
+      source_range src_range = get_range_from_loc (line_table, strlocs[i]);
 
       if (src_range.m_start >= LINE_MAP_MAX_LOCATION_WITH_COLS)
 	/* If so, we can't reliably determine where the token started within
@@ -1534,187 +1539,6 @@ test_builtins ()
   ASSERT_PRED1 (is_location_from_builtin_token, BUILTINS_LOCATION);
 }
 
-#if 0
-
-/* Verify a complicated example of locations within a string literal.
-
-   This uses the example from cpplib.h, with the following column
-   numbering:
-000000000111111111122222222223333333333
-123456789012345678901234567890123456789
-    printf ("foo \x25\151 bar"  "baz\n",
-    .........^^^^.....................: fragment 0: 4 chars at 1 col per char
-    .............^^^^^^^^.............: fragment 1: 2 chars at 4 cols per char
-    .....................^^^^.........: fragment 2: 4 chars at 1 col per char
-    .............................^^^..: fragment 3: 3 chars at 1 col per char
-    ................................^^: fragment 4: 1 char at 2 cols per char
-
-    We populate a cpp_string_location with the above data, then
-    verify reading the data back.  */
-
-/* A helper function for verifying the above data.  */
-
-static void
-assert_complex_substring_locations (const cpp_string_location *strloc,
-				    int default_range_bits)
-{
-  /* Verify that we can correctly retrieve the range for each
-     character.  */
-
-  /* "foo ".  */
-  ASSERT_CHAR_AT_RANGE (0, 14, 14);
-  ASSERT_CHAR_AT_RANGE (1, 15, 15);
-  ASSERT_CHAR_AT_RANGE (2, 16, 16);
-  ASSERT_CHAR_AT_RANGE (3, 17, 17);
-
-  /* "\x25\151".  */
-  ASSERT_CHAR_AT_RANGE (4, 18, 21);
-  ASSERT_CHAR_AT_RANGE (5, 22, 25);
-
-  /* " bar".  */
-  ASSERT_CHAR_AT_RANGE (6, 26, 26);
-  ASSERT_CHAR_AT_RANGE (7, 27, 27);
-  ASSERT_CHAR_AT_RANGE (8, 28, 28);
-  ASSERT_CHAR_AT_RANGE (9, 29, 29);
-
-  /* "baz".  */
-  ASSERT_CHAR_AT_RANGE (10, 34, 34);
-  ASSERT_CHAR_AT_RANGE (11, 35, 35);
-  ASSERT_CHAR_AT_RANGE (12, 36, 36);
-
-  /* "\n".  */
-  ASSERT_CHAR_AT_RANGE (13, 37, 38);
-
-  /* Verify that the data was packed as expected.  */
-
-  if (string_packing_verifiable_p ())
-    {
-      ASSERT_FALSE (strloc->trivial_p ());
-      ASSERT_EQ (strloc->m_num_fraglocs, 5);
-    }
-
-  /* If any of the data may use a non default value for range_bits,
-     don't attempt to verify it.  */
-  if (!range_bits_verifiable_p ())
-    return;
-
-  /* "foo ".  */
-  ASSERT_STRING_LOCATION_FRAGMENT
-    (strloc, 0,
-     linemap_position_for_column (line_table, 14),
-     4, 1, default_range_bits, 1);
-
-  /* "\x25\151".  */
-  ASSERT_STRING_LOCATION_FRAGMENT
-    (strloc, 1,
-     linemap_position_for_column (line_table, 18),
-     2, 4, default_range_bits, 1);
-
-  /* " bar".  */
-  ASSERT_STRING_LOCATION_FRAGMENT
-    (strloc, 2,
-     linemap_position_for_column (line_table, 26),
-     4, 1, default_range_bits, 1);
-
-  /* "baz".  */
-  ASSERT_STRING_LOCATION_FRAGMENT
-    (strloc, 3,
-     linemap_position_for_column (line_table, 34),
-     3, 1, default_range_bits, 1);
-
-  /* "\n".  */
-  ASSERT_STRING_LOCATION_FRAGMENT
-    (strloc, 4,
-     linemap_position_for_column (line_table, 37),
-     1, 2, default_range_bits, 1);
-}
-
-/* Helper function for test_complex_substring_locations.
-   Add a character to STRLOC on the current line, occupying the
-   columns ranging from START_COL to END_COL.  */
-
-static void
-add_char_at (cpp_string_location *strloc, int start_col, int end_col)
-{
-  source_range range;
-  range.m_start = linemap_position_for_column (line_table, start_col);
-  range.m_finish = linemap_position_for_column (line_table, end_col);
-  strloc->add_char_at (range, 1, line_table);
-}
-
-/* Populate a cpp_string_location with the above data, then verify
-   reading the data back.
-   Then use the cpp_string_location_reader class to copy the data,
-   and verify that the copy is correct.  */
-
-static void
-test_complex_substring_locations (const line_table_case &case_)
-{
-  temp_line_table tmp_lt (case_);
-
-  /* Populate strloc as per the example above.  */
-  cpp_string_location strloc;
-  strloc.init ();
-
-  /* Create a tempfile containing the content we're describing.
-     Apart from providing a filename, this isn't directly used by the test,
-     but is useful when debugging locations via "(gdb) call inform" and the
-     like.  */
-  temp_source_file tmp (SELFTEST_LOCATION, ".c",
-			"    printf (\"foo \\x25\\151 bar\"  \"baz\",");
-
-  linemap_add (line_table, LC_ENTER, false, tmp.get_filename (), 0);
-  linemap_line_start (line_table, 1, 100);
-
-  /* "foo ".  */
-  for (int col = 14; col <= 17; col++)
-    add_char_at (&strloc, col, col);
-  /* "\x25\151".  */
-  add_char_at (&strloc, 18, 21);
-  add_char_at (&strloc, 22, 25);
-  /* " bar".  */
-  for (int col = 26; col <= 29; col++)
-    add_char_at (&strloc, col, col);
-  /* "baz".  */
-  for (int col = 34; col <= 36; col++)
-    add_char_at (&strloc, col, col);
-  /* "\n".  */
-  add_char_at (&strloc, 37, 38);
-
-  /* Ensure that the line table contains all of the locations within the
-     string.  Specifically, ensure that they are all
-       <= line_table->highest_location, so that
-     linemap_location_from_macro_expansion_p returns false.  */
-  linemap_position_for_column (line_table, 38);
-
-  /* Verify that we can read the correct data back.  */
-  assert_complex_substring_locations (&strloc, case_.m_default_range_bits);
-
-  /* Use the cpp_string_location_reader class to copy the data.  */
-  cpp_string_location dst_strloc;
-  dst_strloc.init ();
-  for (unsigned i = 0; i < strloc.m_num_fraglocs; i++)
-    {
-      const cpp_string_fragment_location &src_fragment
-	= strloc.m_fragloc_array[i];
-      cpp_string_location_reader reader (src_fragment.m_loc,
-					 src_fragment.m_cols_per_item,
-					 src_fragment.m_bits_per_col,
-					 line_table);
-      for (int j = 0; j < src_fragment.m_len; j++)
-	{
-	  source_range range = reader.get_next ();
-	  dst_strloc.add_char_at (range, 1, line_table);
-	}
-    }
-
-  /* Verify that we can read the correct data back from the copy.  */
-  assert_complex_substring_locations (&dst_strloc, case_.m_default_range_bits);
-
-  linemap_add (line_table, LC_LEAVE, false, NULL, 0);
-}
-#endif
-
 /* TODO: a testcase involving more than one linemap.  */
 
 /* Verify reading of input files (e.g. for caret-based diagnostics).  */
@@ -2282,7 +2106,7 @@ test_lexer_string_locations_utf8_source (const line_table_case &case_)
 /* Test of string literal concatenation.  */
 
 static void
-test_lexer_string_locations_concatenation (const line_table_case &case_)
+test_lexer_string_locations_concatenation_1 (const line_table_case &case_)
 {
   /* Digits 0-9.
      .....................000000000.111111.11112222222222
@@ -2291,7 +2115,6 @@ test_lexer_string_locations_concatenation (const line_table_case &case_)
 			 "        \"56789\" /* non-str */\n");
   lexer_test test (case_, content);
 
-  //cpp_string input_strs[2];
   location_t input_locs[2];
 
   /* Verify that we get the expected tokens back.  */
@@ -2300,7 +2123,6 @@ test_lexer_string_locations_concatenation (const line_table_case &case_)
   ASSERT_TOKEN_AS_TEXT_EQ
     (test.m_parser, tok_a,
      "\"01234\"");
-  location_t initial_loc = tok_a->src_loc;
   input_locs[0] = tok_a->src_loc;
 
   const cpp_token *tok_b = test.get_token ();
@@ -2312,12 +2134,8 @@ test_lexer_string_locations_concatenation (const line_table_case &case_)
 
   /* Simulate c-lex.c's lex_string in order to record concatenation.  */
   test.m_concats.record_string_concatenation (2, input_locs);
-#if 0
-  m_parser-
-  cpp_string output_str;
-  cpp_interpret_string (test.m_parser, input_strs, 2, &output_str, CPP_STRING);
-  ASSERT_STREQ ((const char *)output_str.text, "0123456789");
-#endif
+
+  location_t initial_loc = input_locs[0];
 
   for (int i = 0; i < 5; i++)
     ASSERT_CHAR_AT_RANGE (test, initial_loc, i, 10 + i, 10 + i);
@@ -2327,6 +2145,137 @@ test_lexer_string_locations_concatenation (const line_table_case &case_)
   ASSERT_NUM_SUBSTRING_RANGES (test, initial_loc, 10);
 }
 
+/* Another test of string literal concatenation.  */
+
+static void
+test_lexer_string_locations_concatenation_2 (const line_table_case &case_)
+{
+  /* Digits 0-9.
+     .....................000000000.111.11111112222222
+     .....................123456789.012.34567890123456.  */
+  const char *content = ("        \"01\" /* non-str */\n"
+			 "        \"23\" /* non-str */\n"
+			 "        \"45\" /* non-str */\n"
+			 "        \"67\" /* non-str */\n"
+			 "        \"89\" /* non-str */\n");
+  lexer_test test (case_, content);
+
+  location_t input_locs[5];
+
+  /* Verify that we get the expected tokens back.  */
+  for (int i = 0; i < 5; i++)
+    {
+      const cpp_token *tok = test.get_token ();
+      ASSERT_EQ (tok->type, CPP_STRING);
+      input_locs[i] = tok->src_loc;
+    }
+
+  /* Simulate c-lex.c's lex_string in order to record concatenation.  */
+  test.m_concats.record_string_concatenation (5, input_locs);
+
+  location_t initial_loc = input_locs[0];
+
+  /* Within ASSERT_CHAR_AT_RANGE (actually assert_char_at_range), we can
+     detect if the initial loc is after LINE_MAP_MAX_LOCATION_WITH_COLS
+     and expect get_source_range_for_substring to fail.
+     However, for a string concatenation test, we can have a case
+     where the initial string is fully before LINE_MAP_MAX_LOCATION_WITH_COLS,
+     but subsequent strings can be after it.
+     Attempting to detect this within assert_char_at_range
+     would overcomplicate the logic for the common test cases, so
+     we detect it here.  */
+  if (should_have_column_data_p (input_locs[0])
+      && !should_have_column_data_p (input_locs[4]))
+    {
+      /* Verify that get_source_range_for_substring gracefully rejects
+	 this case.  */
+      source_range actual_range;
+      bool result
+	= get_source_range_for_substring (test.m_parser, &test.m_concats,
+					  initial_loc, 0, 0, &actual_range);
+      ASSERT_FALSE (result);
+      return;
+    }
+
+  for (int i = 0; i < 5; i++)
+    for (int j = 0; j < 2; j++)
+      ASSERT_CHAR_AT_RANGE (test, initial_loc, (i * 2) + j, 10 + j, 10 + j);
+
+  ASSERT_NUM_SUBSTRING_RANGES (test, initial_loc, 10);
+}
+
+/* Another test of string literal concatenation, this time combined with
+   various kinds of escaped characters.  */
+
+static void
+test_lexer_string_locations_concatenation_3 (const line_table_case &case_)
+{
+  /* Digits 0-9, expressing digit 5 in ASCII as hex "\x35"
+     digit 6 in ASCII as octal "\066", concatenating multiple strings.  */
+  const char *content
+    /* .000000000.111111.111.1.2222.222.2.2233.333.3333.34444444444555
+       .123456789.012345.678.9.0123.456.7.8901.234.5678.90123456789012. */
+    = ("        \"01234\"  \"\\x35\"  \"\\066\"  \"789\" /* non-str */\n");
+  lexer_test test (case_, content);
+  location_t input_locs[4];
+
+  /* Verify that we get the expected tokens back.  */
+  for (int i = 0; i < 4; i++)
+    {
+      const cpp_token *tok = test.get_token ();
+      ASSERT_EQ (tok->type, CPP_STRING);
+      input_locs[i] = tok->src_loc;
+    }
+
+  /* Simulate c-lex.c's lex_string in order to record concatenation.  */
+  test.m_concats.record_string_concatenation (4, input_locs);
+
+  location_t initial_loc = input_locs[0];
+
+  for (int i = 0; i < 4; i++)
+    ASSERT_CHAR_AT_RANGE (test, initial_loc, i, 10 + i, 10 + i);
+  ASSERT_CHAR_AT_RANGE (test, initial_loc, 5, 19, 22);
+  ASSERT_CHAR_AT_RANGE (test, initial_loc, 6, 27, 30);
+  for (int i = 7; i <= 9; i++)
+    ASSERT_CHAR_AT_RANGE (test, initial_loc, i, 28 + i, 28 + i);
+
+  ASSERT_NUM_SUBSTRING_RANGES (test, initial_loc, 10);
+}
+
+/* Test of string literal in a macro.  */
+
+static void
+test_lexer_string_locations_macro (const line_table_case &case_)
+{
+  /* Digits 0-9.
+     .....................0000000001111111111.22222222223.
+     .....................1234567890123456789.01234567890.  */
+  const char *content = ("#define MACRO     \"0123456789\" /* non-str */\n"
+			 "  MACRO");
+  lexer_test test (case_, content);
+
+  /* Verify that we get the expected tokens back.  */
+  const cpp_token *tok = test.get_token ();
+  ASSERT_EQ (tok->type, CPP_PADDING);
+
+  tok = test.get_token ();
+  ASSERT_EQ (tok->type, CPP_STRING);
+  ASSERT_TOKEN_AS_TEXT_EQ
+    (test.m_parser, tok,
+     "\"0123456789\"");
+
+  /* Verify ranges of individual characters.  We ought to
+     see columns within the macro definition.  */
+  for (int i = 0; i < 9; i++)
+    ASSERT_CHAR_AT_RANGE (test, tok->src_loc, i,
+			  20 + i, 20 + i);
+
+  ASSERT_NUM_SUBSTRING_RANGES (test, tok->src_loc, 10);
+
+  tok = test.get_token ();
+  ASSERT_EQ (tok->type, CPP_PADDING);
+}
+
 /* A table of interesting location_t values, giving one axis of our test
    matrix.  */
 
@@ -2392,7 +2341,10 @@ input_c_tests ()
 	  test_lexer_string_locations_ucn8 (c);
 	  test_lexer_string_locations_u8 (c);
 	  test_lexer_string_locations_utf8_source (c);
-	  test_lexer_string_locations_concatenation (c);
+	  test_lexer_string_locations_concatenation_1 (c);
+	  test_lexer_string_locations_concatenation_2 (c);
+	  test_lexer_string_locations_concatenation_3 (c);
+	  test_lexer_string_locations_macro (c);
 	  num_cases_tested++;
 	}
     }
-- 
1.8.5.3

