From e6cc604ea5eb6ed7ba30c9d7f983505856aaa242 Mon Sep 17 00:00:00 2001
From: David Malcolm <dmalcolm@redhat.com>
Date: Wed, 22 Jun 2016 14:51:29 -0400
Subject: [PATCH 08/31] FIXME: more selftests for string location lexing

---
 gcc/input.c             | 495 +++++++++++++++++++++++++++++++++++++++++++++---
 libcpp/charset.c        |   3 +
 libcpp/include/cpplib.h |   5 +-
 3 files changed, 480 insertions(+), 23 deletions(-)

diff --git a/gcc/input.c b/gcc/input.c
index c36fcbd..d94d681 100644
--- a/gcc/input.c
+++ b/gcc/input.c
@@ -1356,11 +1356,30 @@ test_builtins ()
   ASSERT_PRED1 (is_location_from_builtin_token, BUILTINS_LOCATION);
 }
 
+/* FIXME. */
+static bool
+string_packing_verifiable_p ()
+{
+  return line_table->highest_location <= LINE_MAP_MAX_LOCATION_WITH_COLS;
+}
+
+/* FIXME: If any of the data may use a non default value for range_bits,
+   don't attempt to verify it.  */
+
+static bool
+range_bits_verifiable_p ()
+{
+  return (line_table->highest_location
+	  <= LINE_MAP_MAX_LOCATION_WITH_PACKED_RANGES);
+}
+
 /* Verify that locations within string literals are correctly handled.  */
 
 /* Verify STRLOC's description of locations within a string literal.
    Assert that the character at index IDX begins at column EXPECTED_START_COL
-   and ends at EXPECTED_FINISH_COL.  */
+   and ends at EXPECTED_FINISH_COL, unless the locations are beyond
+   LINE_MAP_MAX_LOCATION_WITH_COLS, in which case don't check their
+   columns.  */
 
 static void
 assert_char_at_range (const location &loc,
@@ -1380,6 +1399,13 @@ assert_char_at_range (const location &loc,
     }
 }
 
+/* Macro for calling assert_char_at_range, supplying SELFTEST_LOCATION for
+   the effective location of any errors.  */
+
+#define ASSERT_CHAR_AT_RANGE(IDX, EXPECTED_START_COL, EXPECTED_FINISH_COL) \
+  assert_char_at_range (SELFTEST_LOCATION, strloc, (IDX),		\
+			(EXPECTED_START_COL), (EXPECTED_FINISH_COL))
+
 /* Locate fragment IDX within STRLOC and assert that its various fields
    equal the provided values.  */
 
@@ -1398,7 +1424,7 @@ assert_string_location_fragment (const location &loc,
 }
 
 /* Macro for calling assert_string_location_fragment, supplying
-   SELFTEST_LOCATION.  */
+   SELFTEST_LOCATION for the effective location of any errors.  */
 
 #define ASSERT_STRING_LOCATION_FRAGMENT(STRLOC, IDX, EXPECTED_START_LOC, \
 					EXPECTED_LEN, EXPECTED_COLS_PER_CHAR, \
@@ -1445,12 +1471,14 @@ test_simple_substring_locations (const line_table_case &case_)
     }
 
   /* Verify that the data was packed as expected.  */
-  ASSERT_TRUE (strloc.trivial_p ());
-  ASSERT_EQ (strloc.m_num_fraglocs, 1);
-  if (line_table->highest_location
-      <= LINE_MAP_MAX_LOCATION_WITH_PACKED_RANGES)
-    ASSERT_STRING_LOCATION_FRAGMENT (&strloc, 0, loc, len, 1,
-				     line_table->default_range_bits);
+  if (string_packing_verifiable_p ())
+    {
+      ASSERT_TRUE (strloc.trivial_p ());
+      ASSERT_EQ (strloc.m_num_fraglocs, 1);
+      if (range_bits_verifiable_p ())
+	ASSERT_STRING_LOCATION_FRAGMENT (&strloc, 0, loc, len, 1,
+					 line_table->default_range_bits);
+    }
 
   linemap_add (line_table, LC_LEAVE, false, NULL, 0);
 }
@@ -1479,9 +1507,6 @@ assert_complex_substring_locations (const cpp_string_location *strloc,
 {
   /* Verify that we can correctly retrieve the range for each
      character.  */
-#define ASSERT_CHAR_AT_RANGE(IDX, EXPECTED_START_COL, EXPECTED_FINISH_COL) \
-  assert_char_at_range (SELFTEST_LOCATION, strloc, (IDX),		\
-			(EXPECTED_START_COL), (EXPECTED_FINISH_COL))
 
   /* "foo ".  */
   ASSERT_CHAR_AT_RANGE (0, 14, 14);
@@ -1507,22 +1532,17 @@ assert_complex_substring_locations (const cpp_string_location *strloc,
   /* "\n".  */
   ASSERT_CHAR_AT_RANGE (13, 37, 38);
 
-#undef ASSERT_CHAR_AT_RANGE
-
   /* Verify that the data was packed as expected.  */
 
-  ASSERT_FALSE (strloc->trivial_p ());
-
-  /* If any of the data might be in no-columns mode, don't
-     attempt to predict the packing.  */
-  if (line_table->highest_location > LINE_MAP_MAX_LOCATION_WITH_COLS)
-    return;
-
-  ASSERT_EQ (strloc->m_num_fraglocs, 5);
+  if (string_packing_verifiable_p ())
+    {
+      ASSERT_FALSE (strloc->trivial_p ());
+      ASSERT_EQ (strloc->m_num_fraglocs, 5);
+    }
 
   /* If any of the data may use a non default value for range_bits,
      don't attempt to verify it.  */
-  if (line_table->highest_location > LINE_MAP_MAX_LOCATION_WITH_PACKED_RANGES)
+  if (!range_bits_verifiable_p ())
     return;
 
   /* "foo ".  */
@@ -1767,6 +1787,431 @@ test_lexer (const line_table_case &case_)
   cpp_finish (parser, NULL);
 }
 
+/* FIXME.  */
+
+struct lexer_test
+{
+  lexer_test (const line_table_case &case_, const char *content);
+  ~lexer_test ();
+
+  const cpp_token *get_token ();
+
+  temp_source_file m_tempfile;
+  temp_line_table m_tmp_lt;
+  cpp_reader *m_parser;
+};
+
+/* FIXME.  */
+
+lexer_test::lexer_test (const line_table_case &case_, const char *content) :
+  /* Create a tempfile and write the text to it.  */
+  m_tempfile (SELFTEST_LOCATION, ".c", content),
+  m_tmp_lt (case_),
+  m_parser (cpp_create_reader (CLK_GNUC99, NULL, line_table))
+{
+  cpp_init_iconv (m_parser);
+
+  /* Parse the file.  */
+  const char *fname = cpp_read_main_file (m_parser,
+					  m_tempfile.get_filename ());
+  ASSERT_NE (fname, NULL);
+}
+
+/* FIXME.  */
+
+lexer_test::~lexer_test ()
+{
+  location_t loc;
+  const cpp_token *tok;
+
+  tok = cpp_get_token_with_location (m_parser, &loc);
+  ASSERT_NE (tok, NULL);
+  ASSERT_EQ (tok->type, CPP_EOF);
+
+  cpp_finish (m_parser, NULL);
+}
+
+/* FIXME.  */
+
+const cpp_token *
+lexer_test::get_token ()
+{
+  location_t loc;
+  const cpp_token *tok;
+
+  tok = cpp_get_token_with_location (m_parser, &loc);
+  ASSERT_NE (tok, NULL);
+  return tok;
+}
+
+/* FIXME.  */
+static void
+test_lexer_string_locations_simple (const line_table_case &case_)
+{
+  /* Digits 0-9 (with 0 at column 10), the simple way.
+     ....................000000000.11111111112.
+     ....................123456789.01234567890.  */
+  const char *content = "        \"0123456789\"\n";
+  lexer_test test (case_, content);
+
+  /* Verify that we get the expected token back, with the correct
+     location information.  */
+  const cpp_token *tok = test.get_token ();
+  ASSERT_EQ (tok->type, CPP_STRING);
+  ASSERT_TOKEN_AS_TEXT_EQ (test.m_parser, tok, "\"0123456789\"");
+  ASSERT_TOKEN_LOC_EQ (tok, test.m_tempfile.get_filename (), 1, 9, 20);
+
+  /* At this point in lexing, the quote characters are treated as part of
+     the string (they are stripped off by cpp_interpret_string).  */
+
+  ASSERT_EQ (tok->val.str.len, 12);
+
+  const cpp_string_location *strloc = &tok->val.str.loc;
+
+  /* If any of the data might be in no-columns mode, don't
+     attempt to predict the packing.  */
+  if (string_packing_verifiable_p ())
+    {
+      ASSERT_TRUE (strloc->trivial_p ());
+      ASSERT_EQ (strloc->m_num_fraglocs, 1);
+    }
+
+  /* Verify ranges of individual characters (including the quotes).  */
+  for (int i = 0; i < 12; i++)
+    ASSERT_CHAR_AT_RANGE(i, 9 + i, 9 + i);
+
+  ASSERT_EQ (strloc->m_num_fraglocs, 1);
+  if (range_bits_verifiable_p ())
+    ASSERT_STRING_LOCATION_FRAGMENT
+      (strloc, 0,
+       linemap_position_for_column (line_table, 9),
+       12, 1, case_.m_default_range_bits);
+
+  /* Use cpp_interpret_string to extract the quoted characters
+     into "dst_string".  */
+  cpp_string dst_string;
+  bool result = cpp_interpret_string (test.m_parser, &tok->val.str, 1,
+				      &dst_string, CPP_STRING);
+  ASSERT_TRUE (result);
+
+  /* Verify dst_string.  */
+  strloc = &dst_string.loc;
+
+  /* Verify ranges of individual characters.  This no longer includes the
+     quotes.  */
+  for (int i = 0; i < 9; i++)
+    ASSERT_CHAR_AT_RANGE(i, 10 + i, 10 + i);
+
+  if (string_packing_verifiable_p ())
+    {
+      /* Verify packing.  */
+      ASSERT_EQ (strloc->m_num_fraglocs, 1);
+      if (range_bits_verifiable_p ())
+	ASSERT_STRING_LOCATION_FRAGMENT
+	  (strloc, 0,
+	   linemap_position_for_column (line_table, 10),
+	   10, 1, case_.m_default_range_bits);
+    }
+}
+
+/* FIXME.  */
+static void
+test_lexer_string_locations_hex (const line_table_case &case_)
+{
+  /* Digits 0-9, expressing digit 5 in ASCII as "\x35"
+     and with a space in place of digit 6, to terminate the escaped
+     hex code.
+     ....................000000000.111111.11112222.
+     ....................123456789.012345.67890123.  */
+  const char *content = "        \"01234\\x35 789\"\n";
+  lexer_test test (case_, content);
+
+  /* Verify that we get the expected token back, with the correct
+     location information.  */
+  const cpp_token *tok = test.get_token ();
+  ASSERT_EQ (tok->type, CPP_STRING);
+  ASSERT_TOKEN_AS_TEXT_EQ (test.m_parser, tok, "\"01234\\x35 789\"");
+  ASSERT_TOKEN_LOC_EQ (tok, test.m_tempfile.get_filename (), 1, 9, 23);
+
+  /* At this point in lexing, the quote characters are treated as part of
+     the string (they are stripped off by cpp_interpret_string).  */
+
+  ASSERT_EQ (tok->val.str.len, 15);
+
+  const cpp_string_location *strloc = &tok->val.str.loc;
+
+  /* Verify ranges of individual characters (including the quotes).  */
+  for (int i = 0; i < 15; i++)
+    ASSERT_CHAR_AT_RANGE(i, 9 + i, 9 + i);
+
+  /* If any of the data might be in no-columns mode, don't
+     attempt to predict the packing.  */
+  if (string_packing_verifiable_p ())
+    {
+      ASSERT_TRUE (strloc->trivial_p ());
+      ASSERT_EQ (strloc->m_num_fraglocs, 1);
+      if (range_bits_verifiable_p ())
+	ASSERT_STRING_LOCATION_FRAGMENT
+	  (strloc, 0,
+	   linemap_position_for_column (line_table, 9),
+	   15, 1, case_.m_default_range_bits);
+    }
+
+  /* Use cpp_interpret_string to extract the quoted characters
+     into "dst_string".  */
+  cpp_string dst_string;
+  bool result = cpp_interpret_string (test.m_parser, &tok->val.str, 1,
+				      &dst_string, CPP_STRING);
+  ASSERT_TRUE (result);
+
+  /* Verify dst_string.  */
+  strloc = &dst_string.loc;
+
+  /* Verify ranges of individual characters.  This no longer includes the
+     quotes.  */
+  for (int i = 0; i < 5; i++)
+    ASSERT_CHAR_AT_RANGE(i, 10 + i, 10 + i);
+  ASSERT_CHAR_AT_RANGE(5, 15, 18);
+  for (int i = 6; i <= 9; i++)
+    ASSERT_CHAR_AT_RANGE(i, 13 + i, 13 + i);
+
+  if (string_packing_verifiable_p ())
+    {
+      /* Verify packing.  */
+      ASSERT_EQ (strloc->m_num_fraglocs, 3);
+      if (range_bits_verifiable_p ())
+	{
+	  ASSERT_STRING_LOCATION_FRAGMENT
+	    (strloc, 0,
+	     linemap_position_for_column (line_table, 10),
+	     5, 1, case_.m_default_range_bits);
+	  ASSERT_STRING_LOCATION_FRAGMENT
+	    (strloc, 1,
+	     linemap_position_for_column (line_table, 15),
+	     1, 4, case_.m_default_range_bits);
+	  ASSERT_STRING_LOCATION_FRAGMENT
+	    (strloc, 2,
+	     linemap_position_for_column (line_table, 19),
+	 4, 1, case_.m_default_range_bits);
+	}
+    }
+}
+
+/* FIXME.  */
+static void
+test_lexer_string_locations_oct (const line_table_case &case_)
+{
+  /* Digits 0-9, expressing digit 5 in ASCII as "\065"
+     and with a space in place of digit 6, to terminate the escaped
+     octal code.
+     ....................000000000.111111.11112222.
+     ....................123456789.012345.67890123.  */
+  const char *content = "        \"01234\\065 789\"\n";
+  lexer_test test (case_, content);
+
+  /* Verify that we get the expected token back, with the correct
+     location information.  */
+  const cpp_token *tok = test.get_token ();
+  ASSERT_EQ (tok->type, CPP_STRING);
+  ASSERT_TOKEN_AS_TEXT_EQ (test.m_parser, tok, "\"01234\\065 789\"");
+
+  /* Use cpp_interpret_string to extract the quoted characters
+     into "dst_string".  */
+  cpp_string dst_string;
+  bool result = cpp_interpret_string (test.m_parser, &tok->val.str, 1,
+				      &dst_string, CPP_STRING);
+  ASSERT_TRUE (result);
+
+  /* Verify dst_string.  */
+  const cpp_string_location *strloc = &dst_string.loc;
+
+  /* Verify ranges of individual characters.  This no longer includes the
+     quotes.  */
+  for (int i = 0; i < 5; i++)
+    ASSERT_CHAR_AT_RANGE(i, 10 + i, 10 + i);
+  ASSERT_CHAR_AT_RANGE(5, 15, 18);
+  for (int i = 6; i <= 9; i++)
+    ASSERT_CHAR_AT_RANGE(i, 13 + i, 13 + i);
+
+  if (string_packing_verifiable_p ())
+    {
+      /* Verify packing.  */
+      ASSERT_EQ (strloc->m_num_fraglocs, 3);
+      if (range_bits_verifiable_p ())
+	{
+	  ASSERT_STRING_LOCATION_FRAGMENT
+	    (strloc, 0,
+	     linemap_position_for_column (line_table, 10),
+	     5, 1, case_.m_default_range_bits);
+	  ASSERT_STRING_LOCATION_FRAGMENT
+	    (strloc, 1,
+	     linemap_position_for_column (line_table, 15),
+	     1, 4, case_.m_default_range_bits);
+	  ASSERT_STRING_LOCATION_FRAGMENT
+	    (strloc, 2,
+	     linemap_position_for_column (line_table, 19),
+	     4, 1, case_.m_default_range_bits);
+	}
+    }
+}
+
+/* FIXME.  */
+static void
+test_lexer_string_locations_ucn4 (const line_table_case &case_)
+{
+  /* Digits 0-9, expressing digits 5 and 6 as Roman numerals expressed
+     as UCN 4.
+     ....................000000000.111111.111122.222222223.
+     ....................123456789.012345.678901.234567890.  */
+  const char *content = "        \"01234\\u2174\\u2175789\"\n";
+  lexer_test test (case_, content);
+
+  /* Verify that we get the expected token back, with the correct
+     location information.  */
+  const cpp_token *tok = test.get_token ();
+  ASSERT_EQ (tok->type, CPP_STRING);
+  ASSERT_TOKEN_AS_TEXT_EQ (test.m_parser, tok, "\"01234\\u2174\\u2175789\"");
+
+  /* Use cpp_interpret_string to extract the quoted characters
+     into "dst_string".  */
+  cpp_string dst_string;
+  bool result = cpp_interpret_string (test.m_parser, &tok->val.str, 1,
+				      &dst_string, CPP_STRING);
+  ASSERT_TRUE (result);
+
+  /* Verify dst_string.  */
+  const cpp_string_location *strloc = &dst_string.loc;
+
+  /* Verify ranges of individual characters.  This no longer includes the
+     quotes.  */
+  for (int i = 0; i < 4; i++)
+    ASSERT_CHAR_AT_RANGE (i, 10 + i, 10 + i);
+  ASSERT_CHAR_AT_RANGE (5, 15, 20);
+  ASSERT_CHAR_AT_RANGE (6, 21, 26);
+  for (int i = 7; i <= 9; i++)
+    ASSERT_CHAR_AT_RANGE (i, 20 + i, 20 + i);
+
+  if (string_packing_verifiable_p ())
+    {
+      ASSERT_EQ (strloc->m_num_fraglocs, 3);
+      if (range_bits_verifiable_p ())
+	{
+	  ASSERT_STRING_LOCATION_FRAGMENT
+	    (strloc, 0,
+	     linemap_position_for_column (line_table, 10),
+	     5, 1, case_.m_default_range_bits);
+	  ASSERT_STRING_LOCATION_FRAGMENT
+	    (strloc, 1,
+	     linemap_position_for_column (line_table, 15),
+	     2, 6, case_.m_default_range_bits);
+	  ASSERT_STRING_LOCATION_FRAGMENT
+	    (strloc, 2,
+	     linemap_position_for_column (line_table, 27),
+	     3, 1, case_.m_default_range_bits);
+	}
+    }
+}
+
+/* FIXME.  */
+static void
+test_lexer_string_locations_ucn8 (const line_table_case &case_)
+{
+  /* Digits 0-9, expressing digits 5 and 6 as Roman numerals as UCN 8.
+     ....................000000000.111111.1111222222.22223333333333.
+     ....................123456789.012345.6789012345.67890123456789.  */
+  const char *content = "        \"01234\\U00002174\\U00002175789\"\n";
+  lexer_test test (case_, content);
+
+  /* Verify that we get the expected token back, with the correct
+     location information.  */
+  const cpp_token *tok = test.get_token ();
+  ASSERT_EQ (tok->type, CPP_STRING);
+  ASSERT_TOKEN_AS_TEXT_EQ (test.m_parser, tok,
+			   "\"01234\\U00002174\\U00002175789\"");
+
+  /* Use cpp_interpret_string to extract the quoted characters
+     into "dst_string".  */
+  cpp_string dst_string;
+  bool result = cpp_interpret_string (test.m_parser, &tok->val.str, 1,
+				      &dst_string, CPP_STRING);
+  ASSERT_TRUE (result);
+
+  /* Verify dst_string.  */
+  const cpp_string_location *strloc = &dst_string.loc;
+
+  /* Verify ranges of individual characters.  This no longer includes the
+     quotes.  */
+  for (int i = 0; i < 4; i++)
+    ASSERT_CHAR_AT_RANGE (i, 10 + i, 10 + i);
+  ASSERT_CHAR_AT_RANGE (5, 15, 24);
+  ASSERT_CHAR_AT_RANGE (6, 25, 34);
+  for (int i = 7; i <= 9; i++)
+    ASSERT_CHAR_AT_RANGE (i, 28 + i, 28 + i);
+
+  if (string_packing_verifiable_p ())
+    {
+      ASSERT_EQ (strloc->m_num_fraglocs, 3);
+      if (range_bits_verifiable_p ())
+	{
+	  ASSERT_STRING_LOCATION_FRAGMENT
+	    (strloc, 0,
+	     linemap_position_for_column (line_table, 10),
+	     5, 1, case_.m_default_range_bits);
+	  ASSERT_STRING_LOCATION_FRAGMENT
+	    (strloc, 1,
+	     linemap_position_for_column (line_table, 15),
+	     2, 10, case_.m_default_range_bits);
+	  ASSERT_STRING_LOCATION_FRAGMENT
+	    (strloc, 2,
+	     linemap_position_for_column (line_table, 35),
+	     3, 1, case_.m_default_range_bits);
+	}
+    }
+}
+
+/* FIXME.  */
+static void
+test_lexer_string_locations_u8 (const line_table_case &case_)
+{
+  /* Digits 0-9.
+     ....................000000000.11111111112.
+     ....................123456789.01234567890.  */
+  const char *content = "      u8\"0123456789\"\n";
+  lexer_test test (case_, content);
+
+  /* Verify that we get the expected token back, with the correct
+     location information.  */
+  const cpp_token *tok = test.get_token ();
+  ASSERT_EQ (tok->type, CPP_UTF8STRING);
+  ASSERT_TOKEN_AS_TEXT_EQ (test.m_parser, tok, "u8\"0123456789\"");
+
+  /* Use cpp_interpret_string to extract the quoted characters
+     into "dst_string".  */
+  cpp_string dst_string;
+  bool result = cpp_interpret_string (test.m_parser, &tok->val.str, 1,
+				      &dst_string, CPP_STRING);
+  ASSERT_TRUE (result);
+
+  /* Verify dst_string.  */
+  const cpp_string_location *strloc = &dst_string.loc;
+
+  /* Verify ranges of individual characters.  This no longer includes the
+     quotes.  */
+  for (int i = 0; i < 9; i++)
+    ASSERT_CHAR_AT_RANGE (i, 10 + i, 10 + i);
+
+  if (string_packing_verifiable_p ())
+    {
+      /* Verify packing.  */
+      ASSERT_EQ (strloc->m_num_fraglocs, 1);
+      if (range_bits_verifiable_p ())
+	ASSERT_STRING_LOCATION_FRAGMENT
+	  (strloc, 0,
+	   linemap_position_for_column (line_table, 10),
+	   10, 1, case_.m_default_range_bits);
+    }
+}
+
 /* A table of interesting location_t values, giving one axis of our test
    matrix.  */
 
@@ -1827,6 +2272,12 @@ input_c_tests ()
 	  test_simple_substring_locations (c);
 	  test_complex_substring_locations (c);
 	  test_lexer (c);
+	  test_lexer_string_locations_simple (c);
+	  test_lexer_string_locations_hex (c);
+	  test_lexer_string_locations_oct (c);
+	  test_lexer_string_locations_ucn4 (c);
+	  test_lexer_string_locations_ucn8 (c);
+	  test_lexer_string_locations_u8 (c);
 
 	  num_cases_tested++;
 	}
diff --git a/libcpp/charset.c b/libcpp/charset.c
index c8a4b25..8234fbf 100644
--- a/libcpp/charset.c
+++ b/libcpp/charset.c
@@ -1910,6 +1910,9 @@ void
 cpp_string_location::init_raw (source_location loc, int len, int cols_per_char,
 			       line_maps *line_table)
 {
+  if (loc > LINE_MAP_MAX_LOCATION_WITH_COLS)
+    cols_per_char = 0;
+
   line_map_realloc reallocator = (line_table->reallocator
 				  ? line_table->reallocator
 				  : (line_map_realloc) xrealloc);
diff --git a/libcpp/include/cpplib.h b/libcpp/include/cpplib.h
index a2a6686..d613ae7 100644
--- a/libcpp/include/cpplib.h
+++ b/libcpp/include/cpplib.h
@@ -190,7 +190,10 @@ struct GTY(()) cpp_string_fragment_location {
 
   source_location get_char_finish (int idx) const
   {
-    return get_char_start (idx) + ((m_cols_per_char - 1) << m_bits_per_col);
+    if (m_cols_per_char > 0)
+      return get_char_start (idx) + ((m_cols_per_char - 1) << m_bits_per_col);
+    else
+      return m_loc;
   }
 
   source_range get_char_range (int idx) const
-- 
1.8.5.3

