From 5ed8c088542b0d348d762e4945bc71c1bd6e126f Mon Sep 17 00:00:00 2001
From: David Malcolm <dmalcolm@redhat.com>
Date: Mon, 31 Aug 2015 21:04:46 -0400
Subject: [PATCH 06/25] FIXME: Location tracking within string-literals

TODO:
  - blurb
    - location of early version of patch
  - FIXMEs

This is a combination of 3 commits (and more)

"libcpp: add location tracking within string literals"
"Track locations within string literals in tree_string"

This has not been optimized yet.

libcpp/ChangeLog:

This is the 2nd commit message:

This patch uses the string-literal location generated in libcpp in the
previous patch, and stores it in tree_string (adding a new field there).

This hasn't been optimized.  Perhaps the case of a single unbroken run
of 1-column per-char is the most common case, so we could only bother to
store the character range info for those string literal that are exceptions
to this rule.

The patch adds unit testing via a plugin.

Screenshot:
https://dmalcolm.fedorapeople.org/gcc/2015-09-09/string-literals.html

gcc/c-family/ChangeLog:
	* c-common.c (fname_as_string): Initialize loc field of "cstr",
	and call init_raw on strname.loc.
	* c-lex.c (cb_ident): Initialize loc field of "cstr".
	(ensure_string_has_location): New function.
	(lex_string): Call ensure_string_has_location on the cpp_string;
	pass in istr.loc to the call to build_string.
	(lex_charconst): Call ensure_string_has_location on the cpp_string;

gcc/cp/ChangeLog:
	* parser.c (cp_parser_string_literal): Call init_raw on the
	str.loc.  Pass in the istr.loc to the call to build_string.

gcc/ChangeLog:
	* gcc-rich-location.c (cpp_string_fragment_location::debug): New.
	(cpp_string_location::debug): New.
	* input.c (selftest::string_packing_verifiable_p): New function.
	(selftest::range_bits_verifiable_p): New function.
	(selftest::assert_char_at_range): New function.
	(ASSERT_CHAR_AT_RANGE): New macro.
	(selftest::assert_string_location_fragment): New function.
	(ASSERT_STRING_LOCATION_FRAGMENT): New macro.
	(selftest::test_simple_substring_locations): New function.
	(selftest::assert_complex_substring_locations): New function.
	(selftest::add_char_at): New function.
	(selftest::test_complex_substring_locations): New function.
	(selftest::struct lexer_test): New struct.
	(selftest::lexer_test::lexer_test): New constructor.
	(selftest::lexer_test::~lexer_test): New destructor.
	(selftest::lexer_test::get_token): New method.
	(selftest::test_lexer_string_locations_simple): New function.
	(selftest::test_lexer_string_locations_hex): New function.
	(selftest::test_lexer_string_locations_oct): New function.
	(selftest::test_lexer_string_locations_ucn4): New function.
	(selftest::test_lexer_string_locations_ucn8): New function.
	(selftest::test_lexer_string_locations_u8): New function.
	(selftest::input_c_tests): Call the new test functions once per
	case within the line_table test matrix.
	* selftest.h (ASSERT_STREQ_AT): Fix typo in comment.
	* tree-core.h (struct cpp_string_location): Add forward
	declaration.
	(struct tree_string): Add cpp_string_location * field "loc".
	* tree.c: Include cpplib.h.
	(build_string): Initialize TREE_STRING_LOCATION (s) to NULL.
	(cpp_string_location_stats): New global.
	(build_string): New overload.
	* tree.h (TREE_STRING_LOCATION): New macro.
	(cpp_string_location_stats): New struct and global.
	(build_string): Add overload taking an additional
	cpp_string_location * param.

gcc/testsuite/ChangeLog:
	* gcc.dg/plugin/diagnostic-test-string-literals-1.c: New file.
	* gcc.dg/plugin/diagnostic_plugin_test_string_literals.c: New file.
	* gcc.dg/plugin/plugin.exp (plugin_test_list): Add
	diagnostic_plugin_test_string_literals.c and
	diagnostic-test-string-literals-1.c.

libcpp/ChangeLog:
	* charset.c (struct _cpp_strbuf): Add cpp_string_location field
	"loc".
	(conversion_loop): Add "loc_reader" param and, if non-NULL, call its
	add_char_at method.
	(convert_utf8_utf16): Add "loc_reader" param and pass it to
	conversion_loop.
	(convert_utf8_utf32): Likewise.
	(convert_utf16_utf8): Likewise.
	(convert_utf32_utf8): Likewise.
	(convert_no_conversion): Add "loc_reader" param and, if non-NULL,
	call its add_n_chars_at method.
	(convert_using_iconv): Add dummy cpp_string_location_reader *
	param.
	(APPLY_CONVERSION): Add LOC_READER param.
	(cpp_host_to_exec_charset): Call init on tbuf's loc.  Supply NULL
	to LOC_READER param of APPLY_CONVERSION.
	(_cpp_valid_ucn): Add "char_range" and "loc_reader" params.  Write
	back to "char_range".
	(convert_ucn): Add "char_range" and "loc_reader" params, passing
	them to _cpp_valid_ucn call and to APPLY_CONVERSION site.
	(convert_hex): Add "char_range" and "loc_reader" params; use them
	to track source range information.
	(convert_oct): Likewise.
	(convert_escape): Add loc_reader param and use it to track source
	range information.
	(cpp_interpret_string): Initialize tbuf.loc.  Create an on-stack
	cpp_string_location_reader and use it to track source range
	information.
	(cpp_interpret_charconst): Initialize str.loc.
	(_cpp_convert_input): Initialize to.loc.  Add NULL when calling
	APPLY_CONVERSION.
	(cpp_string_location::init): New method.
	(cpp_string_location::init_raw): New method.
	(cpp_string_location::add_char_at): New method.
	(cpp_string_location::add_n_chars_at): New method.
	(cpp_string_location::get_loc_at_index): New method.
	(cpp_string_location::get_range_at_index): New method.
	(cpp_string_location::get_range_between_indices): New method.
	(cpp_string_location::trivial_p): New method.
	(cpp_string_location_reader::cpp_string_location_reader): New ctors.
	(cpp_string_location_reader::get_next): New method.
	* directives.c (do_line): Initialize s.loc;
	(do_linemarker): Likewise.
	* expr.c (_cpp_parse_expr): Call init_raw on the token's str.loc.
	* include/cpplib.h (struct cpp_string_fragment_location): New struct.
	(struct cpp_string_location): New struct.
	(class cpp_string_location_reader): New class.
	(struct cpp_string): Add field "loc", a cpp_string_location.
	* internal.h (convert_f): Add cpp_string_location_reader * param.
	(_cpp_valid_ucn): Add source_range * and
	cpp_string_location_reader * params.
	* lex.c (forms_identifier_p): Add NULL argument to _cpp_valid_ucn.
	(lex_number): Initialize number->loc.
	(create_literal): Call init_raw on the token's str.loc.
	* macro.c (new_string_token): Call init on the token's str.loc.
---
 gcc/c-family/c-common.c                            |   3 +-
 gcc/c-family/c-lex.c                               |  26 +-
 gcc/cp/parser.c                                    |   4 +-
 gcc/gcc-rich-location.c                            |  29 +
 gcc/input.c                                        | 810 +++++++++++++++++++++
 gcc/selftest.h                                     |   2 +-
 .../plugin/diagnostic-test-string-literals-1.c     | 172 +++++
 .../diagnostic_plugin_test_string_literals.c       | 205 ++++++
 gcc/testsuite/gcc.dg/plugin/plugin.exp             |   2 +
 gcc/tree-core.h                                    |   3 +
 gcc/tree.c                                         |  24 +
 gcc/tree.h                                         |  12 +
 libcpp/charset.c                                   | 406 ++++++++++-
 libcpp/directives.c                                |   4 +-
 libcpp/expr.c                                      |   2 +
 libcpp/include/cpplib.h                            | 156 ++++
 libcpp/internal.h                                  |   7 +-
 libcpp/lex.c                                       |  12 +-
 libcpp/macro.c                                     |   1 +
 19 files changed, 1831 insertions(+), 49 deletions(-)
 create mode 100644 gcc/testsuite/gcc.dg/plugin/diagnostic-test-string-literals-1.c
 create mode 100644 gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_string_literals.c

diff --git a/gcc/c-family/c-common.c b/gcc/c-family/c-common.c
index 936ddfb..18c4856 100644
--- a/gcc/c-family/c-common.c
+++ b/gcc/c-family/c-common.c
@@ -945,7 +945,7 @@ fname_as_string (int pretty_p)
   const char *name = "top level";
   char *namep;
   int vrb = 2, len;
-  cpp_string cstr = { 0, 0 }, strname;
+  cpp_string cstr = { 0, 0, {NULL, 0, 0} }, strname;
 
   if (!pretty_p)
     {
@@ -962,6 +962,7 @@ fname_as_string (int pretty_p)
   snprintf (namep, len, "\"%s\"", name);
   strname.text = (unsigned char *) namep;
   strname.len = len - 1;
+  strname.loc.init_raw (UNKNOWN_LOCATION, len, 1, 1, line_table);
 
   if (cpp_interpret_string (parse_in, &strname, 1, &cstr, CPP_STRING))
     {
diff --git a/gcc/c-family/c-lex.c b/gcc/c-family/c-lex.c
index 8f33d86..8a722cf 100644
--- a/gcc/c-family/c-lex.c
+++ b/gcc/c-family/c-lex.c
@@ -165,7 +165,7 @@ cb_ident (cpp_reader * ARG_UNUSED (pfile),
   if (!flag_no_ident)
     {
       /* Convert escapes in the string.  */
-      cpp_string cstr = { 0, 0 };
+      cpp_string cstr = { 0, 0, { NULL, 0, 0 } };
       if (cpp_interpret_string (pfile, str, 1, &cstr, CPP_STRING))
 	{
 	  targetm.asm_out.output_ident ((const char *) cstr.text);
@@ -1072,6 +1072,20 @@ interpret_fixed (const cpp_token *token, unsigned int flags)
   return value;
 }
 
+/* Ensure that src->loc has location data, hardcoding it to a single
+   character at SRC_LOC if data is not already present.  */
+
+static void
+ensure_string_has_location (const cpp_string *str, source_location src_loc)
+{
+  if (!str->loc.m_fragloc_array)
+    {
+      cpp_string *mutstr = const_cast <cpp_string *> (str);
+      cpp_string_location *strloc = &mutstr->loc;
+      strloc->init_raw (src_loc, mutstr->len, 1, 1, line_table);
+    }
+}
+
 /* Convert a series of STRING, WSTRING, STRING16, STRING32 and/or
    UTF8STRING tokens into a tree, performing string constant
    concatenation.  TOK is the first of these.  VALP is the location to
@@ -1103,7 +1117,9 @@ lex_string (const cpp_token *tok, tree *valp, bool objc_string, bool translate)
   /* Try to avoid the overhead of creating and destroying an obstack
      for the common case of just one string.  */
   cpp_string str = tok->val.str;
+  ensure_string_has_location (&str, tok->src_loc);
   cpp_string *strs = &str;
+  location_t str0_loc = tok->src_loc;
 
   /* objc_at_sign_was_seen is only used when doing Objective-C string
      concatenation.  It is 'true' if we have seen an '@' before the
@@ -1142,16 +1158,20 @@ lex_string (const cpp_token *tok, tree *valp, bool objc_string, bool translate)
 	  else
 	    error ("unsupported non-standard concatenation of string literals");
 	}
+      /* FALLTHROUGH */
 
     case CPP_STRING:
       if (!concats)
 	{
 	  gcc_obstack_init (&str_ob);
+	  ensure_string_has_location (&str, str0_loc);
 	  obstack_grow (&str_ob, &str, sizeof (cpp_string));
 	}
 
       concats++;
+      ensure_string_has_location (&tok->val.str, tok->src_loc);
       obstack_grow (&str_ob, &tok->val.str, sizeof (cpp_string));
+
       if (objc_string)
 	objc_at_sign_was_seen = false;
       goto retry;
@@ -1174,7 +1194,7 @@ lex_string (const cpp_token *tok, tree *valp, bool objc_string, bool translate)
        ? cpp_interpret_string : cpp_interpret_string_notranslate)
       (parse_in, strs, concats + 1, &istr, type))
     {
-      value = build_string (istr.len, (const char *) istr.text);
+      value = build_string (istr.len, (const char *) istr.text, &istr.loc);
       free (CONST_CAST (unsigned char *, istr.text));
     }
   else
@@ -1241,6 +1261,8 @@ lex_charconst (const cpp_token *token)
   unsigned int chars_seen;
   int unsignedp = 0;
 
+  ensure_string_has_location (&token->val.str, token->src_loc);
+
   result = cpp_interpret_charconst (parse_in, token,
 				    &chars_seen, &unsignedp);
 
diff --git a/gcc/cp/parser.c b/gcc/cp/parser.c
index 3e8270e..c408bf2 100644
--- a/gcc/cp/parser.c
+++ b/gcc/cp/parser.c
@@ -3878,6 +3878,7 @@ cp_parser_string_literal (cp_parser *parser, bool translate, bool wide_ok,
 
       str.text = (const unsigned char *)TREE_STRING_POINTER (string_tree);
       str.len = TREE_STRING_LENGTH (string_tree);
+      str.loc.init_raw (tok->location, str.len, 1, 1, line_table);
       count = 1;
 
       if (curr_tok_is_userdef_p)
@@ -3903,6 +3904,7 @@ cp_parser_string_literal (cp_parser *parser, bool translate, bool wide_ok,
 	  count++;
 	  str.text = (const unsigned char *)TREE_STRING_POINTER (string_tree);
 	  str.len = TREE_STRING_LENGTH (string_tree);
+	  str.loc.init_raw (tok->location, str.len, 1, 1, line_table);
 
 	  if (curr_tok_is_userdef_p)
 	    {
@@ -3976,7 +3978,7 @@ cp_parser_string_literal (cp_parser *parser, bool translate, bool wide_ok,
   if ((translate ? cpp_interpret_string : cpp_interpret_string_notranslate)
       (parse_in, strs, count, &istr, type))
     {
-      value = build_string (istr.len, (const char *)istr.text);
+      value = build_string (istr.len, (const char *)istr.text, &istr.loc);
       free (CONST_CAST (unsigned char *, istr.text));
 
       switch (type)
diff --git a/gcc/gcc-rich-location.c b/gcc/gcc-rich-location.c
index 15c0700..8ac45f9 100644
--- a/gcc/gcc-rich-location.c
+++ b/gcc/gcc-rich-location.c
@@ -74,3 +74,32 @@ gcc_rich_location::add_fixit_misspelled_id (location_t misspelled_token_loc,
     = get_range_from_loc (line_table, misspelled_token_loc);
   add_fixit_replace (misspelled_token_range, IDENTIFIER_POINTER (hint_id));
 }
+
+/* Debugging method.  Print a diagnostic showing the given fragment.  */
+
+void
+cpp_string_fragment_location::debug (const char *msg) const
+{
+  source_range range = get_covered_range ();
+  location_t loc
+    = make_location (range.m_start, range.m_start, range.m_finish);
+  gcc_rich_location richloc (loc);
+  inform_at_rich_loc (&richloc, "%s", msg);
+}
+
+/* Debugging method.  Print diagnostic showings the fragment(s) that make
+   up the cpp_string_location.  */
+
+void
+cpp_string_location::debug () const
+{
+  char buf[64];
+  for (unsigned int i = 0; i < m_num_fraglocs; i++)
+    {
+      cpp_string_fragment_location *fragment = &m_fragloc_array[i];
+      snprintf (buf, sizeof (buf),
+		"fragment %i; %i cols per item; %i bytes per item",
+		i, fragment->m_cols_per_item, fragment->m_bytes_per_item);
+      fragment->debug (buf);
+    }
+}
diff --git a/gcc/input.c b/gcc/input.c
index 0016555..3d497dd 100644
--- a/gcc/input.c
+++ b/gcc/input.c
@@ -1356,6 +1356,323 @@ test_builtins ()
   ASSERT_PRED1 (is_location_from_builtin_token, BUILTINS_LOCATION);
 }
 
+/* Verify that locations within string literals are correctly handled.  */
+
+/* Determine if we should verify the packing within cpp_string_location
+   instances.  If any location is above LINE_MAP_MAX_LOCATION_WITH_COLS,
+   the precise packing is likely to be unpredictable: some or all of
+   the characters might be above the threshold.  */
+
+static bool
+string_packing_verifiable_p ()
+{
+  return line_table->highest_location <= LINE_MAP_MAX_LOCATION_WITH_COLS;
+}
+
+/* Determine if we should attempt to verify the range bits within
+   cpp_string_fragment_location instances.
+   If any of the data may use a non default value for range_bits,
+   don't attempt to verify it.  */
+
+static bool
+range_bits_verifiable_p ()
+{
+  return (line_table->highest_location
+	  <= LINE_MAP_MAX_LOCATION_WITH_PACKED_RANGES);
+}
+
+/* Verify STRLOC's description of locations within a string literal.
+   Assert that the character at index IDX begins at column EXPECTED_START_COL
+   and ends at EXPECTED_FINISH_COL, unless the locations are beyond
+   LINE_MAP_MAX_LOCATION_WITH_COLS, in which case don't check their
+   columns.  */
+
+static void
+assert_char_at_range (const location &loc,
+		      const cpp_string_location *strloc, int idx,
+		      int expected_start_col, int expected_finish_col)
+{
+  source_range actual_range = strloc->get_range_at_index (idx);
+  if (should_have_column_data_p (actual_range.m_start))
+    {
+      int actual_start_col = LOCATION_COLUMN (actual_range.m_start);
+      ASSERT_EQ_AT (loc, expected_start_col, actual_start_col);
+    }
+  if (should_have_column_data_p (actual_range.m_finish))
+    {
+      int actual_finish_col = LOCATION_COLUMN (actual_range.m_finish);
+      ASSERT_EQ_AT (loc, expected_finish_col, actual_finish_col);
+    }
+}
+
+/* Macro for calling assert_char_at_range, supplying SELFTEST_LOCATION for
+   the effective location of any errors.  */
+
+#define ASSERT_CHAR_AT_RANGE(IDX, EXPECTED_START_COL, EXPECTED_FINISH_COL) \
+  assert_char_at_range (SELFTEST_LOCATION, strloc, (IDX),		\
+			(EXPECTED_START_COL), (EXPECTED_FINISH_COL))
+
+/* Locate fragment IDX within STRLOC and assert that its various fields
+   equal the provided values.  */
+
+static void
+assert_string_location_fragment (const location &loc,
+				 const cpp_string_location *strloc, int idx,
+				 location_t expected_start_loc,
+				 int expected_len, int expected_cols_per_item,
+				 int expected_bits_per_col,
+				 int expected_bytes_per_item)
+{
+  const cpp_string_fragment_location &fragment = strloc->m_fragloc_array[idx];
+  ASSERT_EQ_AT (loc, fragment.m_loc, expected_start_loc);
+  ASSERT_EQ_AT (loc, fragment.m_len, expected_len);
+  ASSERT_EQ_AT (loc, fragment.m_cols_per_item, expected_cols_per_item);
+  ASSERT_EQ_AT (loc, fragment.m_bits_per_col, expected_bits_per_col);
+  ASSERT_EQ_AT (loc, fragment.m_bytes_per_item, expected_bytes_per_item);
+}
+
+/* Macro for calling assert_string_location_fragment, supplying
+   SELFTEST_LOCATION for the effective location of any errors.  */
+
+#define ASSERT_STRING_LOCATION_FRAGMENT(STRLOC, IDX, EXPECTED_START_LOC, \
+					EXPECTED_LEN, EXPECTED_COLS_PER_CHAR, \
+					EXPECTED_BITS_PER_COL, \
+					EXPECTED_BYTES_PER_ITEM)       \
+  assert_string_location_fragment (SELFTEST_LOCATION, (STRLOC), (IDX), \
+				   (EXPECTED_START_LOC), (EXPECTED_LEN), \
+				   (EXPECTED_COLS_PER_CHAR),		\
+				   (EXPECTED_BITS_PER_COL),		\
+				   (EXPECTED_BYTES_PER_ITEM))
+
+
+/* Verify that locations work within a string literal, for the
+   simple case of a single unbroken run of 1 column-per-character
+   characters.  */
+
+static void
+test_simple_substring_locations (const line_table_case &case_)
+{
+  temp_line_table tmp_lt (case_);
+
+  linemap_add (line_table, LC_ENTER, false,
+	       "test-simple-substring-location.c", 0);
+  linemap_line_start (line_table, 1, 100);
+
+  const int start_column = 5;
+  location_t loc = linemap_position_for_column (line_table, start_column);
+
+  cpp_string_location strloc;
+  const int len = 10;
+  strloc.init_raw (loc, len, 1, 1, line_table);
+
+  /* Ensure that the line table contains all of the locations within the
+     string.  Specifically, ensure that they are all
+       <= line_table->highest_location, so that
+     linemap_location_from_macro_expansion_p returns false.  */
+  linemap_position_for_column (line_table, start_column + len);
+
+  /* Verify that we can correctly retrieve the range for each
+     character.  */
+  for (int i = 0; i < len; i++)
+    {
+      int expected_column = start_column + i;
+      assert_char_at_range (SELFTEST_LOCATION, &strloc, i,
+			    expected_column, expected_column);
+    }
+
+  /* Verify that the data was packed as expected.  */
+  if (string_packing_verifiable_p ())
+    {
+      ASSERT_TRUE (strloc.trivial_p ());
+      ASSERT_EQ (strloc.m_num_fraglocs, 1);
+      if (range_bits_verifiable_p ())
+	ASSERT_STRING_LOCATION_FRAGMENT (&strloc, 0, loc, len, 1,
+					 line_table->default_range_bits, 1);
+    }
+
+  linemap_add (line_table, LC_LEAVE, false, NULL, 0);
+}
+
+/* Verify a complicated example of locations within a string literal.
+
+   This uses the example from cpplib.h, with the following column
+   numbering:
+000000000111111111122222222223333333333
+123456789012345678901234567890123456789
+    printf ("foo \x25\151 bar"  "baz\n",
+    .........^^^^.....................: fragment 0: 4 chars at 1 col per char
+    .............^^^^^^^^.............: fragment 1: 2 chars at 4 cols per char
+    .....................^^^^.........: fragment 2: 4 chars at 1 col per char
+    .............................^^^..: fragment 3: 3 chars at 1 col per char
+    ................................^^: fragment 4: 1 char at 2 cols per char
+
+    We populate a cpp_string_location with the above data, then
+    verify reading the data back.  */
+
+/* A helper function for verifying the above data.  */
+
+static void
+assert_complex_substring_locations (const cpp_string_location *strloc,
+				    int default_range_bits)
+{
+  /* Verify that we can correctly retrieve the range for each
+     character.  */
+
+  /* "foo ".  */
+  ASSERT_CHAR_AT_RANGE (0, 14, 14);
+  ASSERT_CHAR_AT_RANGE (1, 15, 15);
+  ASSERT_CHAR_AT_RANGE (2, 16, 16);
+  ASSERT_CHAR_AT_RANGE (3, 17, 17);
+
+  /* "\x25\151".  */
+  ASSERT_CHAR_AT_RANGE (4, 18, 21);
+  ASSERT_CHAR_AT_RANGE (5, 22, 25);
+
+  /* " bar".  */
+  ASSERT_CHAR_AT_RANGE (6, 26, 26);
+  ASSERT_CHAR_AT_RANGE (7, 27, 27);
+  ASSERT_CHAR_AT_RANGE (8, 28, 28);
+  ASSERT_CHAR_AT_RANGE (9, 29, 29);
+
+  /* "baz".  */
+  ASSERT_CHAR_AT_RANGE (10, 34, 34);
+  ASSERT_CHAR_AT_RANGE (11, 35, 35);
+  ASSERT_CHAR_AT_RANGE (12, 36, 36);
+
+  /* "\n".  */
+  ASSERT_CHAR_AT_RANGE (13, 37, 38);
+
+  /* Verify that the data was packed as expected.  */
+
+  if (string_packing_verifiable_p ())
+    {
+      ASSERT_FALSE (strloc->trivial_p ());
+      ASSERT_EQ (strloc->m_num_fraglocs, 5);
+    }
+
+  /* If any of the data may use a non default value for range_bits,
+     don't attempt to verify it.  */
+  if (!range_bits_verifiable_p ())
+    return;
+
+  /* "foo ".  */
+  ASSERT_STRING_LOCATION_FRAGMENT
+    (strloc, 0,
+     linemap_position_for_column (line_table, 14),
+     4, 1, default_range_bits, 1);
+
+  /* "\x25\151".  */
+  ASSERT_STRING_LOCATION_FRAGMENT
+    (strloc, 1,
+     linemap_position_for_column (line_table, 18),
+     2, 4, default_range_bits, 1);
+
+  /* " bar".  */
+  ASSERT_STRING_LOCATION_FRAGMENT
+    (strloc, 2,
+     linemap_position_for_column (line_table, 26),
+     4, 1, default_range_bits, 1);
+
+  /* "baz".  */
+  ASSERT_STRING_LOCATION_FRAGMENT
+    (strloc, 3,
+     linemap_position_for_column (line_table, 34),
+     3, 1, default_range_bits, 1);
+
+  /* "\n".  */
+  ASSERT_STRING_LOCATION_FRAGMENT
+    (strloc, 4,
+     linemap_position_for_column (line_table, 37),
+     1, 2, default_range_bits, 1);
+}
+
+/* Helper function for test_complex_substring_locations.
+   Add a character to STRLOC on the current line, occupying the
+   columns ranging from START_COL to END_COL.  */
+
+static void
+add_char_at (cpp_string_location *strloc, int start_col, int end_col)
+{
+  source_range range;
+  range.m_start = linemap_position_for_column (line_table, start_col);
+  range.m_finish = linemap_position_for_column (line_table, end_col);
+  strloc->add_char_at (range, 1, line_table);
+}
+
+/* Populate a cpp_string_location with the above data, then verify
+   reading the data back.
+   Then use the cpp_string_location_reader class to copy the data,
+   and verify that the copy is correct.  */
+
+static void
+test_complex_substring_locations (const line_table_case &case_)
+{
+  temp_line_table tmp_lt (case_);
+
+  /* Populate strloc as per the example above.  */
+  cpp_string_location strloc;
+  strloc.init ();
+
+  /* Create a tempfile containing the content we're describing.
+     Apart from providing a filename, this isn't directly used by the test,
+     but is useful when debugging locations via "(gdb) call inform" and the
+     like.  */
+  temp_source_file tmp (SELFTEST_LOCATION, ".c",
+			"    printf (\"foo \\x25\\151 bar\"  \"baz\",");
+
+  linemap_add (line_table, LC_ENTER, false, tmp.get_filename (), 0);
+  linemap_line_start (line_table, 1, 100);
+
+  /* "foo ".  */
+  for (int col = 14; col <= 17; col++)
+    add_char_at (&strloc, col, col);
+  /* "\x25\151".  */
+  add_char_at (&strloc, 18, 21);
+  add_char_at (&strloc, 22, 25);
+  /* " bar".  */
+  for (int col = 26; col <= 29; col++)
+    add_char_at (&strloc, col, col);
+  /* "baz".  */
+  for (int col = 34; col <= 36; col++)
+    add_char_at (&strloc, col, col);
+  /* "\n".  */
+  add_char_at (&strloc, 37, 38);
+
+  /* Ensure that the line table contains all of the locations within the
+     string.  Specifically, ensure that they are all
+       <= line_table->highest_location, so that
+     linemap_location_from_macro_expansion_p returns false.  */
+  linemap_position_for_column (line_table, 38);
+
+  /* Verify that we can read the correct data back.  */
+  assert_complex_substring_locations (&strloc, case_.m_default_range_bits);
+
+  /* Use the cpp_string_location_reader class to copy the data.  */
+  cpp_string_location dst_strloc;
+  dst_strloc.init ();
+  for (unsigned i = 0; i < strloc.m_num_fraglocs; i++)
+    {
+      const cpp_string_fragment_location &src_fragment
+	= strloc.m_fragloc_array[i];
+      cpp_string_location_reader reader (src_fragment.m_loc,
+					 src_fragment.m_cols_per_item,
+					 src_fragment.m_bits_per_col,
+					 line_table);
+      for (int j = 0; j < src_fragment.m_len; j++)
+	{
+	  source_range range = reader.get_next ();
+	  dst_strloc.add_char_at (range, 1, line_table);
+	}
+    }
+
+  /* Verify that we can read the correct data back from the copy.  */
+  assert_complex_substring_locations (&dst_strloc, case_.m_default_range_bits);
+
+  linemap_add (line_table, LC_LEAVE, false, NULL, 0);
+}
+
+/* TODO: a testcase involving more than one linemap.  */
+
 /* Verify reading of input files (e.g. for caret-based diagnostics).  */
 
 static void
@@ -1480,6 +1797,491 @@ test_lexer (const line_table_case &case_)
   cpp_finish (parser, NULL);
 }
 
+/* A struct for writing lexer tests.  */
+
+struct lexer_test
+{
+  lexer_test (const line_table_case &case_, const char *content);
+  ~lexer_test ();
+
+  const cpp_token *get_token ();
+
+  temp_source_file m_tempfile;
+  temp_line_table m_tmp_lt;
+  cpp_reader *m_parser;
+};
+
+/* Constructor.  Override line_table with a new instance based on CASE_,
+   and write CONTENT to a tempfile.  Create a cpp_reader, and use it to
+   start parsing the tempfile.  */
+
+lexer_test::lexer_test (const line_table_case &case_, const char *content) :
+  /* Create a tempfile and write the text to it.  */
+  m_tempfile (SELFTEST_LOCATION, ".c", content),
+  m_tmp_lt (case_),
+  m_parser (cpp_create_reader (CLK_GNUC99, NULL, line_table))
+{
+  cpp_init_iconv (m_parser);
+
+  /* Parse the file.  */
+  const char *fname = cpp_read_main_file (m_parser,
+					  m_tempfile.get_filename ());
+  ASSERT_NE (fname, NULL);
+}
+
+/* Destructor.  Verify that the next token in m_parser is EOF.  */
+
+lexer_test::~lexer_test ()
+{
+  location_t loc;
+  const cpp_token *tok;
+
+  tok = cpp_get_token_with_location (m_parser, &loc);
+  ASSERT_NE (tok, NULL);
+  ASSERT_EQ (tok->type, CPP_EOF);
+
+  cpp_finish (m_parser, NULL);
+}
+
+/* Get the next token from m_parser.  */
+
+const cpp_token *
+lexer_test::get_token ()
+{
+  location_t loc;
+  const cpp_token *tok;
+
+  tok = cpp_get_token_with_location (m_parser, &loc);
+  ASSERT_NE (tok, NULL);
+  return tok;
+}
+
+/* Lex a simple string literal.  Verify the substring location data, before
+   and after running cpp_interpret_string on it.  */
+
+static void
+test_lexer_string_locations_simple (const line_table_case &case_)
+{
+  /* Digits 0-9 (with 0 at column 10), the simple way.
+     ....................000000000.11111111112.
+     ....................123456789.01234567890.  */
+  const char *content = "        \"0123456789\"\n";
+  lexer_test test (case_, content);
+
+  /* Verify that we get the expected token back, with the correct
+     location information.  */
+  const cpp_token *tok = test.get_token ();
+  ASSERT_EQ (tok->type, CPP_STRING);
+  ASSERT_TOKEN_AS_TEXT_EQ (test.m_parser, tok, "\"0123456789\"");
+  ASSERT_TOKEN_LOC_EQ (tok, test.m_tempfile.get_filename (), 1, 9, 20);
+
+  /* At this point in lexing, the quote characters are treated as part of
+     the string (they are stripped off by cpp_interpret_string).  */
+
+  ASSERT_EQ (tok->val.str.len, 12);
+
+  const cpp_string_location *strloc = &tok->val.str.loc;
+
+  /* If any of the data might be in no-columns mode, don't
+     attempt to predict the packing.  */
+  if (string_packing_verifiable_p ())
+    {
+      ASSERT_TRUE (strloc->trivial_p ());
+      ASSERT_EQ (strloc->m_num_fraglocs, 1);
+    }
+
+  /* Verify ranges of individual characters (including the quotes).  */
+  for (int i = 0; i < 12; i++)
+    ASSERT_CHAR_AT_RANGE(i, 9 + i, 9 + i);
+
+  ASSERT_EQ (strloc->m_num_fraglocs, 1);
+  if (range_bits_verifiable_p ())
+    ASSERT_STRING_LOCATION_FRAGMENT
+      (strloc, 0,
+       linemap_position_for_column (line_table, 9),
+       12, 1, case_.m_default_range_bits, 1);
+
+  /* Use cpp_interpret_string to extract the quoted characters
+     into "dst_string".  */
+  cpp_string dst_string;
+  bool result = cpp_interpret_string (test.m_parser, &tok->val.str, 1,
+				      &dst_string, CPP_STRING);
+  ASSERT_TRUE (result);
+
+  /* Verify dst_string.  */
+  strloc = &dst_string.loc;
+
+  /* Verify ranges of individual characters.  This no longer includes the
+     quotes.  */
+  for (int i = 0; i < 9; i++)
+    ASSERT_CHAR_AT_RANGE(i, 10 + i, 10 + i);
+
+  if (string_packing_verifiable_p ())
+    {
+      /* Verify packing.  */
+      ASSERT_EQ (strloc->m_num_fraglocs, 1);
+      if (range_bits_verifiable_p ())
+	ASSERT_STRING_LOCATION_FRAGMENT
+	  (strloc, 0,
+	   linemap_position_for_column (line_table, 10),
+	   10, 1, case_.m_default_range_bits, 1);
+    }
+}
+
+/* Lex a string literal containing a hex-escaped character.
+   Verify the substring location data, before and after running
+   cpp_interpret_string on it.  */
+
+static void
+test_lexer_string_locations_hex (const line_table_case &case_)
+{
+  /* Digits 0-9, expressing digit 5 in ASCII as "\x35"
+     and with a space in place of digit 6, to terminate the escaped
+     hex code.
+     ....................000000000.111111.11112222.
+     ....................123456789.012345.67890123.  */
+  const char *content = "        \"01234\\x35 789\"\n";
+  lexer_test test (case_, content);
+
+  /* Verify that we get the expected token back, with the correct
+     location information.  */
+  const cpp_token *tok = test.get_token ();
+  ASSERT_EQ (tok->type, CPP_STRING);
+  ASSERT_TOKEN_AS_TEXT_EQ (test.m_parser, tok, "\"01234\\x35 789\"");
+  ASSERT_TOKEN_LOC_EQ (tok, test.m_tempfile.get_filename (), 1, 9, 23);
+
+  /* At this point in lexing, the quote characters are treated as part of
+     the string (they are stripped off by cpp_interpret_string).  */
+
+  ASSERT_EQ (tok->val.str.len, 15);
+
+  const cpp_string_location *strloc = &tok->val.str.loc;
+
+  /* Verify ranges of individual characters (including the quotes).  */
+  for (int i = 0; i < 15; i++)
+    ASSERT_CHAR_AT_RANGE(i, 9 + i, 9 + i);
+
+  /* If any of the data might be in no-columns mode, don't
+     attempt to predict the packing.  */
+  if (string_packing_verifiable_p ())
+    {
+      ASSERT_TRUE (strloc->trivial_p ());
+      ASSERT_EQ (strloc->m_num_fraglocs, 1);
+      if (range_bits_verifiable_p ())
+	ASSERT_STRING_LOCATION_FRAGMENT
+	  (strloc, 0,
+	   linemap_position_for_column (line_table, 9),
+	   15, 1, case_.m_default_range_bits, 1);
+    }
+
+  /* Use cpp_interpret_string to extract the quoted characters
+     into "dst_string".  */
+  cpp_string dst_string;
+  bool result = cpp_interpret_string (test.m_parser, &tok->val.str, 1,
+				      &dst_string, CPP_STRING);
+  ASSERT_TRUE (result);
+
+  /* Verify dst_string.  */
+  strloc = &dst_string.loc;
+
+  /* Verify ranges of individual characters.  This no longer includes the
+     quotes.  */
+  for (int i = 0; i < 5; i++)
+    ASSERT_CHAR_AT_RANGE(i, 10 + i, 10 + i);
+  ASSERT_CHAR_AT_RANGE(5, 15, 18);
+  for (int i = 6; i <= 9; i++)
+    ASSERT_CHAR_AT_RANGE(i, 13 + i, 13 + i);
+
+  if (string_packing_verifiable_p ())
+    {
+      /* Verify packing.  */
+      ASSERT_EQ (strloc->m_num_fraglocs, 3);
+      if (range_bits_verifiable_p ())
+	{
+	  ASSERT_STRING_LOCATION_FRAGMENT
+	    (strloc, 0,
+	     linemap_position_for_column (line_table, 10),
+	     5, 1, case_.m_default_range_bits, 1);
+	  ASSERT_STRING_LOCATION_FRAGMENT
+	    (strloc, 1,
+	     linemap_position_for_column (line_table, 15),
+	     1, 4, case_.m_default_range_bits, 1);
+	  ASSERT_STRING_LOCATION_FRAGMENT
+	    (strloc, 2,
+	     linemap_position_for_column (line_table, 19),
+	     4, 1, case_.m_default_range_bits, 1);
+	}
+    }
+}
+
+/* Lex a string literal containing an octal-escaped character.
+   Verify the substring location data after running cpp_interpret_string
+   on it.  */
+
+static void
+test_lexer_string_locations_oct (const line_table_case &case_)
+{
+  /* Digits 0-9, expressing digit 5 in ASCII as "\065"
+     and with a space in place of digit 6, to terminate the escaped
+     octal code.
+     ....................000000000.111111.11112222.
+     ....................123456789.012345.67890123.  */
+  const char *content = "        \"01234\\065 789\"\n";
+  lexer_test test (case_, content);
+
+  /* Verify that we get the expected token back, with the correct
+     location information.  */
+  const cpp_token *tok = test.get_token ();
+  ASSERT_EQ (tok->type, CPP_STRING);
+  ASSERT_TOKEN_AS_TEXT_EQ (test.m_parser, tok, "\"01234\\065 789\"");
+
+  /* Use cpp_interpret_string to extract the quoted characters
+     into "dst_string".  */
+  cpp_string dst_string;
+  bool result = cpp_interpret_string (test.m_parser, &tok->val.str, 1,
+				      &dst_string, CPP_STRING);
+  ASSERT_TRUE (result);
+
+  /* Verify dst_string.  */
+  const cpp_string_location *strloc = &dst_string.loc;
+
+  /* Verify ranges of individual characters.  This no longer includes the
+     quotes.  */
+  for (int i = 0; i < 5; i++)
+    ASSERT_CHAR_AT_RANGE(i, 10 + i, 10 + i);
+  ASSERT_CHAR_AT_RANGE(5, 15, 18);
+  for (int i = 6; i <= 9; i++)
+    ASSERT_CHAR_AT_RANGE(i, 13 + i, 13 + i);
+
+  if (string_packing_verifiable_p ())
+    {
+      /* Verify packing.  */
+      ASSERT_EQ (strloc->m_num_fraglocs, 3);
+      if (range_bits_verifiable_p ())
+	{
+	  ASSERT_STRING_LOCATION_FRAGMENT
+	    (strloc, 0,
+	     linemap_position_for_column (line_table, 10),
+	     5, 1, case_.m_default_range_bits, 1);
+	  ASSERT_STRING_LOCATION_FRAGMENT
+	    (strloc, 1,
+	     linemap_position_for_column (line_table, 15),
+	     1, 4, case_.m_default_range_bits, 1);
+	  ASSERT_STRING_LOCATION_FRAGMENT
+	    (strloc, 2,
+	     linemap_position_for_column (line_table, 19),
+	     4, 1, case_.m_default_range_bits, 1);
+	}
+    }
+}
+
+/* Lex a string literal containing UCN 4 characters.
+   Verify the substring location data after running cpp_interpret_string
+   on it.  */
+
+static void
+test_lexer_string_locations_ucn4 (const line_table_case &case_)
+{
+  /* Digits 0-9, expressing digits 5 and 6 as Roman numerals expressed
+     as UCN 4.
+     ....................000000000.111111.111122.222222223.
+     ....................123456789.012345.678901.234567890.  */
+  const char *content = "        \"01234\\u2174\\u2175789\"\n";
+  lexer_test test (case_, content);
+
+  /* Verify that we get the expected token back, with the correct
+     location information.  */
+  const cpp_token *tok = test.get_token ();
+  ASSERT_EQ (tok->type, CPP_STRING);
+  ASSERT_TOKEN_AS_TEXT_EQ (test.m_parser, tok, "\"01234\\u2174\\u2175789\"");
+
+  /* Use cpp_interpret_string to extract the quoted characters
+     into "dst_string".  */
+  cpp_string dst_string;
+  bool result = cpp_interpret_string (test.m_parser, &tok->val.str, 1,
+				      &dst_string, CPP_STRING);
+  ASSERT_TRUE (result);
+
+  /* Verify dst_string.  */
+  const cpp_string_location *strloc = &dst_string.loc;
+
+  /* The string should now be encoded in the execution character
+     set.  Assuming that that is UTF-8, we should have the following:
+     -----------  ----  -----  -------  ----------------
+     Byte offset  Byte  Octal  Unicode  Source Column(s)
+     -----------  ----  -----  -------  ----------------
+     0            0x30         '0'      10
+     1            0x31         '1'      11
+     2            0x32         '2'      12
+     3            0x33         '3'      13
+     4            0x34         '4'      14
+     5            0xE2  \342   U+2174   15-20
+     6            0x85  \205    (cont)  15-20
+     7            0xB4  \264    (cont)  15-20
+     8            0xE2  \342   U+2175   21-26
+     9            0x85  \205    (cont)  21-26
+     10           0xB5  \265    (cont)  21-26
+     11           0x37         '7'      27
+     12           0x38         '8'      28
+     13           0x39         '9'      29
+     -----------  ----  -----  -------  ---------------.  */
+
+  /* Verify ranges of individual characters.  This no longer includes the
+     quotes.
+     '01234'.  */
+  for (int i = 0; i < 4; i++)
+    ASSERT_CHAR_AT_RANGE (i, 10 + i, 10 + i);
+  /* U+2174.  */
+  for (int i = 5; i < 7; i++)
+    ASSERT_CHAR_AT_RANGE (i, 15, 20);
+  /* U+2175.  */
+  for (int i = 8; i < 10; i++)
+    ASSERT_CHAR_AT_RANGE (i, 21, 26);
+  /* '789'.  */
+  for (int i = 11; i <= 13; i++)
+    ASSERT_CHAR_AT_RANGE (i, 16 + i, 16 + i);
+
+  /* We should have the following fragments:
+     fragment 0: 4 items, 1 byte per item, 1 column per item
+     fragment 1: 2 items, 3 bytes per item, 6 columns per item
+     fragment 2: 3 items, 1 byte per item, 1 column per item.  */
+  if (string_packing_verifiable_p ())
+    {
+      ASSERT_EQ (strloc->m_num_fraglocs, 3);
+      if (range_bits_verifiable_p ())
+	{
+	  ASSERT_STRING_LOCATION_FRAGMENT
+	    (strloc, 0,
+	     linemap_position_for_column (line_table, 10),
+	     5, 1, case_.m_default_range_bits, 1);
+	  ASSERT_STRING_LOCATION_FRAGMENT
+	    (strloc, 1,
+	     linemap_position_for_column (line_table, 15),
+	     2, 6, case_.m_default_range_bits, 3);
+	  ASSERT_STRING_LOCATION_FRAGMENT
+	    (strloc, 2,
+	     linemap_position_for_column (line_table, 27),
+	     3, 1, case_.m_default_range_bits, 1);
+	}
+    }
+}
+
+/* Lex a string literal containing UCN 8 characters.
+   Verify the substring location data after running cpp_interpret_string
+   on it.  */
+
+static void
+test_lexer_string_locations_ucn8 (const line_table_case &case_)
+{
+  /* Digits 0-9, expressing digits 5 and 6 as Roman numerals as UCN 8.
+     ....................000000000.111111.1111222222.22223333333333.
+     ....................123456789.012345.6789012345.67890123456789.  */
+  const char *content = "        \"01234\\U00002174\\U00002175789\"\n";
+  lexer_test test (case_, content);
+
+  /* Verify that we get the expected token back, with the correct
+     location information.  */
+  const cpp_token *tok = test.get_token ();
+  ASSERT_EQ (tok->type, CPP_STRING);
+  ASSERT_TOKEN_AS_TEXT_EQ (test.m_parser, tok,
+			   "\"01234\\U00002174\\U00002175789\"");
+
+  /* Use cpp_interpret_string to extract the quoted characters
+     into "dst_string".  */
+  cpp_string dst_string;
+  bool result = cpp_interpret_string (test.m_parser, &tok->val.str, 1,
+				      &dst_string, CPP_STRING);
+  ASSERT_TRUE (result);
+
+  /* Verify dst_string.  */
+  const cpp_string_location *strloc = &dst_string.loc;
+
+  /* The UTF-8 encoding of the string is identical to that from
+     the ucn4 testcase above; the only difference is the column
+     locations.  */
+
+  /* Verify ranges of individual characters.  This no longer includes the
+     quotes.
+     '01234'.  */
+  for (int i = 0; i < 4; i++)
+    ASSERT_CHAR_AT_RANGE (i, 10 + i, 10 + i);
+  /* U+2174.  */
+  for (int i = 5; i < 7; i++)
+    ASSERT_CHAR_AT_RANGE (i, 15, 24);
+  /* U+2175.  */
+  for (int i = 8; i < 10; i++)
+    ASSERT_CHAR_AT_RANGE (i, 25, 34);
+  /* '789' at columns 35-37  */
+  for (int i = 11; i <= 13; i++)
+    ASSERT_CHAR_AT_RANGE (i, 24 + i, 24 + i);
+
+  if (string_packing_verifiable_p ())
+    {
+      ASSERT_EQ (strloc->m_num_fraglocs, 3);
+      if (range_bits_verifiable_p ())
+	{
+	  ASSERT_STRING_LOCATION_FRAGMENT
+	    (strloc, 0,
+	     linemap_position_for_column (line_table, 10),
+	     5, 1, case_.m_default_range_bits, 1);
+	  ASSERT_STRING_LOCATION_FRAGMENT
+	    (strloc, 1,
+	     linemap_position_for_column (line_table, 15),
+	     2, 10, case_.m_default_range_bits, 3);
+	  ASSERT_STRING_LOCATION_FRAGMENT
+	    (strloc, 2,
+	     linemap_position_for_column (line_table, 35),
+	     3, 1, case_.m_default_range_bits, 1);
+	}
+    }
+}
+
+/* Lex a u8-string literal.
+   Verify the substring location data after running cpp_interpret_string
+   on it.  */
+
+static void
+test_lexer_string_locations_u8 (const line_table_case &case_)
+{
+  /* Digits 0-9.
+     ....................000000000.11111111112.
+     ....................123456789.01234567890.  */
+  const char *content = "      u8\"0123456789\"\n";
+  lexer_test test (case_, content);
+
+  /* Verify that we get the expected token back, with the correct
+     location information.  */
+  const cpp_token *tok = test.get_token ();
+  ASSERT_EQ (tok->type, CPP_UTF8STRING);
+  ASSERT_TOKEN_AS_TEXT_EQ (test.m_parser, tok, "u8\"0123456789\"");
+
+  /* Use cpp_interpret_string to extract the quoted characters
+     into "dst_string".  */
+  cpp_string dst_string;
+  bool result = cpp_interpret_string (test.m_parser, &tok->val.str, 1,
+				      &dst_string, CPP_STRING);
+  ASSERT_TRUE (result);
+
+  /* Verify dst_string.  */
+  const cpp_string_location *strloc = &dst_string.loc;
+
+  /* Verify ranges of individual characters.  This no longer includes the
+     quotes.  */
+  for (int i = 0; i < 9; i++)
+    ASSERT_CHAR_AT_RANGE (i, 10 + i, 10 + i);
+
+  if (string_packing_verifiable_p ())
+    {
+      /* Verify packing.  */
+      ASSERT_EQ (strloc->m_num_fraglocs, 1);
+      if (range_bits_verifiable_p ())
+	ASSERT_STRING_LOCATION_FRAGMENT
+	  (strloc, 0,
+	   linemap_position_for_column (line_table, 10),
+	   10, 1, case_.m_default_range_bits, 1);
+    }
+}
+
 /* A table of interesting location_t values, giving one axis of our test
    matrix.  */
 
@@ -1537,7 +2339,15 @@ input_c_tests ()
 
 	  /* Run all tests for the given case within the test matrix.  */
 	  test_accessing_ordinary_linemaps (c);
+	  test_simple_substring_locations (c);
+	  test_complex_substring_locations (c);
 	  test_lexer (c);
+	  test_lexer_string_locations_simple (c);
+	  test_lexer_string_locations_hex (c);
+	  test_lexer_string_locations_oct (c);
+	  test_lexer_string_locations_ucn4 (c);
+	  test_lexer_string_locations_ucn8 (c);
+	  test_lexer_string_locations_u8 (c);
 
 	  num_cases_tested++;
 	}
diff --git a/gcc/selftest.h b/gcc/selftest.h
index 967e76b..c3a253b 100644
--- a/gcc/selftest.h
+++ b/gcc/selftest.h
@@ -167,7 +167,7 @@ extern int num_passes;
 			    (EXPECTED), (ACTUAL));		    \
   SELFTEST_END_STMT
 
-/* Like ASSERT_STREQ_AT, but treat LOC as the effective location of the
+/* Like ASSERT_STREQ, but treat LOC as the effective location of the
    selftest.  */
 
 #define ASSERT_STREQ_AT(LOC, EXPECTED, ACTUAL)			    \
diff --git a/gcc/testsuite/gcc.dg/plugin/diagnostic-test-string-literals-1.c b/gcc/testsuite/gcc.dg/plugin/diagnostic-test-string-literals-1.c
new file mode 100644
index 0000000..73c5b2b
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/plugin/diagnostic-test-string-literals-1.c
@@ -0,0 +1,172 @@
+/* { dg-do compile } */
+/* { dg-options "-O -fdiagnostics-show-caret" } */
+
+/* This is a collection of unittests for ranges within string literals,
+   using diagnostic_plugin_test_string_literals, which handles
+   "__emit_string_literal_range" by generating a warning at the given
+   subset of a string literal.
+
+   The indices are 0-based.  It's easiest to verify things using string
+   literals that are runs of 0-based digits (to avoid having to count
+   characters).  */
+
+extern void __emit_string_literal_range (const char *literal,
+					 int start_idx, int end_idx);
+
+void
+test_simple_string_literal (void)
+{
+  __emit_string_literal_range ("0123456789", /* { dg-warning "range" } */
+			       6, 7);
+/* { dg-begin-multiline-output "" }
+   __emit_string_literal_range ("0123456789",
+                                       ^~
+   { dg-end-multiline-output "" } */
+}
+
+void
+test_concatenated_string_literal (void)
+{
+  __emit_string_literal_range ("01234" "56789", /* { dg-warning "range" } */
+			       3, 6);
+/* { dg-begin-multiline-output "" }
+   __emit_string_literal_range ("01234" "56789",
+                                    ^~~~~~~
+   { dg-end-multiline-output "" } */
+}
+
+void
+test_multiline_string_literal (void)
+{
+  __emit_string_literal_range ("01234" /* { dg-warning "range" } */
+                               "56789",
+                               3, 6);
+/* { dg-begin-multiline-output "" }
+   __emit_string_literal_range ("01234"
+                                    ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+                                "56789",
+                                ~~~  
+   { dg-end-multiline-output "" } */
+  /* FIXME: why does the above need two trailing spaces?  */
+}
+
+/* Tests of various unicode encodings.
+
+   Digits 0 through 9 are unicode code points:
+      U+0030 DIGIT ZERO
+      ...
+      U+0039 DIGIT NINE
+   However, these are not always valid as UCN (see the comment in
+   libcpp/charset.c:_cpp_valid_ucn).
+
+   Hence we need to test UCN using an alternative unicode
+   representation of numbers; let's use Roman numerals,
+   (though these start at one, not zero):
+      U+2170 SMALL ROMAN NUMERAL ONE
+      ...
+      U+2174 SMALL ROMAN NUMERAL FIVE  ("v")
+      U+2175 SMALL ROMAN NUMERAL SIX   ("vi")
+      ...
+      U+2178 SMALL ROMAN NUMERAL NINE.  */
+
+void
+test_hex (void)
+{
+  /* Digits 0-9, expressing digit 5 in ASCII as "\x35"
+     and with a space in place of digit 6, to terminate the escaped
+     hex code.  */
+  __emit_string_literal_range ("01234\x35 789", /* { dg-warning "range" } */
+			       3, 7);
+/* { dg-begin-multiline-output "" }
+   __emit_string_literal_range ("01234\x35 789"
+                                    ^~~~~~~~
+   { dg-end-multiline-output "" } */
+}
+
+void
+test_oct (void)
+{
+  /* Digits 0-9, expressing digit 5 in ASCII as "\065"
+     and with a space in place of digit 6, to terminate the escaped
+     octal code.  */
+  __emit_string_literal_range ("01234\065 789", /* { dg-warning "range" } */
+			       3, 7);
+/* { dg-begin-multiline-output "" }
+   __emit_string_literal_range ("01234\065 789"
+                                    ^~~~~~~~
+   { dg-end-multiline-output "" } */
+}
+
+void
+test_multiple (void)
+{
+  /* Digits 0-9, expressing digit 5 in ASCII as hex "\x35"
+     digit 6 in ASCII as octal "\066", concatenating multiple strings.  */
+  __emit_string_literal_range ("01234"  "\x35"  "\066"  "789", /* { dg-warning "range" } */
+			       3, 8);
+/* { dg-begin-multiline-output "" }
+   __emit_string_literal_range ("01234"  "\x35"  "\066"  "789",
+                                    ^~~~~~~~~~~~~~~~~~~~~~~~
+   { dg-end-multiline-output "" } */
+}
+
+void
+test_ucn4 (void)
+{
+  /* Digits 0-9, expressing digits 5 and 6 as Roman numerals expressed
+     as UCN 4.
+     The resulting string is encoded as UTF-8.  Most of the digits are 1 byte
+     each, but digits 5 and 6 are encoded with 3 bytes each.
+     Hence to underline digits 4-7 we need to underling using bytes 4-11 in
+     the UTF-8 encoding.  */
+  __emit_string_literal_range ("01234\u2174\u2175789", /* { dg-warning "range" } */
+			       4, 11);
+/* { dg-begin-multiline-output "" }
+   __emit_string_literal_range ("01234\u2174\u2175789",
+                                     ^~~~~~~~~~~~~~
+   { dg-end-multiline-output "" } */
+}
+
+void
+test_ucn8 (void)
+{
+  /* Digits 0-9, expressing digits 5 and 6 as Roman numerals as UCN 8.
+     The resulting string is the same as as in test_ucn4 above, and hence
+     has the same UTF-8 encoding, and so we again need to underline bytes
+     4-11 in the UTF-8 encoding in order to underline digits 4-7.  */
+  __emit_string_literal_range ("01234\U00002174\U00002175789", /* { dg-warning "range" } */
+			       4, 11);
+/* { dg-begin-multiline-output "" }
+   __emit_string_literal_range ("01234\U00002174\U00002175789",
+                                     ^~~~~~~~~~~~~~~~~~~~~~
+   { dg-end-multiline-output "" } */
+}
+
+void
+test_u8 (void)
+{
+  /* Digits 0-9.  */
+  __emit_string_literal_range (u8"0123456789", /* { dg-warning "range" } */
+			       4, 7);
+/* { dg-begin-multiline-output "" }
+   __emit_string_literal_range (u8"0123456789",
+                                       ^~~~
+   { dg-end-multiline-output "" } */
+}
+
+void
+test_macro (void)
+{
+#define START "01234"  /* { dg-warning "range" } */
+  __emit_string_literal_range (START
+                               "56789",
+                               3, 6);
+/* { dg-begin-multiline-output "" }
+ #define START "01234"
+                   ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+   __emit_string_literal_range (START
+   ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+                                "56789",
+                                ~~~
+   { dg-end-multiline-output "" } */
+}
diff --git a/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_string_literals.c b/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_string_literals.c
new file mode 100644
index 0000000..44ec480
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_string_literals.c
@@ -0,0 +1,205 @@
+/* This plugin uses the diagnostics code to verify tracking of source code
+   locations within string literals.  */
+/* { dg-options "-O" } */
+
+#include "gcc-plugin.h"
+#include "config.h"
+#include "system.h"
+#include "coretypes.h"
+#include "tm.h"
+#include "tree.h"
+#include "stringpool.h"
+#include "toplev.h"
+#include "basic-block.h"
+#include "hash-table.h"
+#include "vec.h"
+#include "ggc.h"
+#include "basic-block.h"
+#include "tree-ssa-alias.h"
+#include "internal-fn.h"
+#include "gimple-fold.h"
+#include "tree-eh.h"
+#include "gimple-expr.h"
+#include "is-a.h"
+#include "gimple.h"
+#include "gimple-iterator.h"
+#include "tree.h"
+#include "tree-pass.h"
+#include "intl.h"
+#include "plugin-version.h"
+#include "diagnostic.h"
+#include "context.h"
+#include "print-tree.h"
+#include "cpplib.h"
+
+int plugin_is_GPL_compatible;
+
+/* A custom pass for printing string literal location information.  */
+
+const pass_data pass_data_test_string_literals =
+{
+  GIMPLE_PASS, /* type */
+  "test_string_literals", /* name */
+  OPTGROUP_NONE, /* optinfo_flags */
+  TV_NONE, /* tv_id */
+  PROP_ssa, /* properties_required */
+  0, /* properties_provided */
+  0, /* properties_destroyed */
+  0, /* todo_flags_start */
+  0, /* todo_flags_finish */
+};
+
+class pass_test_string_literals : public gimple_opt_pass
+{
+public:
+  pass_test_string_literals(gcc::context *ctxt)
+    : gimple_opt_pass(pass_data_test_string_literals, ctxt)
+  {}
+
+  /* opt_pass methods: */
+  bool gate (function *) { return true; }
+  virtual unsigned int execute (function *);
+
+}; // class pass_test_string_literals
+
+/* Determine if STMT is a call with NUM_ARGS arguments to a function
+   named FUNCNAME.
+   If so, return STMT as a gcall *.  Otherwise return NULL.  */
+
+static gcall *
+check_for_named_call (gimple *stmt,
+		      const char *funcname, unsigned int num_args)
+{
+  gcc_assert (funcname);
+
+  gcall *call = dyn_cast <gcall *> (stmt);
+  if (!call)
+    return NULL;
+
+  tree fndecl = gimple_call_fndecl (call);
+  if (!fndecl)
+    return NULL;
+
+  if (strcmp (IDENTIFIER_POINTER (DECL_NAME (fndecl)), funcname))
+    return NULL;
+
+  if (gimple_call_num_args (call) != num_args)
+    {
+      error_at (stmt->location, "expected number of args: %i (got %i)",
+		num_args, gimple_call_num_args (call));
+      return NULL;
+    }
+
+  return call;
+}
+
+/* Emit a warning covering SRC_RANGE, with the caret at the start of
+   SRC_RANGE.  */
+
+static void
+emit_warning (source_range src_range)
+{
+  location_t loc
+    = make_location (src_range.m_start, src_range.m_start, src_range.m_finish);
+  warning_at (loc, 0, "range %i:%i-%i:%i",
+	      LOCATION_LINE (src_range.m_start),
+	      LOCATION_COLUMN (src_range.m_start),
+	      LOCATION_LINE (src_range.m_finish),
+	      LOCATION_COLUMN (src_range.m_finish));
+}
+
+/* Support code for verifying that we are correctly tracking ranges
+   within string literals, for use by diagnostic-test-string-literals-*.c.
+   Emit a warning showing the range of a string literal, for each call to
+   a function named "__emit_string_literal_range".
+   The initial argument should be a string literal; arguments 2 and 3
+   should be integer constants, giving the range within the string
+   to be printed.  */
+
+static void
+test_string_literals (gimple *stmt)
+{
+  gcall *call = check_for_named_call (stmt, "__emit_string_literal_range", 3);
+  if (!call)
+    return;
+
+  /* We expect an ADDR_EXPR with a STRING_CST inside it for the
+     initial arg.  */
+  tree t_addr_string = gimple_call_arg (call, 0);
+  if (TREE_CODE (t_addr_string) != ADDR_EXPR)
+    {
+      error_at (call->location, "string literal required for arg 1");
+      return;
+    }
+
+  tree t_string = TREE_OPERAND (t_addr_string, 0);
+  if (TREE_CODE (t_string) != STRING_CST)
+    {
+      error_at (call->location, "string literal required for arg 1");
+      return;
+    }
+
+  tree t_start_idx = gimple_call_arg (call, 1);
+  if (TREE_CODE (t_start_idx) != INTEGER_CST)
+    {
+      error_at (call->location, "integer constant required for arg 2");
+      return;
+    }
+  int start_idx = TREE_INT_CST_LOW (t_start_idx);
+
+  tree t_end_idx = gimple_call_arg (call, 2);
+  if (TREE_CODE (t_end_idx) != INTEGER_CST)
+    {
+      error_at (call->location, "integer constant required for arg 3");
+      return;
+    }
+  int end_idx = TREE_INT_CST_LOW (t_end_idx);
+
+  cpp_string_location *strloc = TREE_STRING_LOCATION (t_string);
+  gcc_assert (strloc);
+  source_range src_range
+    = strloc->get_range_between_indices (start_idx, end_idx);
+  emit_warning (src_range);
+}
+
+/* Call test_string_literals on every statement within FUN.  */
+
+unsigned int
+pass_test_string_literals::execute (function *fun)
+{
+  gimple_stmt_iterator gsi;
+  basic_block bb;
+
+  FOR_EACH_BB_FN (bb, fun)
+    for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
+      {
+	gimple *stmt = gsi_stmt (gsi);
+	test_string_literals (stmt);
+      }
+
+  return 0;
+}
+
+/* Entrypoint for the plugin.  Create and register the custom pass.  */
+
+int
+plugin_init (struct plugin_name_args *plugin_info,
+	     struct plugin_gcc_version *version)
+{
+  struct register_pass_info pass_info;
+  const char *plugin_name = plugin_info->base_name;
+  int argc = plugin_info->argc;
+  struct plugin_argument *argv = plugin_info->argv;
+
+  if (!plugin_default_version_check (version, &gcc_version))
+    return 1;
+
+  pass_info.pass = new pass_test_string_literals (g);
+  pass_info.reference_pass_name = "ssa";
+  pass_info.ref_pass_instance_number = 1;
+  pass_info.pos_op = PASS_POS_INSERT_AFTER;
+  register_callback (plugin_name, PLUGIN_PASS_MANAGER_SETUP, NULL,
+		     &pass_info);
+
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.dg/plugin/plugin.exp b/gcc/testsuite/gcc.dg/plugin/plugin.exp
index f039c8d..3c2383a 100644
--- a/gcc/testsuite/gcc.dg/plugin/plugin.exp
+++ b/gcc/testsuite/gcc.dg/plugin/plugin.exp
@@ -71,6 +71,8 @@ set plugin_test_list [list \
 	  diagnostic-test-expressions-1.c } \
     { diagnostic_plugin_show_trees.c \
 	  diagnostic-test-show-trees-1.c } \
+    { diagnostic_plugin_test_string_literals.c \
+	  diagnostic-test-string-literals-1.c } \
     { location_overflow_plugin.c \
 	  location-overflow-test-1.c \
 	  location-overflow-test-2.c } \
diff --git a/gcc/tree-core.h b/gcc/tree-core.h
index db5b470..a5fe5be 100644
--- a/gcc/tree-core.h
+++ b/gcc/tree-core.h
@@ -1244,9 +1244,12 @@ struct GTY(()) tree_fixed_cst {
   struct fixed_value * fixed_cst_ptr;
 };
 
+struct cpp_string_location;
+
 struct GTY(()) tree_string {
   struct tree_typed typed;
   int length;
+  cpp_string_location *loc;
   char str[1];
 };
 
diff --git a/gcc/tree.c b/gcc/tree.c
index bc60190..0c13de1 100644
--- a/gcc/tree.c
+++ b/gcc/tree.c
@@ -62,6 +62,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "print-tree.h"
 #include "ipa-utils.h"
 #include "selftest.h"
+#include "cpplib.h"
 
 /* Tree code classes.  */
 
@@ -2003,12 +2004,35 @@ build_string (int len, const char *str)
   TREE_SET_CODE (s, STRING_CST);
   TREE_CONSTANT (s) = 1;
   TREE_STRING_LENGTH (s) = len;
+  TREE_STRING_LOCATION (s) = NULL;
   memcpy (s->string.str, str, len);
   s->string.str[len] = '\0';
 
   return s;
 }
 
+/* As above, but with per-character location information.  */
+
+struct cpp_string_location_stats cpp_string_location_stats;
+
+tree
+build_string (int len, const char *str, cpp_string_location *strloc)
+{
+  tree s = build_string (len, str);
+
+  /* Need to allocate a copy:  */
+  TREE_STRING_LOCATION (s) = ggc_alloc <cpp_string_location> ();
+  *TREE_STRING_LOCATION (s) = *strloc;
+
+  /* Maintain stats on string locations.  */
+  cpp_string_location_stats.count_all++;
+  if (strloc->trivial_p ())
+    cpp_string_location_stats.count_trivial++;
+
+  return s;
+}
+
+
 /* Return a newly constructed COMPLEX_CST node whose value is
    specified by the real and imaginary parts REAL and IMAG.
    Both REAL and IMAG should be constant nodes.  TYPE, if specified,
diff --git a/gcc/tree.h b/gcc/tree.h
index 99ed8ff..b83b9c9 100644
--- a/gcc/tree.h
+++ b/gcc/tree.h
@@ -1005,9 +1005,20 @@ extern void omp_clause_range_check_failed (const_tree, const char *, int,
 /* In a STRING_CST */
 /* In C terms, this is sizeof, not strlen.  */
 #define TREE_STRING_LENGTH(NODE) (STRING_CST_CHECK (NODE)->string.length)
+#define TREE_STRING_LOCATION(NODE) (STRING_CST_CHECK (NODE)->string.loc)
 #define TREE_STRING_POINTER(NODE) \
   ((const char *)(STRING_CST_CHECK (NODE)->string.str))
 
+extern struct cpp_string_location_stats
+{
+  /* How many have been used to construct STRING_CST.  */
+  int count_all;
+
+  /* How many of these consisted of a single run of 1-byte-per-char
+     bytes.  */
+  int count_trivial;
+} cpp_string_location_stats;
+
 /* In a COMPLEX_CST node.  */
 #define TREE_REALPART(NODE) (COMPLEX_CST_CHECK (NODE)->complex.real)
 #define TREE_IMAGPART(NODE) (COMPLEX_CST_CHECK (NODE)->complex.imag)
@@ -3957,6 +3968,7 @@ extern tree build_minus_one_cst (tree);
 extern tree build_all_ones_cst (tree);
 extern tree build_zero_cst (tree);
 extern tree build_string (int, const char *);
+extern tree build_string (int, const char *, cpp_string_location *);
 extern tree build_tree_list_stat (tree, tree MEM_STAT_DECL);
 #define build_tree_list(t, q) build_tree_list_stat (t, q MEM_STAT_INFO)
 extern tree build_tree_list_vec_stat (const vec<tree, va_gc> *MEM_STAT_DECL);
diff --git a/libcpp/charset.c b/libcpp/charset.c
index 2d07942..27a94da 100644
--- a/libcpp/charset.c
+++ b/libcpp/charset.c
@@ -99,6 +99,7 @@ struct _cpp_strbuf
   uchar *text;
   size_t asize;
   size_t len;
+  cpp_string_location loc;
 };
 
 /* This is enough to hold any string that fits on a single 80-column
@@ -453,7 +454,8 @@ one_utf16_to_utf8 (iconv_t bigend, const uchar **inbufp, size_t *inbytesleftp,
 static inline bool
 conversion_loop (int (*const one_conversion)(iconv_t, const uchar **, size_t *,
 					     uchar **, size_t *),
-		 iconv_t cd, const uchar *from, size_t flen, struct _cpp_strbuf *to)
+		 iconv_t cd, const uchar *from, size_t flen, struct _cpp_strbuf *to,
+		 cpp_string_location_reader *loc_reader)
 {
   const uchar *inbuf;
   uchar *outbuf;
@@ -468,8 +470,18 @@ conversion_loop (int (*const one_conversion)(iconv_t, const uchar **, size_t *,
   for (;;)
     {
       do
-	rval = one_conversion (cd, &inbuf, &inbytesleft,
-			       &outbuf, &outbytesleft);
+	{
+	  size_t old_outbytesleft = outbytesleft;
+	  rval = one_conversion (cd, &inbuf, &inbytesleft,
+				 &outbuf, &outbytesleft);
+	  if (loc_reader)
+	    {
+	      size_t bytes_written = outbytesleft - old_outbytesleft;
+	      to->loc.add_char_at (loc_reader->get_next (),
+				   bytes_written,
+				   loc_reader->get_line_maps ());
+	    }
+	}
       while (inbytesleft && !rval);
 
       if (__builtin_expect (inbytesleft == 0, 1))
@@ -494,7 +506,8 @@ conversion_loop (int (*const one_conversion)(iconv_t, const uchar **, size_t *,
 /* These functions convert entire strings between character sets.
    They all have the signature
 
-   bool (*)(iconv_t cd, const uchar *from, size_t flen, struct _cpp_strbuf *to);
+   bool (*)(iconv_t cd, const uchar *from, size_t flen, struct _cpp_strbuf *to,
+            cpp_string_location_reader *loc_reader, bool single_item);
 
    The input string FROM is converted as specified by the function
    name plus the iconv descriptor CD (which may be fake), and the
@@ -503,36 +516,42 @@ conversion_loop (int (*const one_conversion)(iconv_t, const uchar **, size_t *,
 /* These four use the custom conversion code above.  */
 static bool
 convert_utf8_utf16 (iconv_t cd, const uchar *from, size_t flen,
-		    struct _cpp_strbuf *to)
+		    struct _cpp_strbuf *to, cpp_string_location_reader *loc_reader,
+		    bool single_item ATTRIBUTE_UNUSED)
 {
-  return conversion_loop (one_utf8_to_utf16, cd, from, flen, to);
+  return conversion_loop (one_utf8_to_utf16, cd, from, flen, to, loc_reader);
 }
 
 static bool
 convert_utf8_utf32 (iconv_t cd, const uchar *from, size_t flen,
-		    struct _cpp_strbuf *to)
+		    struct _cpp_strbuf *to, cpp_string_location_reader *loc_reader,
+		    bool single_item ATTRIBUTE_UNUSED)
 {
-  return conversion_loop (one_utf8_to_utf32, cd, from, flen, to);
+  return conversion_loop (one_utf8_to_utf32, cd, from, flen, to, loc_reader);
 }
 
 static bool
 convert_utf16_utf8 (iconv_t cd, const uchar *from, size_t flen,
-		    struct _cpp_strbuf *to)
+		    struct _cpp_strbuf *to, cpp_string_location_reader *loc_reader,
+		    bool single_item ATTRIBUTE_UNUSED)
 {
-  return conversion_loop (one_utf16_to_utf8, cd, from, flen, to);
+  return conversion_loop (one_utf16_to_utf8, cd, from, flen, to, loc_reader);
 }
 
 static bool
 convert_utf32_utf8 (iconv_t cd, const uchar *from, size_t flen,
-		    struct _cpp_strbuf *to)
+		    struct _cpp_strbuf *to, cpp_string_location_reader *loc_reader,
+		    bool single_item ATTRIBUTE_UNUSED)
 {
-  return conversion_loop (one_utf32_to_utf8, cd, from, flen, to);
+  return conversion_loop (one_utf32_to_utf8, cd, from, flen, to, loc_reader);
 }
 
 /* Identity conversion, used when we have no alternative.  */
 static bool
 convert_no_conversion (iconv_t cd ATTRIBUTE_UNUSED,
-		       const uchar *from, size_t flen, struct _cpp_strbuf *to)
+		       const uchar *from, size_t flen, struct _cpp_strbuf *to,
+		       cpp_string_location_reader *loc_reader,
+		       bool single_item)
 {
   if (to->len + flen > to->asize)
     {
@@ -542,6 +561,10 @@ convert_no_conversion (iconv_t cd ATTRIBUTE_UNUSED,
     }
   memcpy (to->text + to->len, from, flen);
   to->len += flen;
+  if (single_item)
+    to->loc.add_n_chars_at (1, flen, loc_reader);
+  else
+    to->loc.add_n_chars_at (flen, 1, loc_reader);
   return true;
 }
 
@@ -559,7 +582,9 @@ convert_no_conversion (iconv_t cd ATTRIBUTE_UNUSED,
 
 static bool
 convert_using_iconv (iconv_t cd, const uchar *from, size_t flen,
-		     struct _cpp_strbuf *to)
+		     struct _cpp_strbuf *to,
+		     cpp_string_location_reader */*loc_reader*/,
+		     bool single_item ATTRIBUTE_UNUSED)
 {
   ICONV_CONST char *inbuf;
   char *outbuf;
@@ -606,8 +631,8 @@ convert_using_iconv (iconv_t cd, const uchar *from, size_t flen,
 /* Arrange for the above custom conversion logic to be used automatically
    when conversion between a suitable pair of character sets is requested.  */
 
-#define APPLY_CONVERSION(CONVERTER, FROM, FLEN, TO) \
-   CONVERTER.func (CONVERTER.cd, FROM, FLEN, TO)
+#define APPLY_CONVERSION(CONVERTER, FROM, FLEN, TO, LOC_READER, SINGLE_ITEM) \
+  CONVERTER.func (CONVERTER.cd, FROM, FLEN, TO, LOC_READER, SINGLE_ITEM)
 
 struct cpp_conversion
 {
@@ -792,8 +817,9 @@ cpp_host_to_exec_charset (cpp_reader *pfile, cppchar_t c)
   tbuf.asize = 1;
   tbuf.text = XNEWVEC (uchar, tbuf.asize);
   tbuf.len = 0;
+  tbuf.loc.init ();
 
-  if (!APPLY_CONVERSION (pfile->narrow_cset_desc, sbuf, 1, &tbuf))
+  if (!APPLY_CONVERSION (pfile->narrow_cset_desc, sbuf, 1, &tbuf, NULL, true))
     {
       cpp_errno (pfile, CPP_DL_ICE, "converting to execution character set");
       return 0;
@@ -985,7 +1011,9 @@ ucn_valid_in_identifier (cpp_reader *pfile, cppchar_t c,
 bool
 _cpp_valid_ucn (cpp_reader *pfile, const uchar **pstr,
 		const uchar *limit, int identifier_pos,
-		struct normalize_state *nst, cppchar_t *cp)
+		struct normalize_state *nst, cppchar_t *cp,
+		source_range *char_range,
+		cpp_string_location_reader *loc_reader)
 {
   cppchar_t result, c;
   unsigned int length;
@@ -1021,6 +1049,8 @@ _cpp_valid_ucn (cpp_reader *pfile, const uchar **pstr,
       if (!ISXDIGIT (c))
 	break;
       str++;
+      if (char_range)
+	char_range->m_finish = loc_reader->get_next ().m_finish;
       result = (result << 4) + hex_value (c);
     }
   while (--length && str < limit);
@@ -1090,7 +1120,9 @@ _cpp_valid_ucn (cpp_reader *pfile, const uchar **pstr,
    An advanced pointer is returned.  Issues all relevant diagnostics.  */
 static const uchar *
 convert_ucn (cpp_reader *pfile, const uchar *from, const uchar *limit,
-	     struct _cpp_strbuf *tbuf, struct cset_converter cvt)
+	     struct _cpp_strbuf *tbuf, struct cset_converter cvt,
+	     source_range char_range,
+	     cpp_string_location_reader *loc_reader)
 {
   cppchar_t ucn;
   uchar buf[6];
@@ -1100,7 +1132,12 @@ convert_ucn (cpp_reader *pfile, const uchar *from, const uchar *limit,
   struct normalize_state nst = INITIAL_NORMALIZE_STATE;
 
   from++;  /* Skip u/U.  */
-  _cpp_valid_ucn (pfile, &from, limit, 0, &nst, &ucn);
+
+  /* The u/U is part of the spelling of this character.  */
+  char_range.m_finish = loc_reader->get_next ().m_finish;
+
+  _cpp_valid_ucn (pfile, &from, limit, 0, &nst,
+		  &ucn, &char_range, loc_reader);
 
   rval = one_cppchar_to_utf8 (ucn, &bufp, &bytesleft);
   if (rval)
@@ -1109,9 +1146,22 @@ convert_ucn (cpp_reader *pfile, const uchar *from, const uchar *limit,
       cpp_errno (pfile, CPP_DL_ERROR,
 		 "converting UCN to source character set");
     }
-  else if (!APPLY_CONVERSION (cvt, buf, 6 - bytesleft, tbuf))
-    cpp_errno (pfile, CPP_DL_ERROR,
-	       "converting UCN to execution character set");
+  else
+    {
+      /* Set up a cpp_string_location_reader to supply a
+	 location for the single character, covering all of
+	 char_range.  */
+      cpp_string_location_reader buf_loc_reader
+	(char_range.m_start,
+	 (((char_range.m_finish - char_range.m_start)
+	   >> loc_reader->get_bits_per_col ()) + 1),
+	 loc_reader->get_bits_per_col (),
+	 loc_reader->get_line_maps ());
+      if (!APPLY_CONVERSION (cvt, buf, 6 - bytesleft, tbuf, &buf_loc_reader,
+			     true))
+	cpp_errno (pfile, CPP_DL_ERROR,
+		   "converting UCN to execution character set");
+    }
 
   return from;
 }
@@ -1174,7 +1224,9 @@ emit_numeric_escape (cpp_reader *pfile, cppchar_t n,
    number.  You can, e.g. generate surrogate pairs this way.  */
 static const uchar *
 convert_hex (cpp_reader *pfile, const uchar *from, const uchar *limit,
-	     struct _cpp_strbuf *tbuf, struct cset_converter cvt)
+	     struct _cpp_strbuf *tbuf, struct cset_converter cvt,
+	     source_range char_range,
+	     cpp_string_location_reader *loc_reader)
 {
   cppchar_t c, n = 0, overflow = 0;
   int digits_found = 0;
@@ -1185,13 +1237,19 @@ convert_hex (cpp_reader *pfile, const uchar *from, const uchar *limit,
     cpp_warning (pfile, CPP_W_TRADITIONAL,
 	         "the meaning of '\\x' is different in traditional C");
 
-  from++;  /* Skip 'x'.  */
+  /* Skip 'x'.  */
+  from++;
+
+  /* The 'x' is part of the spelling of this character.  */
+  char_range.m_finish = loc_reader->get_next ().m_finish;
+
   while (from < limit)
     {
       c = *from;
       if (! hex_p (c))
 	break;
       from++;
+      char_range.m_finish = loc_reader->get_next ().m_finish;
       overflow |= n ^ (n << 4 >> 4);
       n = (n << 4) + hex_value (c);
       digits_found = 1;
@@ -1213,6 +1271,9 @@ convert_hex (cpp_reader *pfile, const uchar *from, const uchar *limit,
 
   emit_numeric_escape (pfile, n, tbuf, cvt);
 
+  tbuf->loc.add_char_at (char_range, 1,
+			 pfile->line_table);
+
   return from;
 }
 
@@ -1224,7 +1285,9 @@ convert_hex (cpp_reader *pfile, const uchar *from, const uchar *limit,
    number.  */
 static const uchar *
 convert_oct (cpp_reader *pfile, const uchar *from, const uchar *limit,
-	     struct _cpp_strbuf *tbuf, struct cset_converter cvt)
+	     struct _cpp_strbuf *tbuf, struct cset_converter cvt,
+	     source_range char_range,
+	     cpp_string_location_reader *loc_reader)
 {
   size_t count = 0;
   cppchar_t c, n = 0;
@@ -1238,6 +1301,7 @@ convert_oct (cpp_reader *pfile, const uchar *from, const uchar *limit,
       if (c < '0' || c > '7')
 	break;
       from++;
+      char_range.m_finish = loc_reader->get_next ().m_finish;
       overflow |= n ^ (n << 3 >> 3);
       n = (n << 3) + c - '0';
     }
@@ -1251,6 +1315,9 @@ convert_oct (cpp_reader *pfile, const uchar *from, const uchar *limit,
 
   emit_numeric_escape (pfile, n, tbuf, cvt);
 
+  tbuf->loc.add_char_at (char_range, 1,
+			 pfile->line_table);
+
   return from;
 }
 
@@ -1260,7 +1327,8 @@ convert_oct (cpp_reader *pfile, const uchar *from, const uchar *limit,
    pointer.  Handles all relevant diagnostics.  */
 static const uchar *
 convert_escape (cpp_reader *pfile, const uchar *from, const uchar *limit,
-		struct _cpp_strbuf *tbuf, struct cset_converter cvt)
+		struct _cpp_strbuf *tbuf, struct cset_converter cvt,
+		cpp_string_location_reader *loc_reader)
 {
   /* Values of \a \b \e \f \n \r \t \v respectively.  */
 #if HOST_CHARSET == HOST_CHARSET_ASCII
@@ -1273,20 +1341,26 @@ convert_escape (cpp_reader *pfile, const uchar *from, const uchar *limit,
 
   uchar c;
 
+  /* Record the location of the backslash.  */
+  source_range char_range = loc_reader->get_next ();
+
   c = *from;
   switch (c)
     {
       /* UCNs, hex escapes, and octal escapes are processed separately.  */
     case 'u': case 'U':
-      return convert_ucn (pfile, from, limit, tbuf, cvt);
+      return convert_ucn (pfile, from, limit, tbuf, cvt,
+			  char_range, loc_reader);
 
     case 'x':
-      return convert_hex (pfile, from, limit, tbuf, cvt);
+      return convert_hex (pfile, from, limit, tbuf, cvt,
+			  char_range, loc_reader);
       break;
 
     case '0':  case '1':  case '2':  case '3':
     case '4':  case '5':  case '6':  case '7':
-      return convert_oct (pfile, from, limit, tbuf, cvt);
+      return convert_oct (pfile, from, limit, tbuf, cvt,
+			  char_range, loc_reader);
 
       /* Various letter escapes.  Get the appropriate host-charset
 	 value into C.  */
@@ -1339,7 +1413,7 @@ convert_escape (cpp_reader *pfile, const uchar *from, const uchar *limit,
     }
 
   /* Now convert what we have to the execution character set.  */
-  if (!APPLY_CONVERSION (cvt, &c, 1, tbuf))
+  if (!APPLY_CONVERSION (cvt, &c, 1, tbuf, loc_reader, true))
     cpp_errno (pfile, CPP_DL_ERROR,
 	       "converting escape sequence to execution character set");
 
@@ -1388,14 +1462,21 @@ cpp_interpret_string (cpp_reader *pfile, const cpp_string *from, size_t count,
   tbuf.asize = MAX (OUTBUF_BLOCK_SIZE, from->len);
   tbuf.text = XNEWVEC (uchar, tbuf.asize);
   tbuf.len = 0;
+  tbuf.loc.init ();
 
   for (i = 0; i < count; i++)
     {
+      cpp_string_location_reader loc_reader (&from[i].loc, pfile->line_table);
       p = from[i].text;
       if (*p == 'u')
 	{
-	  if (*++p == '8')
-	    p++;
+	  p++;
+	  loc_reader.get_next ();
+	  if (*p == '8')
+	    {
+	      p++;
+	      loc_reader.get_next ();
+	    }
 	}
       else if (*p == 'L' || *p == 'U') p++;
       if (*p == 'R')
@@ -1414,13 +1495,16 @@ cpp_interpret_string (cpp_reader *pfile, const cpp_string *from, size_t count,
 
 	  /* Raw strings are all normal characters; these can be fed
 	     directly to convert_cset.  */
-	  if (!APPLY_CONVERSION (cvt, p, limit - p, &tbuf))
+	  if (!APPLY_CONVERSION (cvt, p, limit - p, &tbuf, &loc_reader, true))
 	    goto fail;
 
 	  continue;
 	}
 
-      p++; /* Skip leading quote.  */
+      /* Skip leading quote.  */
+      p++;
+      loc_reader.get_next ();
+
       limit = from[i].text + from[i].len - 1; /* Skip trailing quote.  */
 
       for (;;)
@@ -1432,13 +1516,14 @@ cpp_interpret_string (cpp_reader *pfile, const cpp_string *from, size_t count,
 	    {
 	      /* We have a run of normal characters; these can be fed
 		 directly to convert_cset.  */
-	      if (!APPLY_CONVERSION (cvt, base, p - base, &tbuf))
+	      if (!APPLY_CONVERSION (cvt, base, p - base, &tbuf, &loc_reader,
+				     false))
 		goto fail;
 	    }
 	  if (p == limit)
 	    break;
 
-	  p = convert_escape (pfile, p + 1, limit, &tbuf, cvt);
+	  p = convert_escape (pfile, p + 1, limit, &tbuf, cvt, &loc_reader);
 	}
     }
   /* NUL-terminate the 'to' buffer and translate it to a cpp_string
@@ -1447,6 +1532,7 @@ cpp_interpret_string (cpp_reader *pfile, const cpp_string *from, size_t count,
   tbuf.text = XRESIZEVEC (uchar, tbuf.text, tbuf.len);
   to->text = tbuf.text;
   to->len = tbuf.len;
+  to->loc = tbuf.loc;
   return true;
 
  fail:
@@ -1611,7 +1697,7 @@ cppchar_t
 cpp_interpret_charconst (cpp_reader *pfile, const cpp_token *token,
 			 unsigned int *pchars_seen, int *unsignedp)
 {
-  cpp_string str = { 0, 0 };
+  cpp_string str = { 0, 0, {NULL, 0, 0} };
   bool wide = (token->type != CPP_CHAR && token->type != CPP_UTF8CHAR);
   int u8 = 2 * int(token->type == CPP_UTF8CHAR);
   cppchar_t result;
@@ -1726,14 +1812,16 @@ _cpp_convert_input (cpp_reader *pfile, const char *input_charset,
       to.text = input;
       to.asize = size;
       to.len = len;
+      to.loc.init ();
     }
   else
     {
       to.asize = MAX (65536, len);
       to.text = XNEWVEC (uchar, to.asize);
       to.len = 0;
+      to.loc.init ();
 
-      if (!APPLY_CONVERSION (input_cset, input, len, &to))
+      if (!APPLY_CONVERSION (input_cset, input, len, &to, NULL, false))
 	cpp_error (pfile, CPP_DL_ERROR,
 		   "failure to convert %s to %s",
 		   CPP_OPTION (pfile, input_charset), SOURCE_CHARSET);
@@ -1818,3 +1906,243 @@ _cpp_default_encoding (void)
 
   return current_encoding;
 }
+
+/* Implementation of class cpp_string_location and
+   class cpp_string_location_reader.
+   We put them in this source file in the hope that they can be
+   inlined into heavy users such as cpp_interpret_string without
+   requiring the compiler itself to be built with LTO.  */
+
+/* Initialize cpp_string_location.  This can't be a constructor since
+   cpp_string_location occurs inside a union.  Specifically, there
+   is a cpp_string_location within struct cpp_string, and the latter
+   is within union cpp_token_u within struct cpp_token.  */
+
+void
+cpp_string_location::init ()
+{
+  m_fragloc_array = NULL;
+  m_num_fraglocs = 0;
+  m_alloc_fraglocs = 0;
+}
+
+/* Another initializer for cpp_string_location; see the note above
+   for why this can't be a constructor.  */
+
+void
+cpp_string_location::init_raw (source_location loc, int len, int cols_per_item,
+			       int bytes_per_item, line_maps *line_table)
+{
+  if (loc > LINE_MAP_MAX_LOCATION_WITH_COLS)
+    cols_per_item = 0;
+
+  line_map_realloc reallocator = (line_table->reallocator
+				  ? line_table->reallocator
+				  : (line_map_realloc) xrealloc);
+  m_fragloc_array = (cpp_string_fragment_location *)reallocator
+    (NULL,
+     sizeof (cpp_string_fragment_location));
+  m_fragloc_array[0].m_len = len;
+
+  /* LOC might be a macro location.  It only makes sense to do
+     column-by-column calculations on ordinary maps, so get the
+     corresponding location in an ordinary map.  */
+  source_location ordinary_loc
+    = linemap_resolve_location (line_table, loc,
+				LRK_SPELLING_LOCATION, NULL);
+  m_fragloc_array[0].m_loc = ordinary_loc;
+  m_fragloc_array[0].m_cols_per_item = cols_per_item;
+  m_fragloc_array[0].m_bits_per_col
+    = LINEMAPS_LAST_ORDINARY_MAP (line_table)->m_range_bits;
+  m_fragloc_array[0].m_bytes_per_item = bytes_per_item;
+  m_num_fraglocs = 1;
+  m_alloc_fraglocs = 1;
+}
+
+
+/* FIXME: comment.  */
+/* FIXME: what if bytes_per_item is not 1?  */
+void
+cpp_string_location::add_char_at (source_range range, int bytes_per_item,
+				  line_maps *line_table)
+{
+  if (m_fragloc_array)
+    {
+      /* Is this a simple run-on character in the next column
+	 within the current fragment?  */
+      cpp_string_fragment_location *current_fragment
+	= get_current_fragment ();
+      source_range next_range = current_fragment->get_next_range ();
+      if (range.m_start == next_range.m_start
+	  && range.m_finish == next_range.m_finish
+	  && bytes_per_item == current_fragment->m_bytes_per_item)
+	/* If so, we can simply increase the length of the current
+	   fragment.  */
+	{
+	  current_fragment->m_len++;
+	  return;
+	}
+
+      /* We need to start a new fragment.  This may require growing
+	 the underlying array.  */
+      if (++m_num_fraglocs > m_alloc_fraglocs)
+	{
+	  m_alloc_fraglocs *= 2;
+	  line_map_realloc reallocator = (line_table->reallocator
+					  ? line_table->reallocator
+					  : (line_map_realloc) xrealloc);
+	  m_fragloc_array = (cpp_string_fragment_location *)reallocator
+	    (m_fragloc_array,
+	     sizeof (cpp_string_fragment_location) * m_alloc_fraglocs);
+	}
+    }
+  else
+    {
+      /* Begin new fragment array.  */
+      line_map_realloc reallocator = (line_table->reallocator
+				      ? line_table->reallocator
+				      : (line_map_realloc) xrealloc);
+      m_fragloc_array = (cpp_string_fragment_location *)reallocator
+	(NULL, sizeof (cpp_string_fragment_location));
+      m_num_fraglocs = 1;
+      m_alloc_fraglocs = 1;
+    }
+
+  cpp_string_fragment_location *current_fragment = get_current_fragment ();
+  current_fragment->m_len = 1;
+  current_fragment->m_loc = range.m_start;
+  current_fragment->m_bits_per_col
+    = LINEMAPS_LAST_ORDINARY_MAP (line_table)->m_range_bits;
+  /* FIXME: does this need reworking?  */
+  current_fragment->m_cols_per_item
+    = (((range.m_finish - range.m_start)
+	>> current_fragment->m_bits_per_col) + 1);
+  current_fragment->m_bytes_per_item = bytes_per_item;
+}
+
+/* FIXME.  */
+void
+cpp_string_location::add_n_chars_at (int flen, int bytes_per_item,
+				     cpp_string_location_reader *loc_reader)
+{
+  if (loc_reader)
+    while (flen--)
+      add_char_at (loc_reader->get_next (), bytes_per_item,
+		   loc_reader->get_line_maps ());
+}
+
+/* FIXME.  */
+source_location
+cpp_string_location::get_loc_at_index (unsigned int char_idx) const
+{
+  unsigned int fragment_idx = 0;
+  while (fragment_idx < m_num_fraglocs)
+    {
+      const cpp_string_fragment_location *fragment
+	= &m_fragloc_array[fragment_idx];
+      for (unsigned int item_idx = 0; item_idx < fragment->m_len; item_idx++)
+	{
+	  if (char_idx < fragment->m_bytes_per_item)
+	    return fragment->get_item_range (item_idx).m_start;
+	  char_idx -= fragment->m_bytes_per_item;
+	}
+      fragment_idx++;
+    }
+
+  /* Error: accessing beyond the end of the array.  */
+  return 0;
+}
+
+/* FIXME.  */
+source_range
+cpp_string_location::get_range_at_index (unsigned int char_idx) const
+{
+  unsigned int fragment_idx = 0;
+  while (fragment_idx < m_num_fraglocs)
+    {
+      const cpp_string_fragment_location *fragment
+	= &m_fragloc_array[fragment_idx];
+      for (unsigned int item_idx = 0; item_idx < fragment->m_len; item_idx++)
+	{
+	  if (char_idx < fragment->m_bytes_per_item)
+	    return fragment->get_item_range (item_idx);
+	  char_idx -= fragment->m_bytes_per_item;
+	}
+      fragment_idx++;
+    }
+
+  /* Error: accessing beyond the end of the array.  */
+  source_range err;
+  err.m_start = 0;
+  err.m_finish = 0;
+  return err;
+}
+
+/* FIXME. FINISH_IDX is within the range.  */
+source_range
+cpp_string_location::get_range_between_indices (unsigned int start_idx,
+						unsigned int finish_idx) const
+{
+  /* This could be optimized if necessary.  */
+  source_range result;
+  result.m_start = get_loc_at_index (start_idx);
+  result.m_finish = get_range_at_index (finish_idx).m_finish;
+  return result;
+}
+
+/* FIXME.  */
+bool
+cpp_string_location::trivial_p () const
+{
+  if (m_num_fraglocs == 1)
+    if (m_fragloc_array[0].m_cols_per_item == 1)
+      /* FIXME: do we also want to check bytes per item?  */
+      return true;
+  return false;
+}
+
+/* Constructor for iterating through the locations in
+   cpp_string_location.  */
+
+cpp_string_location_reader::
+cpp_string_location_reader (const cpp_string_location *strloc,
+			    line_maps *line_table)
+{
+  /* As an optimization, we require that STRLOC must consist of a
+     single fragment.  */
+  linemap_assert (strloc->m_num_fraglocs == 1);
+  m_loc = strloc->m_fragloc_array[0].m_loc;
+  m_cols_per_char = strloc->m_fragloc_array[0].m_cols_per_item;
+  m_bits_per_col = strloc->m_fragloc_array[0].m_bits_per_col;
+  m_line_table = line_table;
+}
+
+/* Constructor for iterating through an arbitrary buffer.  */
+
+cpp_string_location_reader::
+cpp_string_location_reader (source_location src_loc,
+			    int cols_per_char,
+			    int bits_per_col,
+			    line_maps *line_table)
+: m_cols_per_char (cols_per_char),
+  m_bits_per_col (bits_per_col),
+  m_line_table (line_table)
+{
+  /* LOC might be a macro location.  It only makes sense to do
+     column-by-column calculations on ordinary maps, so get the
+     corresponding location in an ordinary map.  */
+  m_loc
+    = linemap_resolve_location (line_table, src_loc,
+				LRK_SPELLING_LOCATION, NULL);
+}
+
+/* FIXME.  */
+source_range
+cpp_string_location_reader::get_next ()
+{
+  source_range result;
+  result.m_start = m_loc;
+  result.m_finish = m_loc + ((m_cols_per_char - 1) << m_bits_per_col);
+  m_loc += m_cols_per_char << m_bits_per_col;
+  return result;
+}
diff --git a/libcpp/directives.c b/libcpp/directives.c
index 772b835..b44a0af 100644
--- a/libcpp/directives.c
+++ b/libcpp/directives.c
@@ -950,7 +950,7 @@ do_line (cpp_reader *pfile)
   token = cpp_get_token (pfile);
   if (token->type == CPP_STRING)
     {
-      cpp_string s = { 0, 0 };
+      cpp_string s = { 0, 0, { NULL, 0, 0 } };
       if (cpp_interpret_string_notranslate (pfile, &token->val.str, 1,
 					    &s, CPP_STRING))
 	new_file = (const char *)s.text;
@@ -1007,7 +1007,7 @@ do_linemarker (cpp_reader *pfile)
   token = cpp_get_token (pfile);
   if (token->type == CPP_STRING)
     {
-      cpp_string s = { 0, 0 };
+      cpp_string s = { 0, 0, { NULL, 0, 0 } };
       if (cpp_interpret_string_notranslate (pfile, &token->val.str,
 					    1, &s, CPP_STRING))
 	new_file = (const char *)s.text;
diff --git a/libcpp/expr.c b/libcpp/expr.c
index 5cdca6f..072af5b 100644
--- a/libcpp/expr.c
+++ b/libcpp/expr.c
@@ -1228,6 +1228,8 @@ _cpp_parse_expr (cpp_reader *pfile, bool is_if)
 			      "missing binary operator before token \"%s\"",
 			      cpp_token_as_text (pfile, op.token));
 	  want_value = false;
+	  ((cpp_token *)op.token)->val.str.loc.init_raw (op.loc, 1, 1, 1, /* FIXME */
+							 pfile->line_table);
 	  top->value = eval_token (pfile, op.token, op.loc);
 	  continue;
 
diff --git a/libcpp/include/cpplib.h b/libcpp/include/cpplib.h
index 543f3b9..a6b4420 100644
--- a/libcpp/include/cpplib.h
+++ b/libcpp/include/cpplib.h
@@ -173,10 +173,166 @@ enum c_lang {CLK_GNUC89 = 0, CLK_GNUC99, CLK_GNUC11,
 	     CLK_GNUCXX, CLK_CXX98, CLK_GNUCXX11, CLK_CXX11,
 	     CLK_GNUCXX14, CLK_CXX14, CLK_GNUCXX1Z, CLK_CXX1Z, CLK_ASM};
 
+/* Location of the individual chars in a cpp_string.
+   Specifically, this stores a run of characters of len, starting at loc,
+   with a consistent number of columns per char.
+   See the description below for cpp_string_location.  */
+struct GTY(()) cpp_string_fragment_location {
+  source_location m_loc;
+  unsigned int m_len : 16;
+  unsigned int m_cols_per_item : 8;
+  unsigned int m_bits_per_col : 4;
+  unsigned int m_bytes_per_item : 4;
+
+  source_location get_item_start (int item_idx) const
+  {
+    return m_loc + ((item_idx * m_cols_per_item) << m_bits_per_col);
+  }
+
+  source_location get_item_finish (int item_idx) const
+  {
+    if (m_cols_per_item > 0)
+      return get_item_start (item_idx) + ((m_cols_per_item - 1) << m_bits_per_col);
+    else
+      return m_loc;
+  }
+
+  source_range get_item_range (int item_idx) const
+  {
+    source_range result;
+    result.m_start = get_item_start (item_idx);
+    result.m_finish = get_item_finish (item_idx);
+    return result;
+  }
+  source_range get_next_range () const
+  {
+    return get_item_range (m_len);
+  }
+  source_range get_covered_range () const
+  {
+    source_range result;
+    result.m_start = m_loc;
+    result.m_finish = get_item_finish (m_len - 1);
+    return result;
+  }
+  void debug (const char *msg) const;
+};
+
+class cpp_string_location_reader;
+
+/* Location of the individual chars in a cpp_string.
+   This is stored as a dynamically-allocated array of fragments.
+   For example, consider this call to printf:
+
+     printf ("foo \x25\151 bar"  "baz\n",
+             "not an int");
+
+   The string constant for the first parameter is composed of
+   the concatenation of two string literals, with hexadecimal
+   encoding of a '%' and octal encoding of a 'i', giving a
+   resulting STRING_CST of:
+
+     "foo %i barbaz\n"
+
+   We want to efficiently record the range of locations in the
+   source file of each character so that we can emit warnings about
+   the type mismatch between format specifier "%i" and the non-int
+   second argument.
+
+   We record the locations as a series of fragments, where within
+   each fragment we have a contiguous run of input characters with
+   a consistent number of columns per character.  In the example
+   above the fragments are:
+
+    printf ("foo \x25\151 bar"  "baz\n",
+    .........^^^^.....................: fragment 0: 4 chars at 1 col per char
+    .............^^^^^^^^.............: fragment 1: 2 chars at 4 cols per char
+    .....................^^^^.........: fragment 2: 4 chars at 1 col per char
+    .............................^^^..: fragment 3: 3 chars at 1 col per char
+    ................................^^: fragment 4: 1 char at 2 cols per char
+
+   Note that the hex and octal chars both happen to be 4 cols per char
+   and are contiguous, hence both end up being in fragment 1, whereas the
+   "bar" and "baz" aren't contiguous and hence have to be in separate
+   fragments.
+
+   Note also that having a constant cols-per-char within each fragment
+   means that given an index into the fragment we can directly compute
+   the corresponding source_range.  */
+
+struct GTY(()) cpp_string_location {
+
+  void init ();
+  void init_raw (source_location loc, int len, int cols_per_char,
+		 int bytes_per_item, line_maps *line_table);
+
+  void add_char_at (source_range range, int bytes_per_item,
+		    line_maps *line_table);
+  void add_n_chars_at (int flen, int bytes_per_item,
+		       cpp_string_location_reader *loc_reader);
+
+  source_location get_loc_at_index (unsigned int idx) const;
+  source_range get_range_at_index (unsigned int idx) const;
+  source_range get_range_between_indices (unsigned int start_idx,
+					  unsigned int finish_idx) const;
+
+  void debug () const;
+
+  bool trivial_p () const;
+
+ private:
+  cpp_string_fragment_location *get_current_fragment () const
+  {
+    return &m_fragloc_array[m_num_fraglocs - 1];
+  }
+
+  /* Fields.
+     Ideally we would make these fields private, but this isn't easily
+     doable since gengtype generates functions in gtype-desc.c that
+     access them.  */
+ public:
+
+  /* We seemingly can't use vec<> from libcpp, so do it "by hand"
+     here.  */
+  cpp_string_fragment_location *m_fragloc_array;
+  unsigned int m_num_fraglocs;
+  unsigned int m_alloc_fraglocs;
+};
+
+/* A class for iterating through the source-locations within a
+   string, either from a cpp_string_location, or a temporary buffer.  */
+class cpp_string_location_reader {
+ public:
+  /* Constructor for iterating through the locations in
+     cpp_string_location.
+     As an optimization, we require that STRLOC must consist of a
+     single fragment.  */
+  cpp_string_location_reader (const cpp_string_location *strloc,
+			      line_maps *line_table);
+
+  /* Constructor for iterating through an arbitrary buffer.  */
+  cpp_string_location_reader (source_location src_loc,
+			      int cols_per_char,
+			      int bits_per_col,
+			      line_maps *line_table);
+
+  source_range get_next ();
+
+  int get_bits_per_col () const { return m_bits_per_col; }
+  line_maps *get_line_maps () const { return m_line_table; }
+
+ private:
+  source_location m_loc;
+  int m_cols_per_char;
+  int m_bits_per_col;
+  line_maps *m_line_table;
+};
+
 /* Payload of a NUMBER, STRING, CHAR or COMMENT token.  */
 struct GTY(()) cpp_string {
   unsigned int len;
   const unsigned char *text;
+  cpp_string_location loc;
 };
 
 /* Flags for the cpp_token structure.  */
diff --git a/libcpp/internal.h b/libcpp/internal.h
index ca2b498..0a6b472 100644
--- a/libcpp/internal.h
+++ b/libcpp/internal.h
@@ -42,7 +42,8 @@ struct op;
 struct _cpp_strbuf;
 
 typedef bool (*convert_f) (iconv_t, const unsigned char *, size_t,
-			   struct _cpp_strbuf *);
+			   struct _cpp_strbuf *,
+			   cpp_string_location_reader *loc_reader, bool);
 struct cset_converter
 {
   convert_f func;
@@ -754,7 +755,9 @@ struct normalize_state
 extern bool _cpp_valid_ucn (cpp_reader *, const unsigned char **,
 			    const unsigned char *, int,
 			    struct normalize_state *state,
-			    cppchar_t *);
+			    cppchar_t *,
+			    source_range *char_range,
+			    cpp_string_location_reader *loc_reader);
 extern void _cpp_destroy_iconv (cpp_reader *);
 extern unsigned char *_cpp_convert_input (cpp_reader *, const char *,
 					  unsigned char *, size_t, size_t,
diff --git a/libcpp/lex.c b/libcpp/lex.c
index 236418d..0bbd952 100644
--- a/libcpp/lex.c
+++ b/libcpp/lex.c
@@ -1247,7 +1247,7 @@ forms_identifier_p (cpp_reader *pfile, int first,
       cppchar_t s;
       buffer->cur += 2;
       if (_cpp_valid_ucn (pfile, &buffer->cur, buffer->rlimit, 1 + !first,
-			  state, &s))
+			  state, &s, NULL, NULL))
 	return true;
       buffer->cur -= 2;
     }
@@ -1407,6 +1407,15 @@ lex_number (cpp_reader *pfile, cpp_string *number,
   const uchar *base;
   uchar *dest;
 
+  /* FIXME: should it really use a new "cpp_number", rather than cpp_string?
+     We need to init this, or we get a crash accessing uninited data
+     during GC, since,
+       struct GTY(()) cpp_token
+     has union cpp_token_u with
+       desc ("cpp_token_val_index (&%1)")))
+     and this gives CPP_TOKEN_FLD_STR for numbers (and strings).  */
+  number->loc.init ();
+
   base = pfile->buffer->cur - 1;
   do
     {
@@ -1446,6 +1455,7 @@ create_literal (cpp_reader *pfile, cpp_token *token, const uchar *base,
   token->type = type;
   token->val.str.len = len;
   token->val.str.text = dest;
+  token->val.str.loc.init_raw (token->src_loc, len, 1, 1, pfile->line_table);
 }
 
 /* Subroutine of lex_raw_string: Append LEN chars from BASE to the buffer
diff --git a/libcpp/macro.c b/libcpp/macro.c
index a3b8348..1a6c869 100644
--- a/libcpp/macro.c
+++ b/libcpp/macro.c
@@ -217,6 +217,7 @@ new_string_token (cpp_reader *pfile, unsigned char *text, unsigned int len)
   token->type = CPP_STRING;
   token->val.str.len = len;
   token->val.str.text = text;
+  token->val.str.loc.init ();
   token->flags = 0;
   return token;
 }
-- 
1.8.5.3

