From 4a25106c3ad35f27dac4cc99c4a5f407d4d0a265 Mon Sep 17 00:00:00 2001 From: David Malcolm Date: Tue, 24 Feb 2026 17:44:31 -0500 Subject: [PATCH 21/38] Introduce pretty-print-token-buffer.{cc,h} Move the implementation of diagnostic_message_buffer from libdiagnostics to a new pretty-print-token-buffer.{cc,h}, for capturing the tokens from a pretty-print. Implement a new class pp_token_buffer_element for replaying the tokens in a pretty_print_token_buffer into another pretty-print, using "%e". Add selftests. gcc/ChangeLog: * Makefile.in (OBJS-libcommon): Add pretty-print-token-buffer.o. * libgdiagnostics.cc: Drop include of "auto-obstack.h". Include "pretty-print-token-buffer.h". (class copying_token_printer): Move to pretty-print-token-buffer.cc. (struct diagnostic_message_buffer): Reimplement as a subclass of pretty_print_token_buffer. (diagnostic_message_buffer::to_string): Rename to pretty_print_token_buffer::to_string and move to pretty-print-token-buffer.cc. * pretty-print-token-buffer.cc: New file, based on material from libgdiagnostics.cc. * pretty-print-token-buffer.h: New file, based on material from libgdiagnostics.h. * selftest-run-tests.cc (selftest::run_tests): Call selftest::pretty_print_token_buffer_cc_tests. * selftest.h (selftest::pretty_print_token_buffer_cc_tests): New decl. Signed-off-by: David Malcolm --- gcc/Makefile.in | 2 +- gcc/libgdiagnostics.cc | 177 +-------------- gcc/pretty-print-token-buffer.cc | 362 +++++++++++++++++++++++++++++++ gcc/pretty-print-token-buffer.h | 70 ++++++ gcc/selftest-run-tests.cc | 1 + gcc/selftest.h | 1 + 6 files changed, 440 insertions(+), 173 deletions(-) create mode 100644 gcc/pretty-print-token-buffer.cc create mode 100644 gcc/pretty-print-token-buffer.h diff --git a/gcc/Makefile.in b/gcc/Makefile.in index 807e5751642..4047bac2808 100644 --- a/gcc/Makefile.in +++ b/gcc/Makefile.in @@ -1904,7 +1904,7 @@ OBJS-libcommon = \ diagnostics/diagnostics-selftests.o \ gcc-diagnostic-spec.o \ graphviz.o pex.o \ - pretty-print.o intl.o \ + pretty-print.o pretty-print-token-buffer.o intl.o \ json.o json-parsing.o json-diagnostic.o \ pub-sub.o \ xml.o \ diff --git a/gcc/libgdiagnostics.cc b/gcc/libgdiagnostics.cc index dd5109229b3..1cf451273d0 100644 --- a/gcc/libgdiagnostics.cc +++ b/gcc/libgdiagnostics.cc @@ -44,7 +44,7 @@ along with GCC; see the file COPYING3. If not see #include "libgdiagnostics-private.h" #include "pretty-print-format-impl.h" #include "pretty-print-markup.h" -#include "auto-obstack.h" +#include "pretty-print-token-buffer.h" class owned_nullable_string { @@ -252,97 +252,6 @@ private: diagnostics::source_printing_options m_source_printing; }; -/* A token_printer that makes a deep copy of the pp_token_list - into another obstack. */ - -class copying_token_printer : public token_printer -{ -public: - copying_token_printer (obstack &dst_obstack, - pp_token_list &dst_token_list) - : m_dst_obstack (dst_obstack), - m_dst_token_list (dst_token_list) - { - } - - void - print_tokens (pretty_printer *, - const pp_token_list &tokens) final override - { - for (auto iter = tokens.m_first; iter; iter = iter->m_next) - switch (iter->m_kind) - { - default: - gcc_unreachable (); - - case pp_token::kind::text: - { - const pp_token_text *sub = as_a (iter); - /* Copy the text, with null terminator. */ - obstack_grow (&m_dst_obstack, sub->m_value.get (), - strlen (sub->m_value.get ()) + 1); - m_dst_token_list.push_back_text - (label_text::borrow (XOBFINISH (&m_dst_obstack, - const char *))); - } - break; - - case pp_token::kind::begin_color: - { - pp_token_begin_color *sub = as_a (iter); - /* Copy the color, with null terminator. */ - obstack_grow (&m_dst_obstack, sub->m_value.get (), - strlen (sub->m_value.get ()) + 1); - m_dst_token_list.push_back - (label_text::borrow (XOBFINISH (&m_dst_obstack, - const char *))); - } - break; - case pp_token::kind::end_color: - m_dst_token_list.push_back (); - break; - - case pp_token::kind::begin_quote: - m_dst_token_list.push_back (); - break; - case pp_token::kind::end_quote: - m_dst_token_list.push_back (); - break; - - case pp_token::kind::begin_url: - { - pp_token_begin_url *sub = as_a (iter); - /* Copy the URL, with null terminator. */ - obstack_grow (&m_dst_obstack, sub->m_value.get (), - strlen (sub->m_value.get ()) + 1); - m_dst_token_list.push_back - (label_text::borrow (XOBFINISH (&m_dst_obstack, - const char *))); - } - break; - case pp_token::kind::end_url: - m_dst_token_list.push_back (); - break; - - case pp_token::kind::event_id: - { - pp_token_event_id *sub = as_a (iter); - m_dst_token_list.push_back (sub->m_event_id); - } - break; - - case pp_token::kind::custom_data: - /* These should have been eliminated by replace_custom_tokens. */ - gcc_unreachable (); - break; - } - } - -private: - obstack &m_dst_obstack; - pp_token_list &m_dst_token_list; -}; - class sarif_sink : public sink { public: @@ -352,31 +261,13 @@ public: const diagnostics::sarif_generation_options &sarif_gen_opts); }; -struct diagnostic_message_buffer +struct diagnostic_message_buffer : public pretty_print_token_buffer { - diagnostic_message_buffer () - : m_tokens (m_obstack) - { - } - - diagnostic_message_buffer (const char *gmsgid, - va_list *args) - : m_tokens (m_obstack) + diagnostic_message_buffer () {} + diagnostic_message_buffer (const char *gmsgid, va_list *args) + : pretty_print_token_buffer (gmsgid, args) { - text_info text (gmsgid, args, errno); - pretty_printer pp; - pp.set_output_stream (nullptr); - copying_token_printer tok_printer (m_obstack, m_tokens); - pp.set_token_printer (&tok_printer); - pp_format (&pp, &text); - pp_output_formatted_text (&pp, nullptr); } - - - std::string to_string () const; - - auto_obstack m_obstack; - pp_token_list m_tokens; }; /* A pp_element subclass that replays the saved tokens in a @@ -1566,64 +1457,6 @@ sarif_sink (diagnostic_manager &mgr, mgr.get_dc ().add_sink (std::move (inner_sink)); } -// struct diagnostic_message_buffer - -std::string -diagnostic_message_buffer::to_string () const -{ - std::string result; - - /* Convert to text, dropping colorization, URLs, etc. */ - for (auto iter = m_tokens.m_first; iter; iter = iter->m_next) - switch (iter->m_kind) - { - default: - gcc_unreachable (); - - case pp_token::kind::text: - { - pp_token_text *sub = as_a (iter); - result += sub->m_value.get (); - } - break; - - case pp_token::kind::begin_color: - case pp_token::kind::end_color: - // Skip - break; - - case pp_token::kind::begin_quote: - result += open_quote; - break; - - case pp_token::kind::end_quote: - result += close_quote; - break; - - case pp_token::kind::begin_url: - case pp_token::kind::end_url: - // Skip - break; - - case pp_token::kind::event_id: - { - pp_token_event_id *sub = as_a (iter); - gcc_assert (sub->m_event_id.known_p ()); - result += '('; - result += std::to_string (sub->m_event_id.one_based ()); - result += ')'; - } - break; - - case pp_token::kind::custom_data: - /* We don't have a way of handling custom_data tokens here. */ - gcc_unreachable (); - break; - } - - return result; -} - /* struct diagnostic_manager. */ void diff --git a/gcc/pretty-print-token-buffer.cc b/gcc/pretty-print-token-buffer.cc new file mode 100644 index 00000000000..61c4a7608b3 --- /dev/null +++ b/gcc/pretty-print-token-buffer.cc @@ -0,0 +1,362 @@ +/* Capturing the results of pretty_print for later playback. + Copyright (C) 2023-2026 Free Software Foundation, Inc. + Contributed by David Malcolm . + +This file is part of GCC. + +GCC is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free +Software Foundation; either version 3, or (at your option) any later +version. + +GCC is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +for more details. + +You should have received a copy of the GNU General Public License +along with GCC; see the file COPYING3. If not see +. */ + +#include "config.h" +#define INCLUDE_STRING +#include "system.h" +#include "coretypes.h" +#include "intl.h" +#include "pretty-print.h" +#include "pretty-print-token-buffer.h" +#include "selftest.h" + +/* A token_printer that makes a deep copy of the pp_token_list + into another obstack. */ + +class copying_token_printer : public token_printer +{ +public: + copying_token_printer (obstack &dst_obstack, + pp_token_list &dst_token_list) + : m_dst_obstack (dst_obstack), + m_dst_token_list (dst_token_list) + { + } + + void + print_tokens (pretty_printer *, + const pp_token_list &tokens) final override + { + for (auto iter = tokens.m_first; iter; iter = iter->m_next) + switch (iter->m_kind) + { + default: + gcc_unreachable (); + + case pp_token::kind::text: + { + const pp_token_text *sub = as_a (iter); + /* Copy the text, with null terminator. */ + obstack_grow (&m_dst_obstack, sub->m_value.get (), + strlen (sub->m_value.get ()) + 1); + m_dst_token_list.push_back_text + (label_text::borrow (XOBFINISH (&m_dst_obstack, + const char *))); + } + break; + + case pp_token::kind::begin_color: + { + pp_token_begin_color *sub = as_a (iter); + /* Copy the color, with null terminator. */ + obstack_grow (&m_dst_obstack, sub->m_value.get (), + strlen (sub->m_value.get ()) + 1); + m_dst_token_list.push_back + (label_text::borrow (XOBFINISH (&m_dst_obstack, + const char *))); + } + break; + case pp_token::kind::end_color: + m_dst_token_list.push_back (); + break; + + case pp_token::kind::begin_quote: + m_dst_token_list.push_back (); + break; + case pp_token::kind::end_quote: + m_dst_token_list.push_back (); + break; + + case pp_token::kind::begin_url: + { + pp_token_begin_url *sub = as_a (iter); + /* Copy the URL, with null terminator. */ + obstack_grow (&m_dst_obstack, sub->m_value.get (), + strlen (sub->m_value.get ()) + 1); + m_dst_token_list.push_back + (label_text::borrow (XOBFINISH (&m_dst_obstack, + const char *))); + } + break; + case pp_token::kind::end_url: + m_dst_token_list.push_back (); + break; + + case pp_token::kind::event_id: + { + pp_token_event_id *sub = as_a (iter); + m_dst_token_list.push_back (sub->m_event_id); + } + break; + + case pp_token::kind::custom_data: + /* These should have been eliminated by replace_custom_tokens. */ + gcc_unreachable (); + break; + } + } + +private: + obstack &m_dst_obstack; + pp_token_list &m_dst_token_list; +}; + +pretty_print_token_buffer::pretty_print_token_buffer () +: m_obstack (std::make_unique ()), + m_tokens (*m_obstack.get ()) +{ +} + +/* Capture GMSGID and ARGS as a sequence of pretty_print tokens. */ + +pretty_print_token_buffer::pretty_print_token_buffer (const char *gmsgid, + va_list *args) +: m_obstack (std::make_unique ()), + m_tokens (*m_obstack.get ()) +{ + text_info text (gmsgid, args, errno); + pretty_printer pp; + pp.set_output_stream (nullptr); + copying_token_printer tok_printer (*m_obstack.get (), m_tokens); + pp.set_token_printer (&tok_printer); + pp_format (&pp, &text); + pp_output_formatted_text (&pp, nullptr); +} + +pretty_print_token_buffer:: +pretty_print_token_buffer (pretty_print_token_buffer &&other) +: m_obstack (std::move (other.m_obstack)), + m_tokens (std::move (other.m_tokens)) +{ +} + +/* Convert to text, dropping colorization, URLs, etc. */ + +std::string +pretty_print_token_buffer::to_string () const +{ + std::string result; + + for (auto iter = m_tokens.m_first; iter; iter = iter->m_next) + switch (iter->m_kind) + { + default: + gcc_unreachable (); + + case pp_token::kind::text: + { + pp_token_text *sub = as_a (iter); + result += sub->m_value.get (); + } + break; + + case pp_token::kind::begin_color: + case pp_token::kind::end_color: + // Skip + break; + + case pp_token::kind::begin_quote: + result += open_quote; + break; + + case pp_token::kind::end_quote: + result += close_quote; + break; + + case pp_token::kind::begin_url: + case pp_token::kind::end_url: + // Skip + break; + + case pp_token::kind::event_id: + { + pp_token_event_id *sub = as_a (iter); + gcc_assert (sub->m_event_id.known_p ()); + result += '('; + result += std::to_string (sub->m_event_id.one_based ()); + result += ')'; + } + break; + + case pp_token::kind::custom_data: + /* We don't have a way of handling custom_data tokens here. */ + gcc_unreachable (); + break; + } + + return result; +} + +// class pp_token_buffer_element : public pp_element + +void +pp_token_buffer_element::add_to_phase_2 (pp_markup::context &ctxt) +{ + for (auto iter = m_token_buf.m_tokens.m_first; iter; iter = iter->m_next) + switch (iter->m_kind) + { + default: + gcc_unreachable (); + + case pp_token::kind::text: + { + const pp_token_text *sub = as_a (iter); + pp_string (&ctxt.m_pp, sub->m_value.get ()); + ctxt.push_back_any_text (); + } + break; + + case pp_token::kind::begin_color: + { + const pp_token_begin_color *sub + = as_a (iter); + gcc_assert (sub->m_value.get ()); + ctxt.begin_highlight_color (sub->m_value.get ()); + } + break; + + case pp_token::kind::end_color: + ctxt.end_highlight_color (); + break; + + case pp_token::kind::begin_quote: + ctxt.begin_quote (); + break; + + case pp_token::kind::end_quote: + ctxt.end_quote (); + break; + + case pp_token::kind::begin_url: + { + const pp_token_begin_url *sub + = as_a (iter); + gcc_assert (sub->m_value.get ()); + ctxt.begin_url (sub->m_value.get ()); + } + break; + + case pp_token::kind::end_url: + ctxt.end_url (); + break; + + case pp_token::kind::event_id: + { + const pp_token_event_id *sub + = as_a (iter); + gcc_assert (sub->m_event_id.known_p ()); + ctxt.add_event_id (sub->m_event_id); + } + break; + + case pp_token::kind::custom_data: + /* We don't have a way of handling custom_data tokens here. */ + gcc_unreachable (); + break; + } +} + +#if CHECKING_P + +namespace selftest { + +static pretty_print_token_buffer +pp_printf_to_buf (const char *fmt, ...) +{ + va_list args; + va_start (args, fmt); + + pretty_print_token_buffer buf (fmt, &args); + + va_end (args); + + return buf; +} + +static void +test_empty () +{ + pretty_print_token_buffer buf; + pp_token_buffer_element e (buf); + pretty_printer pp; + pp_printf (&pp, "before %e after", &e); + ASSERT_STREQ (pp_formatted_text (&pp), "before after"); +} + +static void +test_print () +{ + pretty_print_token_buffer buf + = pp_printf_to_buf ("x: %qs y: %qs", "foo", "bar"); + + // Check that the individual tokens are captured in "buf". + pp_token *tok0 = buf.m_tokens.m_first; + ASSERT_EQ (tok0->m_kind, pp_token::kind::text); + ASSERT_STREQ (((pp_token_text *)tok0)->m_value.get (), + "x: "); + + pp_token *tok1 = tok0->m_next; + ASSERT_EQ (tok1->m_kind, pp_token::kind::begin_quote); + + pp_token *tok2 = tok1->m_next; + ASSERT_EQ (tok2->m_kind, pp_token::kind::text); + ASSERT_STREQ (((pp_token_text *)tok2)->m_value.get (), + "foo"); + + pp_token *tok3 = tok2->m_next; + ASSERT_EQ (tok3->m_kind, pp_token::kind::end_quote); + + pp_token *tok4 = tok3->m_next; + ASSERT_EQ (tok4->m_kind, pp_token::kind::text); + ASSERT_STREQ (((pp_token_text *)tok4)->m_value.get (), + " y: "); + + pp_token *tok5 = tok4->m_next; + ASSERT_EQ (tok5->m_kind, pp_token::kind::begin_quote); + + pp_token *tok6 = tok5->m_next; + ASSERT_EQ (tok6->m_kind, pp_token::kind::text); + ASSERT_STREQ (((pp_token_text *)tok6)->m_value.get (), + "bar"); + + pp_token *tok7 = tok6->m_next; + ASSERT_EQ (tok7->m_kind, pp_token::kind::end_quote); + ASSERT_EQ (tok7->m_next, nullptr); + + + // Check that we can replay buf via pp_token_buffer_element + pp_token_buffer_element e (buf); + pretty_printer pp; + pp_printf (&pp, "before %e after", &e); + ASSERT_STREQ (pp_formatted_text (&pp), "before x: 'foo' y: 'bar' after"); +} + +/* Run all of the selftests within this file. */ + +void +pretty_print_token_buffer_cc_tests () +{ + test_empty (); + test_print (); +} + +} // namespace selftest + +#endif /* CHECKING_P */ diff --git a/gcc/pretty-print-token-buffer.h b/gcc/pretty-print-token-buffer.h new file mode 100644 index 00000000000..cab691d2c3f --- /dev/null +++ b/gcc/pretty-print-token-buffer.h @@ -0,0 +1,70 @@ +/* Capturing the results of pretty_print for later playback. + Copyright (C) 2023-2026 Free Software Foundation, Inc. + Contributed by David Malcolm . + +This file is part of GCC. + +GCC is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free +Software Foundation; either version 3, or (at your option) any later +version. + +GCC is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +for more details. + +You should have received a copy of the GNU General Public License +along with GCC; see the file COPYING3. If not see +. */ + +#ifndef GCC_PRETTY_PRINT_TOKEN_BUFFER_H +#define GCC_PRETTY_PRINT_TOKEN_BUFFER_H + +#include "pretty-print-format-impl.h" +#include "auto-obstack.h" +#include "pretty-print-markup.h" + +/* A class for capturing the results of pretty-printing as tokens, + potentially for playback into a different pretty-printer. */ + +class pretty_print_token_buffer +{ +public: + pretty_print_token_buffer (); + pretty_print_token_buffer (const char *gmsgid, + va_list *args); + + pretty_print_token_buffer (const pretty_print_token_buffer &) = delete; + pretty_print_token_buffer (pretty_print_token_buffer &&); + + ~pretty_print_token_buffer () = default; + + std::string to_string () const; + + void dump (FILE *out) const { m_tokens.dump (out); } + void DEBUG_FUNCTION dump () const { dump (stderr); } + + std::unique_ptr m_obstack; + pp_token_list m_tokens; +}; + +/* A pp_element subclass for use with "%e" that replays the buffered tokens + from TOKEN_BUF in another formatting call. */ + +class pp_token_buffer_element : public pp_element +{ +public: + pp_token_buffer_element (const pretty_print_token_buffer &token_buf) + : m_token_buf (token_buf) + { + } + + void + add_to_phase_2 (pp_markup::context &ctxt) final override; + +private: + const pretty_print_token_buffer &m_token_buf; +}; + +#endif /* GCC_PRETTY_PRINT_TOKEN_BUFFER_H */ diff --git a/gcc/selftest-run-tests.cc b/gcc/selftest-run-tests.cc index 7ead010360f..86b2340e9b9 100644 --- a/gcc/selftest-run-tests.cc +++ b/gcc/selftest-run-tests.cc @@ -68,6 +68,7 @@ selftest::run_tests () hash_set_tests_cc_tests (); vec_cc_tests (); pretty_print_cc_tests (); + pretty_print_token_buffer_cc_tests (); wide_int_cc_tests (); ggc_tests_cc_tests (); sreal_cc_tests (); diff --git a/gcc/selftest.h b/gcc/selftest.h index 462786dab2f..e3e86ffaf0f 100644 --- a/gcc/selftest.h +++ b/gcc/selftest.h @@ -246,6 +246,7 @@ extern void ordered_hash_map_tests_cc_tests (); extern void path_coverage_cc_tests (); extern void predict_cc_tests (); extern void pretty_print_cc_tests (); +extern void pretty_print_token_buffer_cc_tests (); extern void pub_sub_cc_tests (); extern void range_op_tests (); extern void range_tests (); -- 2.49.0